diff --git a/backend/k8sman.py b/backend/k8sman.py index a64b5b7e..6bd16ef4 100644 --- a/backend/k8sman.py +++ b/backend/k8sman.py @@ -46,6 +46,8 @@ class K8SManager: self.no_delete_jobs = os.environ.get("NO_DELETE_JOBS", "0") != "0" + self.grace_period = int(os.environ.get("GRACE_PERIOD_SECS", "600")) + self.redis_url = os.environ["REDIS_URL"] self.loop = asyncio.get_running_loop() @@ -690,7 +692,7 @@ class K8SManager: requests_memory = "384M" limit_memory = "2G" - requests_cpu = "240m" + requests_cpu = "500m" limit_cpu = "2000m" resources = { @@ -728,7 +730,11 @@ class K8SManager: "mountPath": "/tmp/crawl-config.json", "subPath": "crawl-config.json", "readOnly": True, - } + }, + { + "name": "crawl-data", + "mountPath": "/crawls", + }, ], "envFrom": [ {"configMapRef": {"name": "shared-crawler-config"}}, @@ -764,9 +770,11 @@ class K8SManager: } ], }, - } + }, + {"name": "crawl-data", "emptyDir": {}}, ], "restartPolicy": "OnFailure", + "terminationGracePeriodSeconds": self.grace_period, }, }, }, diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index ecf6f0b0..09b70562 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -23,6 +23,8 @@ data: NO_DELETE_JOBS: "{{ .Values.no_delete_jobs | default 0 }}" + GRACE_PERIOD_SECS: "{{ .Values.grace_period_secs | default 600 }}" + REGISTRATION_ENABLED: "{{ .Values.registration_enabled | default 0 }}" JWT_TOKEN_LIFETIME_MINUTES: "{{ .Values.jwt_token_lifetime_minutes | default 60 }}" diff --git a/chart/templates/minio.yaml b/chart/templates/minio.yaml index b2da8f17..6d8fa28c 100644 --- a/chart/templates/minio.yaml +++ b/chart/templates/minio.yaml @@ -55,7 +55,7 @@ spec: spec: volumes: - - name: data-storage + - name: data-minio persistentVolumeClaim: claimName: minio-storage-pvc @@ -69,7 +69,7 @@ spec: name: auth-secrets volumeMounts: - - name: data-storage + - name: data-minio mountPath: /data subPath: minio diff --git a/chart/templates/redis.yaml b/chart/templates/redis.yaml index d9f33033..b5409022 100644 --- a/chart/templates/redis.yaml +++ b/chart/templates/redis.yaml @@ -55,7 +55,7 @@ spec: spec: volumes: - - name: data-storage + - name: data-redis persistentVolumeClaim: claimName: redis-storage-pvc @@ -66,7 +66,7 @@ spec: args: ["--appendonly", "yes"] volumeMounts: - - name: data-storage + - name: data-redis mountPath: /data subPath: redis