backend: k8s setup minor tweaks:

- add 'emptyDir' volume for crawl directory (to allow any pod restarts to have access to the data)
- rename minio and redis volumes to avoid any confusion
- add pod termination grace-period (default to 600 secs)

backend/k8sman.py

@@ -46,6 +46,8 @@ class K8SManager:
 
         self.no_delete_jobs = os.environ.get("NO_DELETE_JOBS", "0") != "0"
 
+        self.grace_period = int(os.environ.get("GRACE_PERIOD_SECS", "600"))
+
         self.redis_url = os.environ["REDIS_URL"]
 
         self.loop = asyncio.get_running_loop()
@@ -690,7 +692,7 @@ class K8SManager:
         requests_memory = "384M"
         limit_memory = "2G"
 
-        requests_cpu = "240m"
+        requests_cpu = "500m"
         limit_cpu = "2000m"
 
         resources = {
@@ -728,7 +730,11 @@ class K8SManager:
                                         "mountPath": "/tmp/crawl-config.json",
                                         "subPath": "crawl-config.json",
                                         "readOnly": True,
-                                    }
+                                    },
+                                    {
+                                        "name": "crawl-data",
+                                        "mountPath": "/crawls",
+                                    },
                                 ],
                                 "envFrom": [
                                     {"configMapRef": {"name": "shared-crawler-config"}},
@@ -764,9 +770,11 @@ class K8SManager:
                                         }
                                     ],
                                 },
-                            }
+                            },
+                            {"name": "crawl-data", "emptyDir": {}},
                         ],
                         "restartPolicy": "OnFailure",
+                        "terminationGracePeriodSeconds": self.grace_period,
                     },
                 },
             },

chart/templates/configmap.yaml

@@ -23,6 +23,8 @@ data:
 
   NO_DELETE_JOBS: "{{ .Values.no_delete_jobs | default 0 }}"
 
+  GRACE_PERIOD_SECS: "{{ .Values.grace_period_secs | default 600 }}"
+
   REGISTRATION_ENABLED: "{{ .Values.registration_enabled | default 0 }}"
 
   JWT_TOKEN_LIFETIME_MINUTES: "{{ .Values.jwt_token_lifetime_minutes | default 60 }}"

chart/templates/minio.yaml

@@ -55,7 +55,7 @@ spec:
 
     spec:
       volumes:
-        - name: data-storage
+        - name: data-minio
           persistentVolumeClaim:
             claimName: minio-storage-pvc
 
@@ -69,7 +69,7 @@ spec:
                 name: auth-secrets
 
           volumeMounts:
-            - name: data-storage
+            - name: data-minio
               mountPath: /data
               subPath: minio
 

chart/templates/redis.yaml

@@ -55,7 +55,7 @@ spec:
 
     spec:
       volumes:
-        - name: data-storage
+        - name: data-redis
           persistentVolumeClaim:
             claimName: redis-storage-pvc
 
@@ -66,7 +66,7 @@ spec:
 
           args: ["--appendonly", "yes"]
           volumeMounts:
-            - name: data-storage
+            - name: data-redis
               mountPath: /data
               subPath: redis