From 71842be94a6316834b601cc49fae1da43211f283 Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@gmail.com>
Date: Mon, 7 Feb 2022 15:31:17 -0800
Subject: [PATCH] backend: k8s setup minor tweaks: - add 'emptyDir' volume for
 crawl directory (to allow any pod restarts to have access to the data) -
 rename minio and redis volumes to avoid any confusion - add pod termination
 grace-period (default to 600 secs)

---
 backend/k8sman.py              | 14 +++++++++++---
 chart/templates/configmap.yaml |  2 ++
 chart/templates/minio.yaml     |  4 ++--
 chart/templates/redis.yaml     |  4 ++--
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/backend/k8sman.py b/backend/k8sman.py
index a64b5b7e..6bd16ef4 100644
--- a/backend/k8sman.py
+++ b/backend/k8sman.py
@@ -46,6 +46,8 @@ class K8SManager:
 
         self.no_delete_jobs = os.environ.get("NO_DELETE_JOBS", "0") != "0"
 
+        self.grace_period = int(os.environ.get("GRACE_PERIOD_SECS", "600"))
+
         self.redis_url = os.environ["REDIS_URL"]
 
         self.loop = asyncio.get_running_loop()
@@ -690,7 +692,7 @@ class K8SManager:
         requests_memory = "384M"
         limit_memory = "2G"
 
-        requests_cpu = "240m"
+        requests_cpu = "500m"
         limit_cpu = "2000m"
 
         resources = {
@@ -728,7 +730,11 @@ class K8SManager:
                                         "mountPath": "/tmp/crawl-config.json",
                                         "subPath": "crawl-config.json",
                                         "readOnly": True,
-                                    }
+                                    },
+                                    {
+                                        "name": "crawl-data",
+                                        "mountPath": "/crawls",
+                                    },
                                 ],
                                 "envFrom": [
                                     {"configMapRef": {"name": "shared-crawler-config"}},
@@ -764,9 +770,11 @@ class K8SManager:
                                         }
                                     ],
                                 },
-                            }
+                            },
+                            {"name": "crawl-data", "emptyDir": {}},
                         ],
                         "restartPolicy": "OnFailure",
+                        "terminationGracePeriodSeconds": self.grace_period,
                     },
                 },
             },
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index ecf6f0b0..09b70562 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -23,6 +23,8 @@ data:
 
   NO_DELETE_JOBS: "{{ .Values.no_delete_jobs | default 0 }}"
 
+  GRACE_PERIOD_SECS: "{{ .Values.grace_period_secs | default 600 }}"
+
   REGISTRATION_ENABLED: "{{ .Values.registration_enabled | default 0 }}"
 
   JWT_TOKEN_LIFETIME_MINUTES: "{{ .Values.jwt_token_lifetime_minutes | default 60 }}"
diff --git a/chart/templates/minio.yaml b/chart/templates/minio.yaml
index b2da8f17..6d8fa28c 100644
--- a/chart/templates/minio.yaml
+++ b/chart/templates/minio.yaml
@@ -55,7 +55,7 @@ spec:
 
     spec:
       volumes:
-        - name: data-storage
+        - name: data-minio
           persistentVolumeClaim:
             claimName: minio-storage-pvc
 
@@ -69,7 +69,7 @@ spec:
                 name: auth-secrets
 
           volumeMounts:
-            - name: data-storage
+            - name: data-minio
               mountPath: /data
               subPath: minio
 
diff --git a/chart/templates/redis.yaml b/chart/templates/redis.yaml
index d9f33033..b5409022 100644
--- a/chart/templates/redis.yaml
+++ b/chart/templates/redis.yaml
@@ -55,7 +55,7 @@ spec:
 
     spec:
       volumes:
-        - name: data-storage
+        - name: data-redis
           persistentVolumeClaim:
             claimName: redis-storage-pvc
 
@@ -66,7 +66,7 @@ spec:
 
           args: ["--appendonly", "yes"]
           volumeMounts:
-            - name: data-storage
+            - name: data-redis
               mountPath: /data
               subPath: redis