k8s: don't add entire crawl config as env var from configmap, add only specified env vars from configmap

fix issue with crawls with large number of seeds failing due to unusually large env var

backend/btrixcloud/crawlmanager.py

@@ -98,9 +98,6 @@ class BaseCrawlManager(ABC):
             STORE_PATH=storage_path,
             STORE_FILENAME=out_filename,
             STORAGE_NAME=storage_name,
-            USER_ID=str(crawlconfig.userid),
-            ARCHIVE_ID=str(crawlconfig.aid),
-            CRAWL_CONFIG_ID=str(crawlconfig.id),
             PROFILE_FILENAME=profile_filename,
         )
 

backend/btrixcloud/k8s/templates/crawl_job.yaml

@@ -52,9 +52,6 @@ spec:
             - secretRef:
                 name: mongo-auth
 
-            - configMapRef:
-                name: crawl-config-{{ cid }}
-
           env:
             - name: JOB_ID
               valueFrom:
@@ -64,6 +61,39 @@ spec:
             - name: RUN_MANUAL
               value: "{{ manual }}"
 
+            - name: USER_ID
+              value: "{{ userid }}"
+
+            - name: ARCHIVE_ID
+              value: "{{ aid }}"
+
+            - name: CRAWL_CONFIG_ID
+              value: "{{ cid }}"
+
+            - name: STORE_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: crawl-config-{{ cid }}
+                  key: STORE_PATH
+
+            - name: STORE_FILENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: crawl-config-{{ cid }}
+                  key: STORE_PATH
+
+            - name: STORAGE_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: crawl-config-{{ cid }}
+                  key: STORAGE_NAME
+
+            - name: PROFILE_FILENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: crawl-config-{{ cid }}
+                  key: PROFILE_FILENAME
+
       volumes:
         - name: config-volume
           configMap: