Rename api / nginx settings -> backend / frontend, set pull policy job images (#504)

* rename config values
- api -> backend
- nginx -> frontend

* job pods:
- set job_pull_policy from api_pull_policy (same as backend image)
- default to Always, but can be overridden for local deployment (same as backend image)

typo fix: CRAWL_NAMESPACE -> CRAWLER_NAMESPACE (part of #491)
ansible: set default label to :latest instead of :dev for

ansible/group_vars/do/do-values.template.yaml

@@ -13,8 +13,8 @@ crawler_pull_policy: "Always"
 
 # Registry
 {% if use_do_registry %}
-api_image: "{{ registry_endpoint }}/webrecorder/browsertrix-backend:{{ image_tag }}"
+backend_image: "{{ registry_endpoint }}/webrecorder/browsertrix-backend:{{ image_tag }}"
-nginx_image: "{{ registry_endpoint }}/webrecorder/browsertrix-frontend:{{ image_tag }}"
+frontend_image: "{{ registry_endpoint }}/webrecorder/browsertrix-frontend:{{ image_tag }}"
 crawler_image: "{{ registry_endpoint }}/webrecorder/browsertrix-crawler:{{ image_tag }}"
 {% endif %}
 

ansible/group_vars/do/main.yml

@@ -19,7 +19,7 @@ domain: "browsertrix.cloud"
 subdomain: "{{ project_name }}"
 
 use_do_registry: true
-image_tag: "dev"
+image_tag: "latest"
 
 enable_signing: true
 signing_host: "signing"

backend/btrixcloud/crawlmanager.py

@@ -20,6 +20,7 @@ class BaseCrawlManager(ABC):
         super().__init__()
 
         self.job_image = os.environ["JOB_IMAGE"]
+        self.job_pull_policy = os.environ.get("JOB_PULL_POLICY", "Always")
 
         self.no_delete_jobs = os.environ.get("NO_DELETE_JOBS", "0") != "0"
 
@@ -59,6 +60,7 @@ class BaseCrawlManager(ABC):
             "userid": str(userid),
             "oid": str(oid),
             "job_image": self.job_image,
+            "job_pull_policy": self.job_pull_policy,
             "storage_name": storage_name,
             "storage_path": storage_path or "",
             "baseprofile": baseprofile or "",
@@ -178,6 +180,7 @@ class BaseCrawlManager(ABC):
             "userid": str(crawlconfig.userid),
             "oid": str(crawlconfig.oid),
             "job_image": self.job_image,
+            "job_pull_policy": self.job_pull_policy,
             "manual": "1" if manual else "0",
             "crawler_node_type": self.crawler_node_type,
             "schedule": schedule,

backend/btrixcloud/k8s/base_job.py

@@ -19,7 +19,7 @@ class K8SJobMixin(K8sAPI):
     """Crawl Job State"""
 
     def __init__(self):
-        self.namespace = os.environ.get("CRAWL_NAMESPACE") or "crawlers"
+        self.namespace = os.environ.get("CRAWLER_NAMESPACE") or "crawlers"
         self.config_file = "/config/config.yaml"
 
         self.job_id = os.environ.get("JOB_ID")

backend/btrixcloud/k8s/templates/crawl_job.yaml

@@ -41,7 +41,7 @@ spec:
       containers:
         - name: crawl-job
           image: {{ job_image }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: {{ job_pull_policy }}
           command: ["uvicorn", "btrixcloud.k8s.crawl_job:app", "--host", "0.0.0.0", "--access-log", "--log-level", "info"]
 
           volumeMounts:

backend/btrixcloud/k8s/templates/profile_job.yaml

@@ -45,7 +45,7 @@ spec:
       containers:
         - name: crawl-job
           image: {{ job_image }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: {{ job_pull_policy }}
           command: ["python", "-m", "btrixcloud.k8s.profile_job"]
 
           volumeMounts:

chart/examples/k3d-ci.yaml

@@ -3,8 +3,8 @@
 
 
 # don't pull use, existing images
-api_pull_policy: "Never"
+backend_pull_policy: "Never"
-nginx_pull_policy: "Never"
+frontend_pull_policy: "Never"
 
 
 mongo_auth:

chart/examples/local-config.yaml

@@ -10,15 +10,15 @@ local_service_port: 30870
 default_org: "My Local Organization"
 
 # overrides to use existing images in local Docker, otherwise will pull from latest
-# api_pull_policy: "Never"
+# backend_pull_policy: "Never"
-# nginx_pull_policy: "Never"
+# frontend_pull_policy: "Never"
 # crawler_pull_policy: "Never"
 # redis_pull_policy: "Never"
 
 
 # microk8s: if developing locally, can override these to use images from local microk8s repository (on localhost:32000)
-# api_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
+# backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
-# nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
+# frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
 
 
 # optionally, override default mongodb auth, used for all data storage:

chart/examples/microk8s-ci.yaml

@@ -3,12 +3,12 @@
 
 
 # use local images
-api_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
+backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
-nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
+frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
 
 # don't pull use, existing images
-api_pull_policy: "IfNotPresent"
+backend_pull_policy: "IfNotPresent"
-nginx_pull_policy: "IfNotPresent"
+frontend_pull_policy: "IfNotPresent"
 
 
 mongo_auth:

chart/examples/microk8s-hosted.yaml

@@ -21,8 +21,8 @@
 #
 # If developing locally, can override these to use images from local microk8s repository (on localhost:32000)
 #
-# api_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
+# backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest"
-# nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
+# frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest"
 # crawler_image: "localhost:32000/webrecorder/browsertrix-crawler:latest"
 
 

chart/templates/backend.yaml

@@ -10,7 +10,7 @@ spec:
     matchLabels:
       app: {{ .Values.name }}
       role: backend
-  replicas: {{ .Values.api_num_replicas }}
+  replicas: {{ .Values.backend_num_replicas }}
   template:
     metadata:
       labels:
@@ -31,8 +31,8 @@ spec:
 
       containers:
         - name: api
-          image: {{ .Values.api_image }}
+          image: {{ .Values.backend_image }}
-          imagePullPolicy: {{ .Values.api_pull_policy }}
+          imagePullPolicy: {{ .Values.backend_pull_policy }}
           envFrom:
             - configMapRef:
                 name: {{ .Values.name }}-env-config
@@ -43,12 +43,12 @@ spec:
 
           resources:
             limits:
-              cpu: {{ .Values.api_limits_cpu }}
+              cpu: {{ .Values.backend_limits_cpu }}
-              memory: {{ .Values.api_limits_memory }}
+              memory: {{ .Values.backend_limits_memory }}
 
             requests:
-              cpu: {{ .Values.api_requests_cpu }}
+              cpu: {{ .Values.backend_requests_cpu }}
-              memory: {{ .Values.api_requests_memory }}
+              memory: {{ .Values.backend_requests_memory }}
 
           startupProbe:
             httpGet:

chart/templates/configmap.yaml

@@ -27,7 +27,8 @@ data:
 
   DEFAULT_ORG: "{{ .Values.default_org }}"
 
-  JOB_IMAGE: "{{ .Values.api_image }}"
+  JOB_IMAGE: "{{ .Values.backend_image }}"
+  JOB_PULL_POLICY: "{{ .Values.backend_pull_policy }}"
 
   {{- if .Values.crawler_pv_claim }}
   CRAWLER_PV_CLAIM: "{{ .Values.crawler_pv_claim }}"

chart/templates/frontend.yaml

@@ -10,7 +10,7 @@ spec:
     matchLabels:
       app: {{ .Values.name }}
       role: frontend
-  replicas: {{ .Values.nginx_num_replicas | default 1 }}
+  replicas: {{ .Values.frontend_num_replicas | default 1 }}
   template:
     metadata:
       labels:
@@ -32,8 +32,8 @@ spec:
 
       containers:
         - name: nginx
-          image: {{ .Values.nginx_image }}
+          image: {{ .Values.frontend_image }}
-          imagePullPolicy: {{ .Values.nginx_pull_policy }}
+          imagePullPolicy: {{ .Values.frontend_pull_policy }}
           env:
             - name: BACKEND_HOST
               value: {{ .Values.name }}-backend
@@ -54,12 +54,12 @@ spec:
 
           resources:
             limits:
-              cpu: {{ .Values.nginx_limits_cpu }}
+              cpu: {{ .Values.frontend_limits_cpu }}
-              memory: {{ .Values.nginx_limits_memory }}
+              memory: {{ .Values.frontend_limits_memory }}
 
             requests:
-              cpu: {{ .Values.nginx_requests_cpu }}
+              cpu: {{ .Values.frontend_requests_cpu }}
-              memory: {{ .Values.nginx_requests_memory }}
+              memory: {{ .Values.frontend_requests_memory }}
 
           readinessProbe:
             httpGet:

chart/templates/mongo.yaml

@@ -43,7 +43,7 @@ spec:
     matchLabels:
       app: local-mongo
   serviceName: local-mongo
-  replicas: {{ .Values.api_num_replicas }}
+  replicas: {{ .Values.backend_num_replicas }}
   podManagementPolicy: Parallel
   volumeClaimTemplates:
     - metadata:

chart/templates/secrets.yaml

@@ -7,7 +7,7 @@ metadata:
 
 type: Opaque
 stringData:
-  PASSWORD_SECRET: "{{ .Values.api_password_secret }}"
+  PASSWORD_SECRET: "{{ .Values.backend_password_secret }}"
 
 {{- if .Values.minio_local }}
 {{- with (first .Values.storages) }}

chart/templates/signer.yaml

@@ -47,7 +47,7 @@ spec:
   selector:
     matchLabels:
       app: auth-signer
-  replicas: {{ .Values.api_num_replicas }}
+  replicas: {{ .Values.backend_num_replicas }}
   serviceName: auth-signer
   volumeClaimTemplates:
   - metadata:

chart/values.yaml

@@ -41,18 +41,18 @@ default_org: "My Organization"
 
 # API Image
 # =========================================
-api_image: "docker.io/webrecorder/browsertrix-backend:latest"
+backend_image: "docker.io/webrecorder/browsertrix-backend:latest"
-api_pull_policy: "Always"
+backend_pull_policy: "Always"
 
-api_password_secret: "c9085f33ecce4347aa1d69339e16c499"
+backend_password_secret: "c9085f33ecce4347aa1d69339e16c499"
 
-api_num_replicas: 1
+backend_num_replicas: 1
 
-api_requests_cpu: "10m"
+backend_requests_cpu: "10m"
-api_limits_cpu: "768m"
+backend_limits_cpu: "768m"
 
-api_requests_memory: "100Mi"
+backend_requests_memory: "100Mi"
-api_limits_memory: "512Mi"
+backend_limits_memory: "512Mi"
 
 job_cpu: "3m"
 job_memory: "70Mi"
@@ -62,14 +62,14 @@ profile_browser_idle_seconds: 60
 
 # Nginx Image
 # =========================================
-nginx_image: "docker.io/webrecorder/browsertrix-frontend:latest"
+frontend_image: "docker.io/webrecorder/browsertrix-frontend:latest"
-nginx_pull_policy: "Always"
+frontend_pull_policy: "Always"
 
-nginx_requests_cpu: "3m"
+frontend_requests_cpu: "3m"
-nginx_limits_cpu: "10m"
+frontend_limits_cpu: "10m"
 
-nginx_requests_memory: "12Mi"
+frontend_requests_memory: "12Mi"
-nginx_limits_memory: "20Mi"
+frontend_limits_memory: "20Mi"
 
 # if set, maps nginx to a fixed port on host machine
 # must be between 30000 - 32767