From ccd87e0dffc65ddf7dd9cef21cdfcd5f5fd21558 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 18 Jan 2023 20:21:36 -0800 Subject: [PATCH] Rename api / nginx settings -> backend / frontend, set pull policy job images (#504) * rename config values - api -> backend - nginx -> frontend * job pods: - set job_pull_policy from api_pull_policy (same as backend image) - default to Always, but can be overridden for local deployment (same as backend image) typo fix: CRAWL_NAMESPACE -> CRAWLER_NAMESPACE (part of #491) ansible: set default label to :latest instead of :dev for --- ansible/group_vars/do/do-values.template.yaml | 4 +-- ansible/group_vars/do/main.yml | 2 +- backend/btrixcloud/crawlmanager.py | 3 ++ backend/btrixcloud/k8s/base_job.py | 2 +- .../btrixcloud/k8s/templates/crawl_job.yaml | 2 +- .../btrixcloud/k8s/templates/profile_job.yaml | 2 +- chart/examples/k3d-ci.yaml | 4 +-- chart/examples/local-config.yaml | 8 +++--- chart/examples/microk8s-ci.yaml | 8 +++--- chart/examples/microk8s-hosted.yaml | 4 +-- chart/templates/backend.yaml | 14 +++++----- chart/templates/configmap.yaml | 3 +- chart/templates/frontend.yaml | 14 +++++----- chart/templates/mongo.yaml | 2 +- chart/templates/secrets.yaml | 2 +- chart/templates/signer.yaml | 2 +- chart/values.yaml | 28 +++++++++---------- 17 files changed, 54 insertions(+), 50 deletions(-) diff --git a/ansible/group_vars/do/do-values.template.yaml b/ansible/group_vars/do/do-values.template.yaml index e4f3ffa3..a2ed9fcb 100644 --- a/ansible/group_vars/do/do-values.template.yaml +++ b/ansible/group_vars/do/do-values.template.yaml @@ -13,8 +13,8 @@ crawler_pull_policy: "Always" # Registry {% if use_do_registry %} -api_image: "{{ registry_endpoint }}/webrecorder/browsertrix-backend:{{ image_tag }}" -nginx_image: "{{ registry_endpoint }}/webrecorder/browsertrix-frontend:{{ image_tag }}" +backend_image: "{{ registry_endpoint }}/webrecorder/browsertrix-backend:{{ image_tag }}" +frontend_image: "{{ registry_endpoint }}/webrecorder/browsertrix-frontend:{{ image_tag }}" crawler_image: "{{ registry_endpoint }}/webrecorder/browsertrix-crawler:{{ image_tag }}" {% endif %} diff --git a/ansible/group_vars/do/main.yml b/ansible/group_vars/do/main.yml index a475ee12..4ae2dc1b 100644 --- a/ansible/group_vars/do/main.yml +++ b/ansible/group_vars/do/main.yml @@ -19,7 +19,7 @@ domain: "browsertrix.cloud" subdomain: "{{ project_name }}" use_do_registry: true -image_tag: "dev" +image_tag: "latest" enable_signing: true signing_host: "signing" diff --git a/backend/btrixcloud/crawlmanager.py b/backend/btrixcloud/crawlmanager.py index f9d8120a..f20a6cf2 100644 --- a/backend/btrixcloud/crawlmanager.py +++ b/backend/btrixcloud/crawlmanager.py @@ -20,6 +20,7 @@ class BaseCrawlManager(ABC): super().__init__() self.job_image = os.environ["JOB_IMAGE"] + self.job_pull_policy = os.environ.get("JOB_PULL_POLICY", "Always") self.no_delete_jobs = os.environ.get("NO_DELETE_JOBS", "0") != "0" @@ -59,6 +60,7 @@ class BaseCrawlManager(ABC): "userid": str(userid), "oid": str(oid), "job_image": self.job_image, + "job_pull_policy": self.job_pull_policy, "storage_name": storage_name, "storage_path": storage_path or "", "baseprofile": baseprofile or "", @@ -178,6 +180,7 @@ class BaseCrawlManager(ABC): "userid": str(crawlconfig.userid), "oid": str(crawlconfig.oid), "job_image": self.job_image, + "job_pull_policy": self.job_pull_policy, "manual": "1" if manual else "0", "crawler_node_type": self.crawler_node_type, "schedule": schedule, diff --git a/backend/btrixcloud/k8s/base_job.py b/backend/btrixcloud/k8s/base_job.py index df9e1b5c..77e31983 100644 --- a/backend/btrixcloud/k8s/base_job.py +++ b/backend/btrixcloud/k8s/base_job.py @@ -19,7 +19,7 @@ class K8SJobMixin(K8sAPI): """Crawl Job State""" def __init__(self): - self.namespace = os.environ.get("CRAWL_NAMESPACE") or "crawlers" + self.namespace = os.environ.get("CRAWLER_NAMESPACE") or "crawlers" self.config_file = "/config/config.yaml" self.job_id = os.environ.get("JOB_ID") diff --git a/backend/btrixcloud/k8s/templates/crawl_job.yaml b/backend/btrixcloud/k8s/templates/crawl_job.yaml index c63de34d..3fe17e7d 100644 --- a/backend/btrixcloud/k8s/templates/crawl_job.yaml +++ b/backend/btrixcloud/k8s/templates/crawl_job.yaml @@ -41,7 +41,7 @@ spec: containers: - name: crawl-job image: {{ job_image }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ job_pull_policy }} command: ["uvicorn", "btrixcloud.k8s.crawl_job:app", "--host", "0.0.0.0", "--access-log", "--log-level", "info"] volumeMounts: diff --git a/backend/btrixcloud/k8s/templates/profile_job.yaml b/backend/btrixcloud/k8s/templates/profile_job.yaml index 252d8145..6fab307b 100644 --- a/backend/btrixcloud/k8s/templates/profile_job.yaml +++ b/backend/btrixcloud/k8s/templates/profile_job.yaml @@ -45,7 +45,7 @@ spec: containers: - name: crawl-job image: {{ job_image }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ job_pull_policy }} command: ["python", "-m", "btrixcloud.k8s.profile_job"] volumeMounts: diff --git a/chart/examples/k3d-ci.yaml b/chart/examples/k3d-ci.yaml index 806b70e3..12caf8f5 100644 --- a/chart/examples/k3d-ci.yaml +++ b/chart/examples/k3d-ci.yaml @@ -3,8 +3,8 @@ # don't pull use, existing images -api_pull_policy: "Never" -nginx_pull_policy: "Never" +backend_pull_policy: "Never" +frontend_pull_policy: "Never" mongo_auth: diff --git a/chart/examples/local-config.yaml b/chart/examples/local-config.yaml index 61a16ae6..cc4aebdd 100644 --- a/chart/examples/local-config.yaml +++ b/chart/examples/local-config.yaml @@ -10,15 +10,15 @@ local_service_port: 30870 default_org: "My Local Organization" # overrides to use existing images in local Docker, otherwise will pull from latest -# api_pull_policy: "Never" -# nginx_pull_policy: "Never" +# backend_pull_policy: "Never" +# frontend_pull_policy: "Never" # crawler_pull_policy: "Never" # redis_pull_policy: "Never" # microk8s: if developing locally, can override these to use images from local microk8s repository (on localhost:32000) -# api_image: "localhost:32000/webrecorder/browsertrix-backend:latest" -# nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" +# backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" +# frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" # optionally, override default mongodb auth, used for all data storage: diff --git a/chart/examples/microk8s-ci.yaml b/chart/examples/microk8s-ci.yaml index 1109bb8d..ab65429c 100644 --- a/chart/examples/microk8s-ci.yaml +++ b/chart/examples/microk8s-ci.yaml @@ -3,12 +3,12 @@ # use local images -api_image: "localhost:32000/webrecorder/browsertrix-backend:latest" -nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" +backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" +frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" # don't pull use, existing images -api_pull_policy: "IfNotPresent" -nginx_pull_policy: "IfNotPresent" +backend_pull_policy: "IfNotPresent" +frontend_pull_policy: "IfNotPresent" mongo_auth: diff --git a/chart/examples/microk8s-hosted.yaml b/chart/examples/microk8s-hosted.yaml index 3d60a309..8fc1fcad 100644 --- a/chart/examples/microk8s-hosted.yaml +++ b/chart/examples/microk8s-hosted.yaml @@ -21,8 +21,8 @@ # # If developing locally, can override these to use images from local microk8s repository (on localhost:32000) # -# api_image: "localhost:32000/webrecorder/browsertrix-backend:latest" -# nginx_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" +# backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" +# frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" # crawler_image: "localhost:32000/webrecorder/browsertrix-crawler:latest" diff --git a/chart/templates/backend.yaml b/chart/templates/backend.yaml index b196a335..126703ee 100644 --- a/chart/templates/backend.yaml +++ b/chart/templates/backend.yaml @@ -10,7 +10,7 @@ spec: matchLabels: app: {{ .Values.name }} role: backend - replicas: {{ .Values.api_num_replicas }} + replicas: {{ .Values.backend_num_replicas }} template: metadata: labels: @@ -31,8 +31,8 @@ spec: containers: - name: api - image: {{ .Values.api_image }} - imagePullPolicy: {{ .Values.api_pull_policy }} + image: {{ .Values.backend_image }} + imagePullPolicy: {{ .Values.backend_pull_policy }} envFrom: - configMapRef: name: {{ .Values.name }}-env-config @@ -43,12 +43,12 @@ spec: resources: limits: - cpu: {{ .Values.api_limits_cpu }} - memory: {{ .Values.api_limits_memory }} + cpu: {{ .Values.backend_limits_cpu }} + memory: {{ .Values.backend_limits_memory }} requests: - cpu: {{ .Values.api_requests_cpu }} - memory: {{ .Values.api_requests_memory }} + cpu: {{ .Values.backend_requests_cpu }} + memory: {{ .Values.backend_requests_memory }} startupProbe: httpGet: diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 3cbc1e05..a7025ad3 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -27,7 +27,8 @@ data: DEFAULT_ORG: "{{ .Values.default_org }}" - JOB_IMAGE: "{{ .Values.api_image }}" + JOB_IMAGE: "{{ .Values.backend_image }}" + JOB_PULL_POLICY: "{{ .Values.backend_pull_policy }}" {{- if .Values.crawler_pv_claim }} CRAWLER_PV_CLAIM: "{{ .Values.crawler_pv_claim }}" diff --git a/chart/templates/frontend.yaml b/chart/templates/frontend.yaml index de6ac999..50fc92e1 100644 --- a/chart/templates/frontend.yaml +++ b/chart/templates/frontend.yaml @@ -10,7 +10,7 @@ spec: matchLabels: app: {{ .Values.name }} role: frontend - replicas: {{ .Values.nginx_num_replicas | default 1 }} + replicas: {{ .Values.frontend_num_replicas | default 1 }} template: metadata: labels: @@ -32,8 +32,8 @@ spec: containers: - name: nginx - image: {{ .Values.nginx_image }} - imagePullPolicy: {{ .Values.nginx_pull_policy }} + image: {{ .Values.frontend_image }} + imagePullPolicy: {{ .Values.frontend_pull_policy }} env: - name: BACKEND_HOST value: {{ .Values.name }}-backend @@ -54,12 +54,12 @@ spec: resources: limits: - cpu: {{ .Values.nginx_limits_cpu }} - memory: {{ .Values.nginx_limits_memory }} + cpu: {{ .Values.frontend_limits_cpu }} + memory: {{ .Values.frontend_limits_memory }} requests: - cpu: {{ .Values.nginx_requests_cpu }} - memory: {{ .Values.nginx_requests_memory }} + cpu: {{ .Values.frontend_requests_cpu }} + memory: {{ .Values.frontend_requests_memory }} readinessProbe: httpGet: diff --git a/chart/templates/mongo.yaml b/chart/templates/mongo.yaml index 4977a636..e3351c2d 100644 --- a/chart/templates/mongo.yaml +++ b/chart/templates/mongo.yaml @@ -43,7 +43,7 @@ spec: matchLabels: app: local-mongo serviceName: local-mongo - replicas: {{ .Values.api_num_replicas }} + replicas: {{ .Values.backend_num_replicas }} podManagementPolicy: Parallel volumeClaimTemplates: - metadata: diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml index 5dbfcb6f..07a6b596 100644 --- a/chart/templates/secrets.yaml +++ b/chart/templates/secrets.yaml @@ -7,7 +7,7 @@ metadata: type: Opaque stringData: - PASSWORD_SECRET: "{{ .Values.api_password_secret }}" + PASSWORD_SECRET: "{{ .Values.backend_password_secret }}" {{- if .Values.minio_local }} {{- with (first .Values.storages) }} diff --git a/chart/templates/signer.yaml b/chart/templates/signer.yaml index 4ab19140..9669c365 100644 --- a/chart/templates/signer.yaml +++ b/chart/templates/signer.yaml @@ -47,7 +47,7 @@ spec: selector: matchLabels: app: auth-signer - replicas: {{ .Values.api_num_replicas }} + replicas: {{ .Values.backend_num_replicas }} serviceName: auth-signer volumeClaimTemplates: - metadata: diff --git a/chart/values.yaml b/chart/values.yaml index cb554555..698cd01d 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -41,18 +41,18 @@ default_org: "My Organization" # API Image # ========================================= -api_image: "docker.io/webrecorder/browsertrix-backend:latest" -api_pull_policy: "Always" +backend_image: "docker.io/webrecorder/browsertrix-backend:latest" +backend_pull_policy: "Always" -api_password_secret: "c9085f33ecce4347aa1d69339e16c499" +backend_password_secret: "c9085f33ecce4347aa1d69339e16c499" -api_num_replicas: 1 +backend_num_replicas: 1 -api_requests_cpu: "10m" -api_limits_cpu: "768m" +backend_requests_cpu: "10m" +backend_limits_cpu: "768m" -api_requests_memory: "100Mi" -api_limits_memory: "512Mi" +backend_requests_memory: "100Mi" +backend_limits_memory: "512Mi" job_cpu: "3m" job_memory: "70Mi" @@ -62,14 +62,14 @@ profile_browser_idle_seconds: 60 # Nginx Image # ========================================= -nginx_image: "docker.io/webrecorder/browsertrix-frontend:latest" -nginx_pull_policy: "Always" +frontend_image: "docker.io/webrecorder/browsertrix-frontend:latest" +frontend_pull_policy: "Always" -nginx_requests_cpu: "3m" -nginx_limits_cpu: "10m" +frontend_requests_cpu: "3m" +frontend_limits_cpu: "10m" -nginx_requests_memory: "12Mi" -nginx_limits_memory: "20Mi" +frontend_requests_memory: "12Mi" +frontend_limits_memory: "20Mi" # if set, maps nginx to a fixed port on host machine # must be between 30000 - 32767