chart / deployment fixes to run on microk8s: (fixes #385) (#387)

- ingress: fix proxying /data to minio, use another ingress which proxies correct host to ensure presigned urls work
- presigning: determine if signing endpoint url (minio) or access endpoint (cloud bucket) based on if access endpoint is provided, set bool on storage object
- chart: fix indent on incorrect storageClassName configs
- ingress: make 'ingress_class' configurable (set to 'public' for microk8s, default to 'nginx')
- minio: use older minio image which supports legacy fs based setup (for now)
- nginx service: add 'nginx_service_use_node_port' config setting: if true, will use NodePort for frontend,
other will use default (ClusterIP) and only for the frontend / nginx
- chart: remove changing service type for other services

backend/btrixcloud/archives.py

@@ -44,6 +44,7 @@ class S3Storage(BaseModel):
     secret_key: str
     access_endpoint_url: Optional[str]
     region: Optional[str] = ""
+    use_access_for_presign: Optional[bool] = True
 
 
 # ============================================================================

backend/btrixcloud/k8s/k8sman.py

@@ -93,6 +93,9 @@ class K8SManager(BaseCrawlManager, K8sAPI):
             access_key = self._secret_data(storage_secret, "STORE_ACCESS_KEY")
             secret_key = self._secret_data(storage_secret, "STORE_SECRET_KEY")
             region = self._secret_data(storage_secret, "STORE_REGION") or ""
+            use_access_for_presign = (
+                self._secret_data(storage_secret, "STORE_USE_ACCESS_FOR_PRESIGN") == "1"
+            )
 
             self._default_storages[name] = S3Storage(
                 access_key=access_key,
@@ -100,6 +103,7 @@ class K8SManager(BaseCrawlManager, K8sAPI):
                 endpoint_url=endpoint_url,
                 access_endpoint_url=access_endpoint_url,
                 region=region,
+                use_access_for_presign=use_access_for_presign,
             )
 
         return self._default_storages[name]

backend/btrixcloud/k8s/templates/crawler.yaml

@@ -40,9 +40,9 @@ spec:
           requests:
             storage: 1Gi
 
-      {% if volume_storage_class %}
-      storageClassName: {{ volume_storage_class }}
-      {% endif %}
+        {% if volume_storage_class %}
+        storageClassName: {{ volume_storage_class }}
+        {% endif %}
 
   template:
     metadata:
@@ -208,9 +208,9 @@ spec:
           requests:
             storage: {{ requests_hd }}
 
-      {% if volume_storage_class %}
-      storageClassName: {{ volume_storage_class }}
-      {% endif %}
+        {% if volume_storage_class %}
+        storageClassName: {{ volume_storage_class }}
+        {% endif %}
 
   template:
     metadata:

backend/btrixcloud/storages.py

@@ -4,7 +4,6 @@ Storage API
 from typing import Union
 from urllib.parse import urlsplit
 from contextlib import asynccontextmanager
-import os
 
 from fastapi import Depends, HTTPException
 from aiobotocore.session import get_session
@@ -13,10 +12,6 @@ from .archives import Archive, DefaultStorage, S3Storage
 from .users import User
 
 
-# sign access endpoint
-sign_access_endpoint = os.environ.get("SIGN_ACCESS_ENDPOINT")
-
-
 # ============================================================================
 def init_storages_api(archive_ops, crawl_manager, user_dep):
     """API for updating storage for an archive"""
@@ -108,7 +103,11 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600):
     else:
         raise Exception("No Default Storage Found, Invalid Storage Type")
 
-    async with get_s3_client(s3storage, sign_access_endpoint) as (client, bucket, key):
+    async with get_s3_client(s3storage, s3storage.use_access_for_presign) as (
+        client,
+        bucket,
+        key,
+    ):
         key += crawlfile.filename
 
         presigned_url = await client.generate_presigned_url(
@@ -116,7 +115,7 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600):
         )
 
         if (
-            not sign_access_endpoint
+            not s3storage.use_access_for_presign
             and s3storage.access_endpoint_url
             and s3storage.access_endpoint_url != s3storage.endpoint_url
         ):

chart/templates/backend.yaml

@@ -103,12 +103,6 @@ spec:
     app: {{ .Values.name }}
     role: backend
 
-{{- if .Values.service }}
-  {{- if .Values.service.type }}
-  type: {{ .Values.service.type | quote }}
-  {{- end }}
-{{- end }}
-
   ports:
     - protocol: TCP
       port: 8000

chart/templates/configmap.yaml

@@ -47,8 +47,6 @@ data:
 
   WEB_CONCURRENCY: "{{ .Values.backend_workers | default 4 }}"
 
-  SIGN_ACCESS_ENDPOINT: "1"
-
 
 ---
 apiVersion: v1

chart/templates/frontend.yaml

@@ -87,11 +87,9 @@ spec:
     app: {{ .Values.name }}
     role: frontend
 
-{{- if .Values.service }}
-  {{- if .Values.service.type }}
-  type: {{ .Values.service.type | quote }}
+  {{- if .Values.nginx_service_use_node_port }}
+  type: NodePort
   {{- end }}
-{{- end }}
 
   ports:
     - protocol: TCP

chart/templates/ingress.yaml

@@ -7,7 +7,7 @@ metadata:
   name: ingress-main
   namespace: {{ .Release.Namespace }}
   annotations:
-    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }}
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
     nginx.ingress.kubernetes.io/rewrite-target: /$1
     nginx.ingress.kubernetes.io/enable-cors: "true"
@@ -31,16 +31,6 @@ spec:
   - host: {{ .Values.ingress.host }}
     http:
       paths:
-{{- if .Values.minio_local }}
-      - path: /data/(.*)
-        pathType: Prefix
-        backend:
-          service:
-            name: local-minio
-            port:
-              number: 9000
-{{- end }}
-
       - path: /(api/.*)
         pathType: Prefix
         backend:
@@ -65,7 +55,7 @@ metadata:
   name: ingress-authsign
   namespace: {{ .Release.Namespace }}
   annotations:
-    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }}
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
     nginx.ingress.kubernetes.io/rewrite-target: /$1
     nginx.ingress.kubernetes.io/upstream-vhost: "{{ .Values.signer.host }}"
@@ -85,6 +75,35 @@ spec:
 
 {{ end }}
 
+
+{{- if .Values.minio_local }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress-minio
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }}
+    nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/upstream-vhost: "{{ .Values.minio_host }}"
+
+spec:
+  rules:
+  - host: {{ .Values.ingress.host }}
+    http:
+      paths:
+      - path: /data/(.*)
+        pathType: Prefix
+        backend:
+          service:
+            name: local-minio
+            port:
+              number: 9000
+
+
+{{- end }}
+
 {{ if .Values.ingress.tls }}
 ---
 
@@ -106,7 +125,7 @@ spec:
     solvers:
     - http01:
         ingress:
-          class: nginx
+          class: {{ .Values.ingress_class | default "nginx" }}
 
 {{ end }}
 {{ end }}

chart/templates/minio.yaml

@@ -53,8 +53,13 @@ spec:
           image: {{ .Values.minio_image }}
           imagePullPolicy: {{ .Values.minio_pull_policy }}
 
-          command: ['/bin/sh']
-          args: ["mkdir", "-p", "/data/{{ .Values.minio_local_bucket_name }}" ]
+          command:
+              - sh
+              - -c
+              - |
+                  mkdir -p /data/{{ .Values.minio_local_bucket_name }}
+                  mkdir -p /data/.minio.sys
+                  echo '{"version":"1","format":"fs","id":"btrix-data-fs","fs":{"version":"2"}}' > /data/.minio.sys/format.json
 
           volumeMounts:
             - name: data-minio
@@ -86,7 +91,6 @@ metadata:
     app: local-minio
 
 spec:
-  type: NodePort
   selector:
     app: local-minio
 

chart/templates/mongo.yaml

@@ -59,9 +59,9 @@ spec:
           requests:
             storage: 2Gi
 
-      {{- if .Values.volume_storage_class }}
-      storageClassName: {{ .Values.volume_storage_class }}
-      {{- end }}
+        {{- if .Values.volume_storage_class }}
+        storageClassName: {{ .Values.volume_storage_class }}
+        {{- end }}
 
   template:
     metadata:

chart/templates/secrets.yaml

@@ -42,17 +42,20 @@ stringData:
   STORE_SECRET_KEY: "{{ $storage.secret_key }}"
 
   {{- if $storage.bucket_name }}
-  STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}{{ $storage.bucket_name }}"
+  STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}{{ $storage.bucket_name }}/"
   {{- else }}
   STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}"
   {{- end }}
 
   {{- if $storage.access_endpoint_url }}
-  STORE_ACCESS_ENDPOINT_URL: "{{ $storage.access_endpoint_url }}"
+  STORE_ACCESS_ENDPOINT_URL: "{{ $storage.access_endpoint_url }}/"
+  STORE_USE_ACCESS_FOR_PRESIGN: "1"
   {{- else if and $.Values.ingress.host $.Values.minio_local }}
   STORE_ACCESS_ENDPOINT_URL: {{ $.Values.ingress.scheme | default "https" }}://{{ $.Values.ingress.host }}/data/{{ $storage.bucket_name }}/
+  STORE_USE_ACCESS_FOR_PRESIGN: "0"
   {{- else }}
   STORE_ACCESS_ENDPOINT_URL: "{{ $storage.endpoint_url }}"
+  STORE_USE_ACCESS_FOR_PRESIGN: "0"
   {{- end }}
 
   STORE_REGION: "{{ $storage.region }}"

chart/templates/signer.yaml

@@ -135,12 +135,6 @@ spec:
   selector:
     app: auth-signer
 
-{{- if .Values.service }}
-  {{- if .Values.service.type }}
-  type: {{ .Values.service.type | quote }}
-  {{- end }}
-{{- end }}
-
   clusterIP: None
   ports:
     - protocol: TCP

chart/values.yaml

@@ -31,7 +31,7 @@ superuser:
 
 # API Image
 # =========================================
-api_image: "docker.io/webrecorder/browsertrix-backend:dev"
+api_image: "docker.io/webrecorder/browsertrix-backend:latest"
 api_pull_policy: "Always"
 
 api_password_secret: "c9085f33ecce4347aa1d69339e16c499"
@@ -50,7 +50,7 @@ job_memory: "70Mi"
 
 # Nginx Image
 # =========================================
-nginx_image: "docker.io/webrecorder/browsertrix-frontend:dev"
+nginx_image: "docker.io/webrecorder/browsertrix-frontend:latest"
 nginx_pull_policy: "Always"
 
 nginx_requests_cpu: "3m"
@@ -59,6 +59,10 @@ nginx_limits_cpu: "10m"
 nginx_requests_memory: "12Mi"
 nginx_limits_memory: "12Mi"
 
+# if true, will use node port to make the service directly available
+# for testing / local deployments only
+nginx_service_use_node_port: false
+
 
 # MongoDB Image
 # =========================================
@@ -147,7 +151,7 @@ minio_local: true
 minio_scheme: "http"
 minio_host: "local-minio.default:9000"
 
-minio_image: minio/minio
+minio_image: docker.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
 minio_mc_image: minio/mc
 minio_pull_policy: "IfNotPresent"
 
@@ -193,6 +197,8 @@ ingress:
   scheme: "http"
   tls: false
 
+ingress_class: nginx
+
 
 # Signing Options
 # =========================================
@@ -212,9 +218,8 @@ signer_requests_memory: "36Mi"
 signer_limits_memory: "96Mi"
 
 
-# Optional: configure load balancing
-service:
-  type: ClusterIP
+# Optional: configure load balancing annotations
+# service:
 #   annotations:
 #     service.beta.kubernetes.io/aws-load-balancer-internal: "true"
 #     helm.sh/resource-policy: keep