diff --git a/backend/btrixcloud/archives.py b/backend/btrixcloud/archives.py index 7002b27c..a275dea2 100644 --- a/backend/btrixcloud/archives.py +++ b/backend/btrixcloud/archives.py @@ -44,6 +44,7 @@ class S3Storage(BaseModel): secret_key: str access_endpoint_url: Optional[str] region: Optional[str] = "" + use_access_for_presign: Optional[bool] = True # ============================================================================ diff --git a/backend/btrixcloud/k8s/k8sman.py b/backend/btrixcloud/k8s/k8sman.py index 4996aaab..b8ba8a96 100644 --- a/backend/btrixcloud/k8s/k8sman.py +++ b/backend/btrixcloud/k8s/k8sman.py @@ -93,6 +93,9 @@ class K8SManager(BaseCrawlManager, K8sAPI): access_key = self._secret_data(storage_secret, "STORE_ACCESS_KEY") secret_key = self._secret_data(storage_secret, "STORE_SECRET_KEY") region = self._secret_data(storage_secret, "STORE_REGION") or "" + use_access_for_presign = ( + self._secret_data(storage_secret, "STORE_USE_ACCESS_FOR_PRESIGN") == "1" + ) self._default_storages[name] = S3Storage( access_key=access_key, @@ -100,6 +103,7 @@ class K8SManager(BaseCrawlManager, K8sAPI): endpoint_url=endpoint_url, access_endpoint_url=access_endpoint_url, region=region, + use_access_for_presign=use_access_for_presign, ) return self._default_storages[name] diff --git a/backend/btrixcloud/k8s/templates/crawler.yaml b/backend/btrixcloud/k8s/templates/crawler.yaml index a508e4c9..2881fd99 100644 --- a/backend/btrixcloud/k8s/templates/crawler.yaml +++ b/backend/btrixcloud/k8s/templates/crawler.yaml @@ -40,9 +40,9 @@ spec: requests: storage: 1Gi - {% if volume_storage_class %} - storageClassName: {{ volume_storage_class }} - {% endif %} + {% if volume_storage_class %} + storageClassName: {{ volume_storage_class }} + {% endif %} template: metadata: @@ -208,9 +208,9 @@ spec: requests: storage: {{ requests_hd }} - {% if volume_storage_class %} - storageClassName: {{ volume_storage_class }} - {% endif %} + {% if volume_storage_class %} + storageClassName: {{ volume_storage_class }} + {% endif %} template: metadata: diff --git a/backend/btrixcloud/storages.py b/backend/btrixcloud/storages.py index 2c9aba7e..ad5852b5 100644 --- a/backend/btrixcloud/storages.py +++ b/backend/btrixcloud/storages.py @@ -4,7 +4,6 @@ Storage API from typing import Union from urllib.parse import urlsplit from contextlib import asynccontextmanager -import os from fastapi import Depends, HTTPException from aiobotocore.session import get_session @@ -13,10 +12,6 @@ from .archives import Archive, DefaultStorage, S3Storage from .users import User -# sign access endpoint -sign_access_endpoint = os.environ.get("SIGN_ACCESS_ENDPOINT") - - # ============================================================================ def init_storages_api(archive_ops, crawl_manager, user_dep): """API for updating storage for an archive""" @@ -108,7 +103,11 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600): else: raise Exception("No Default Storage Found, Invalid Storage Type") - async with get_s3_client(s3storage, sign_access_endpoint) as (client, bucket, key): + async with get_s3_client(s3storage, s3storage.use_access_for_presign) as ( + client, + bucket, + key, + ): key += crawlfile.filename presigned_url = await client.generate_presigned_url( @@ -116,7 +115,7 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600): ) if ( - not sign_access_endpoint + not s3storage.use_access_for_presign and s3storage.access_endpoint_url and s3storage.access_endpoint_url != s3storage.endpoint_url ): diff --git a/chart/templates/backend.yaml b/chart/templates/backend.yaml index 5961815c..b196a335 100644 --- a/chart/templates/backend.yaml +++ b/chart/templates/backend.yaml @@ -103,12 +103,6 @@ spec: app: {{ .Values.name }} role: backend -{{- if .Values.service }} - {{- if .Values.service.type }} - type: {{ .Values.service.type | quote }} - {{- end }} -{{- end }} - ports: - protocol: TCP port: 8000 diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index c056c361..24138cb3 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -47,8 +47,6 @@ data: WEB_CONCURRENCY: "{{ .Values.backend_workers | default 4 }}" - SIGN_ACCESS_ENDPOINT: "1" - --- apiVersion: v1 diff --git a/chart/templates/frontend.yaml b/chart/templates/frontend.yaml index d75172ef..6396c240 100644 --- a/chart/templates/frontend.yaml +++ b/chart/templates/frontend.yaml @@ -87,11 +87,9 @@ spec: app: {{ .Values.name }} role: frontend -{{- if .Values.service }} - {{- if .Values.service.type }} - type: {{ .Values.service.type | quote }} + {{- if .Values.nginx_service_use_node_port }} + type: NodePort {{- end }} -{{- end }} ports: - protocol: TCP diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml index b2c92226..a233cd20 100644 --- a/chart/templates/ingress.yaml +++ b/chart/templates/ingress.yaml @@ -7,7 +7,7 @@ metadata: name: ingress-main namespace: {{ .Release.Namespace }} annotations: - kubernetes.io/ingress.class: "nginx" + kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }} nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/enable-cors: "true" @@ -31,16 +31,6 @@ spec: - host: {{ .Values.ingress.host }} http: paths: -{{- if .Values.minio_local }} - - path: /data/(.*) - pathType: Prefix - backend: - service: - name: local-minio - port: - number: 9000 -{{- end }} - - path: /(api/.*) pathType: Prefix backend: @@ -65,7 +55,7 @@ metadata: name: ingress-authsign namespace: {{ .Release.Namespace }} annotations: - kubernetes.io/ingress.class: "nginx" + kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }} nginx.ingress.kubernetes.io/ssl-redirect: "false" nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/upstream-vhost: "{{ .Values.signer.host }}" @@ -85,6 +75,35 @@ spec: {{ end }} + +{{- if .Values.minio_local }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-minio + namespace: {{ .Release.Namespace }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress_class | default "nginx" }} + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/upstream-vhost: "{{ .Values.minio_host }}" + +spec: + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: /data/(.*) + pathType: Prefix + backend: + service: + name: local-minio + port: + number: 9000 + + +{{- end }} + {{ if .Values.ingress.tls }} --- @@ -106,7 +125,7 @@ spec: solvers: - http01: ingress: - class: nginx + class: {{ .Values.ingress_class | default "nginx" }} {{ end }} {{ end }} diff --git a/chart/templates/minio.yaml b/chart/templates/minio.yaml index cfb6017b..9c6421a0 100644 --- a/chart/templates/minio.yaml +++ b/chart/templates/minio.yaml @@ -53,8 +53,13 @@ spec: image: {{ .Values.minio_image }} imagePullPolicy: {{ .Values.minio_pull_policy }} - command: ['/bin/sh'] - args: ["mkdir", "-p", "/data/{{ .Values.minio_local_bucket_name }}" ] + command: + - sh + - -c + - | + mkdir -p /data/{{ .Values.minio_local_bucket_name }} + mkdir -p /data/.minio.sys + echo '{"version":"1","format":"fs","id":"btrix-data-fs","fs":{"version":"2"}}' > /data/.minio.sys/format.json volumeMounts: - name: data-minio @@ -86,7 +91,6 @@ metadata: app: local-minio spec: - type: NodePort selector: app: local-minio diff --git a/chart/templates/mongo.yaml b/chart/templates/mongo.yaml index fc7816dd..f2425f00 100644 --- a/chart/templates/mongo.yaml +++ b/chart/templates/mongo.yaml @@ -59,9 +59,9 @@ spec: requests: storage: 2Gi - {{- if .Values.volume_storage_class }} - storageClassName: {{ .Values.volume_storage_class }} - {{- end }} + {{- if .Values.volume_storage_class }} + storageClassName: {{ .Values.volume_storage_class }} + {{- end }} template: metadata: diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml index 0f446c31..ff1d9d3d 100644 --- a/chart/templates/secrets.yaml +++ b/chart/templates/secrets.yaml @@ -42,17 +42,20 @@ stringData: STORE_SECRET_KEY: "{{ $storage.secret_key }}" {{- if $storage.bucket_name }} - STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}{{ $storage.bucket_name }}" + STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}{{ $storage.bucket_name }}/" {{- else }} STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}" {{- end }} {{- if $storage.access_endpoint_url }} - STORE_ACCESS_ENDPOINT_URL: "{{ $storage.access_endpoint_url }}" + STORE_ACCESS_ENDPOINT_URL: "{{ $storage.access_endpoint_url }}/" + STORE_USE_ACCESS_FOR_PRESIGN: "1" {{- else if and $.Values.ingress.host $.Values.minio_local }} STORE_ACCESS_ENDPOINT_URL: {{ $.Values.ingress.scheme | default "https" }}://{{ $.Values.ingress.host }}/data/{{ $storage.bucket_name }}/ + STORE_USE_ACCESS_FOR_PRESIGN: "0" {{- else }} STORE_ACCESS_ENDPOINT_URL: "{{ $storage.endpoint_url }}" + STORE_USE_ACCESS_FOR_PRESIGN: "0" {{- end }} STORE_REGION: "{{ $storage.region }}" diff --git a/chart/templates/signer.yaml b/chart/templates/signer.yaml index 37d3d8a4..4ab19140 100644 --- a/chart/templates/signer.yaml +++ b/chart/templates/signer.yaml @@ -135,12 +135,6 @@ spec: selector: app: auth-signer -{{- if .Values.service }} - {{- if .Values.service.type }} - type: {{ .Values.service.type | quote }} - {{- end }} -{{- end }} - clusterIP: None ports: - protocol: TCP diff --git a/chart/values.yaml b/chart/values.yaml index c43e6ec0..fa517e3a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -31,7 +31,7 @@ superuser: # API Image # ========================================= -api_image: "docker.io/webrecorder/browsertrix-backend:dev" +api_image: "docker.io/webrecorder/browsertrix-backend:latest" api_pull_policy: "Always" api_password_secret: "c9085f33ecce4347aa1d69339e16c499" @@ -50,7 +50,7 @@ job_memory: "70Mi" # Nginx Image # ========================================= -nginx_image: "docker.io/webrecorder/browsertrix-frontend:dev" +nginx_image: "docker.io/webrecorder/browsertrix-frontend:latest" nginx_pull_policy: "Always" nginx_requests_cpu: "3m" @@ -59,6 +59,10 @@ nginx_limits_cpu: "10m" nginx_requests_memory: "12Mi" nginx_limits_memory: "12Mi" +# if true, will use node port to make the service directly available +# for testing / local deployments only +nginx_service_use_node_port: false + # MongoDB Image # ========================================= @@ -147,7 +151,7 @@ minio_local: true minio_scheme: "http" minio_host: "local-minio.default:9000" -minio_image: minio/minio +minio_image: docker.io/minio/minio:RELEASE.2022-10-24T18-35-07Z minio_mc_image: minio/mc minio_pull_policy: "IfNotPresent" @@ -193,6 +197,8 @@ ingress: scheme: "http" tls: false +ingress_class: nginx + # Signing Options # ========================================= @@ -212,9 +218,8 @@ signer_requests_memory: "36Mi" signer_limits_memory: "96Mi" -# Optional: configure load balancing -service: - type: ClusterIP +# Optional: configure load balancing annotations +# service: # annotations: # service.beta.kubernetes.io/aws-load-balancer-internal: "true" # helm.sh/resource-policy: keep