background jobs fix: ensure bucket is parsed correctly (#1359)

Follow-up to #1321 
- correctly parse the endpoint_url into prefix and bucket path
- also add region and s3 provider type to storage secrets

backend/btrixcloud/background_jobs.py

@@ -3,6 +3,8 @@ from datetime import datetime
 from typing import Optional, Tuple, Union, List, Dict, TYPE_CHECKING, cast
 from uuid import UUID
 
+from urllib.parse import urlsplit
+
 from fastapi import APIRouter, Depends, HTTPException
 
 from .storages import StorageOps
@@ -63,9 +65,9 @@ class BackgroundJobOps:
         self.profile_ops = profile_ops
 
     def strip_bucket(self, endpoint_url: str) -> tuple[str, str]:
-        """strip the last path segment (bucket) and return rest of endpoint"""
-        inx = endpoint_url.rfind("/", 0, -1) + 1
-        return endpoint_url[0:inx], endpoint_url[inx:]
+        """split the endpoint_url into the origin and return rest of endpoint as bucket path"""
+        parts = urlsplit(endpoint_url)
+        return parts.scheme + "://" + parts.netloc + "/", parts.path[1:]
 
     async def handle_replica_job_finished(self, job: CreateReplicaJob) -> None:
         """Update replicas in corresponding file objects, based on type"""

chart/app-templates/replica_job.yaml

@@ -42,12 +42,21 @@ spec:
               name: "{{ primary_secret_name }}"
               key: STORE_SECRET_KEY
 
+        - name: RCLONE_CONFIG_PRIMARY_REGION
+          valueFrom:
+            secretKeyRef:
+              name: "{{ primary_secret_name }}"
+              key: STORE_REGION
+
+        - name: RCLONE_CONFIG_PRIMARY_PROVIDER
+          valueFrom:
+            secretKeyRef:
+              name: "{{ primary_secret_name }}"
+              key: STORE_S3_PROVIDER
+
         - name: RCLONE_CONFIG_PRIMARY_ENDPOINT
           value: "{{ primary_endpoint }}"
-          #valueFrom:
-          #  secretKeyRef:
-          #    name: "{{ primary_secret_name }}"
-          #    key: STORE_ENDPOINT_URL
+
 {% endif %}
 
         - name: RCLONE_CONFIG_REPLICA_TYPE
@@ -65,12 +74,20 @@ spec:
               name: "{{ replica_secret_name }}"
               key: STORE_SECRET_KEY
 
+        - name: RCLONE_CONFIG_REPLICA_REGION
+          valueFrom:
+            secretKeyRef:
+              name: "{{ replica_secret_name }}"
+              key: STORE_REGION
+
+        - name: RCLONE_CONFIG_REPLICA_PROVIDER
+          valueFrom:
+            secretKeyRef:
+              name: "{{ replica_secret_name }}"
+              key: STORE_S3_PROVIDER
+
         - name: RCLONE_CONFIG_REPLICA_ENDPOINT
           value: "{{ replica_endpoint }}"
-          #valueFrom:
-          #  secretKeyRef:
-          #    name: "{{ replica_secret_name }}"
-          #    key: STORE_ENDPOINT_URL
 
 {% if job_type == BgJobType.CREATE_REPLICA %}
         command: ["rclone", "-vv", "copyto", "--checksum", "primary:{{ primary_file_path }}", "replica:{{ replica_file_path }}"]

chart/templates/secrets.yaml

@@ -67,5 +67,8 @@ stringData:
   {{- end }}
   STORE_ENDPOINT_NO_BUCKET_URL: "{{ $storage.endpoint_url }}"
 
+  STORE_REGION: "{{ $storage.region }}"
+
+  STORE_S3_PROVIDER: {{ $storage.s3_provider | default "Other" }}
 
 {{- end }}