From d2d7240455cedad8e227b2c3c137b02d8aa3a280 Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@users.noreply.github.com>
Date: Wed, 8 Nov 2023 15:08:23 -0800
Subject: [PATCH] background jobs fix: ensure bucket is parsed correctly
 (#1359)

Follow-up to #1321
- correctly parse the endpoint_url into prefix and bucket path
- also add region and s3 provider type to storage secrets
---
 backend/btrixcloud/background_jobs.py |  8 ++++---
 chart/app-templates/replica_job.yaml  | 33 ++++++++++++++++++++-------
 chart/templates/secrets.yaml          |  3 +++
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/backend/btrixcloud/background_jobs.py b/backend/btrixcloud/background_jobs.py
index 91e23bb3..d0e91410 100644
--- a/backend/btrixcloud/background_jobs.py
+++ b/backend/btrixcloud/background_jobs.py
@@ -3,6 +3,8 @@ from datetime import datetime
 from typing import Optional, Tuple, Union, List, Dict, TYPE_CHECKING, cast
 from uuid import UUID
 
+from urllib.parse import urlsplit
+
 from fastapi import APIRouter, Depends, HTTPException
 
 from .storages import StorageOps
@@ -63,9 +65,9 @@ class BackgroundJobOps:
         self.profile_ops = profile_ops
 
     def strip_bucket(self, endpoint_url: str) -> tuple[str, str]:
-        """strip the last path segment (bucket) and return rest of endpoint"""
-        inx = endpoint_url.rfind("/", 0, -1) + 1
-        return endpoint_url[0:inx], endpoint_url[inx:]
+        """split the endpoint_url into the origin and return rest of endpoint as bucket path"""
+        parts = urlsplit(endpoint_url)
+        return parts.scheme + "://" + parts.netloc + "/", parts.path[1:]
 
     async def handle_replica_job_finished(self, job: CreateReplicaJob) -> None:
         """Update replicas in corresponding file objects, based on type"""
diff --git a/chart/app-templates/replica_job.yaml b/chart/app-templates/replica_job.yaml
index 5a0437f2..92ea2b47 100644
--- a/chart/app-templates/replica_job.yaml
+++ b/chart/app-templates/replica_job.yaml
@@ -42,12 +42,21 @@ spec:
               name: "{{ primary_secret_name }}"
               key: STORE_SECRET_KEY
 
+        - name: RCLONE_CONFIG_PRIMARY_REGION
+          valueFrom:
+            secretKeyRef:
+              name: "{{ primary_secret_name }}"
+              key: STORE_REGION
+
+        - name: RCLONE_CONFIG_PRIMARY_PROVIDER
+          valueFrom:
+            secretKeyRef:
+              name: "{{ primary_secret_name }}"
+              key: STORE_S3_PROVIDER
+
         - name: RCLONE_CONFIG_PRIMARY_ENDPOINT
           value: "{{ primary_endpoint }}"
-          #valueFrom:
-          #  secretKeyRef:
-          #    name: "{{ primary_secret_name }}"
-          #    key: STORE_ENDPOINT_URL
+
 {% endif %}
 
         - name: RCLONE_CONFIG_REPLICA_TYPE
@@ -65,12 +74,20 @@ spec:
               name: "{{ replica_secret_name }}"
               key: STORE_SECRET_KEY
 
+        - name: RCLONE_CONFIG_REPLICA_REGION
+          valueFrom:
+            secretKeyRef:
+              name: "{{ replica_secret_name }}"
+              key: STORE_REGION
+
+        - name: RCLONE_CONFIG_REPLICA_PROVIDER
+          valueFrom:
+            secretKeyRef:
+              name: "{{ replica_secret_name }}"
+              key: STORE_S3_PROVIDER
+
         - name: RCLONE_CONFIG_REPLICA_ENDPOINT
           value: "{{ replica_endpoint }}"
-          #valueFrom:
-          #  secretKeyRef:
-          #    name: "{{ replica_secret_name }}"
-          #    key: STORE_ENDPOINT_URL
 
 {% if job_type == BgJobType.CREATE_REPLICA %}
         command: ["rclone", "-vv", "copyto", "--checksum", "primary:{{ primary_file_path }}", "replica:{{ replica_file_path }}"]
diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml
index 9cfc83d4..30201cfa 100644
--- a/chart/templates/secrets.yaml
+++ b/chart/templates/secrets.yaml
@@ -67,5 +67,8 @@ stringData:
   {{- end }}
   STORE_ENDPOINT_NO_BUCKET_URL: "{{ $storage.endpoint_url }}"
 
+  STORE_REGION: "{{ $storage.region }}"
+
+  STORE_S3_PROVIDER: {{ $storage.s3_provider | default "Other" }}
 
 {{- end }}