support signing in docker deployment: (#166)

- add authsign to docker-compose.yml
- add signing.sample.yaml to be copied to signing.yaml for authsign
- add WACZ_SIGN_URL and WACZ_SIGN_TOKEN to config.sample.env
- signing enabled if WACZ_SIGN_URL is set
- add instructions on how to enable signing to Deployment
- update .gitignore, don't commit 'signing.yaml'
- update images to use public repo browsertrix images

.gitignore

@@ -1,4 +1,5 @@
 **/*.pyc
 **/node_modules/
 **/config.env
-.DS_Store
+**/signing.yaml
+.DS_Store

Deployment.md

@@ -18,10 +18,27 @@ To update/relaunch, use `./docker-restart.sh`.
 
 The API should be available at: `http://localhost:8000/docs`
 
+To allow downloading of WACZ files via the UI from a remote host, set the `STORE_ACCESS_ENDPOINT_URL` to use the domain of the host.
+Otherwise, the files are accesible only through the default Minio service running on port 9000.
+
 
 Note: When deployed in local Docker, failed crawls are not retried currently. Scheduling is handled by a subprocess, which stores active schedule in the DB.
 
 
+### Enabling Signing
+
+Browsertrix Cloud can optionally sign WACZ files with the same key used to generate an SSL cert.
+To use this functionality, the machine running Browsertrix Cloud must be associated with a domain and must have port 80 available on that domain.
+
+To enable signing in the Docker-based deployment:
+
+1) Copy `configs/signing.sample.yaml` to `configs/signing.yaml` and set the domain and email fields in the config. Set `staging` to false to generate real certificates.
+
+2) In `configs.config.env`, also uncomment `WACZ_SIGN_URL`.
+
+
+WACZ files created on minio should now be signed! Be sure to also set `STORE_ACCESS_ENDPOINT_URL` to get downloadable links from the UI downloads view.
+
 
 ## Deploying to Kubernetes
 

backend/dockerman.py

@@ -40,6 +40,9 @@ class DockerManager:
 
         self.crawl_args = os.environ["CRAWL_ARGS"]
 
+        self.wacz_sign_url = os.environ.get("WACZ_SIGN_URL", "")
+        self.wacz_sign_token = os.environ.get("WACZ_SIGN_TOKEN", "")
+
         self.archive_ops = archive_ops
         self.crawl_ops = None
 
@@ -505,6 +508,8 @@ class DockerManager:
             f"STORE_PATH={storage_path}",
             f"WEBHOOK_URL={self.redis_url}/{self.crawls_done_key}",
             f"CRAWL_ARGS={self.crawl_args}",
+            f"WACZ_SIGN_URL={self.wacz_sign_url}",
+            f"WACZ_SIGN_TOKEN={self.wacz_sign_token}",
         ]
 
         labels["btrix.run.schedule"] = schedule

backend/storages.py

@@ -54,12 +54,15 @@ def init_storages_api(archive_ops, crawl_manager, user_dep):
 
 # ============================================================================
 @asynccontextmanager
-async def get_s3_client(storage):
+async def get_s3_client(storage, use_access=False):
     """ context manager for s3 client"""
-    if not storage.endpoint_url.endswith("/"):
-        storage.endpoint_url += "/"
+    endpoint_url = (
+        storage.endpoint_url if not use_access else storage.access_endpoint_url
+    )
+    if not endpoint_url.endswith("/"):
+        endpoint_url += "/"
 
-    parts = urlsplit(storage.endpoint_url)
+    parts = urlsplit(endpoint_url)
     bucket, key = parts.path[1:].split("/", 1)
 
     endpoint_url = parts.scheme + "://" + parts.netloc
@@ -100,7 +103,7 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600):
     else:
         raise Exception("No Default Storage Found, Invalid Storage Type")
 
-    async with get_s3_client(s3storage) as (client, bucket, key):
+    async with get_s3_client(s3storage, True) as (client, bucket, key):
         key += crawlfile.filename
 
         presigned_url = await client.generate_presigned_url(

configs/config.sample.env

@@ -38,4 +38,14 @@ REGISTRATION_ENABLED=1
 
 JWT_TOKEN_LIFETIME_MINUTES=60
 
+# auth sign -- uncomment to enable signing
+# WACZ_SIGN_URL="http://authsign:8080/sign"
+
+# optional token for signing (useful if using remote signing server)
+# WACZ_SIGN_TOKEN=""
+
+
+
+
+
 

configs/signing.sample.yaml

@@ -0,0 +1,21 @@
+signing:
+  domain: example.com  # domain to retrieve a cert for (passed to ACME servers, required)
+  email: test@example.com  # email for acme auth (passed to ACME servers, required)
+  port: 80  # local port for acme domain check (should be 80, change if running behind a proxy)
+
+  output: /data  # dir to store the keys and certs (for internal use)
+
+  staging: False  # generate staging certs
+
+  # optional: set a 'cross-singing' CA and private key
+  # this will be used along with ACME (Lets Encrypt) to sign the same CSR
+  # csca_cert: <ca-cert.pem>
+  # csca_private_key: <ca-private-key.pem>
+
+  # rfc3161 timestamp authority cert chain + timestamp urls
+  # at least one required, if multiple, one is selected at random
+  timestamping:
+    # time server cert chain (cert + ca cert)
+    # pkg:// url to load from python package data
+    - certfile: pkg://authsign.trusted/ts-chain.pem
+      url: http://freetsa.org/tsr  # timeserver URL

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.5'
 services:
   backend:
     build: ./backend
-    image: registry.digitalocean.com/btrix/webrecorder/browsertrix-api
+    image: webrecorder/browsertrix-backend
     volumes:
         - /var/run/docker.sock:/var/run/docker.sock
 
@@ -20,7 +20,7 @@ services:
 
   frontend:
     build: ./frontend
-    image: registry.digitalocean.com/btrix/webrecorder/browsertrix-frontend
+    image: webrecorder/browsertrix-frontend
     ports:
       - 9871:80
 
@@ -50,7 +50,7 @@ services:
 
   minio:
     image: minio/minio
-    command: server /data --console-address :9001
+    command: ["server", "/data", "--console-address", ":9001"]
     ports:
       - 9000:9000
       - 9001:9001
@@ -77,10 +77,29 @@ services:
     image: tianon/true
     container_name: volume_placeholder
 
+# enable to support signing of wacz files
+# port 80 must be open to automatically generate cert via LetsEncrypt
+  authsign:
+    image: webrecorder/authsign:0.3.1
+
+    volumes:
+      - btrix-sign-data:/data
+      - ./configs/signing.yaml:/app-config/signing.yaml
+
+    environment:
+      - CONFIG=/app-config/signing.yaml
+
+    env_file:
+      - ./configs/config.env
+
+    ports:
+      - 80:80
+
 volumes:
   btrix-redis-data:
   btrix-mongo-data:
   btrix-minio-data:
+  btrix-sign-data:
 
 
 networks: