From 84a9079b1f6774a7d10c6c6caecd09f82f69220f Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 28 Feb 2022 14:32:19 -0800 Subject: [PATCH] support signing in docker deployment: (#166) - add authsign to docker-compose.yml - add signing.sample.yaml to be copied to signing.yaml for authsign - add WACZ_SIGN_URL and WACZ_SIGN_TOKEN to config.sample.env - signing enabled if WACZ_SIGN_URL is set - add instructions on how to enable signing to Deployment - update .gitignore, don't commit 'signing.yaml' - update images to use public repo browsertrix images --- .gitignore | 3 ++- Deployment.md | 17 +++++++++++++++++ backend/dockerman.py | 5 +++++ backend/storages.py | 13 ++++++++----- configs/config.sample.env | 10 ++++++++++ configs/signing.sample.yaml | 21 +++++++++++++++++++++ docker-compose.yml | 25 ++++++++++++++++++++++--- 7 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 configs/signing.sample.yaml diff --git a/.gitignore b/.gitignore index 0dfdd05e..8ef9c0df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ **/*.pyc **/node_modules/ **/config.env -.DS_Store \ No newline at end of file +**/signing.yaml +.DS_Store diff --git a/Deployment.md b/Deployment.md index afe011bf..7333a5a1 100644 --- a/Deployment.md +++ b/Deployment.md @@ -18,10 +18,27 @@ To update/relaunch, use `./docker-restart.sh`. The API should be available at: `http://localhost:8000/docs` +To allow downloading of WACZ files via the UI from a remote host, set the `STORE_ACCESS_ENDPOINT_URL` to use the domain of the host. +Otherwise, the files are accesible only through the default Minio service running on port 9000. + Note: When deployed in local Docker, failed crawls are not retried currently. Scheduling is handled by a subprocess, which stores active schedule in the DB. +### Enabling Signing + +Browsertrix Cloud can optionally sign WACZ files with the same key used to generate an SSL cert. +To use this functionality, the machine running Browsertrix Cloud must be associated with a domain and must have port 80 available on that domain. + +To enable signing in the Docker-based deployment: + +1) Copy `configs/signing.sample.yaml` to `configs/signing.yaml` and set the domain and email fields in the config. Set `staging` to false to generate real certificates. + +2) In `configs.config.env`, also uncomment `WACZ_SIGN_URL`. + + +WACZ files created on minio should now be signed! Be sure to also set `STORE_ACCESS_ENDPOINT_URL` to get downloadable links from the UI downloads view. + ## Deploying to Kubernetes diff --git a/backend/dockerman.py b/backend/dockerman.py index adc3c860..825327af 100644 --- a/backend/dockerman.py +++ b/backend/dockerman.py @@ -40,6 +40,9 @@ class DockerManager: self.crawl_args = os.environ["CRAWL_ARGS"] + self.wacz_sign_url = os.environ.get("WACZ_SIGN_URL", "") + self.wacz_sign_token = os.environ.get("WACZ_SIGN_TOKEN", "") + self.archive_ops = archive_ops self.crawl_ops = None @@ -505,6 +508,8 @@ class DockerManager: f"STORE_PATH={storage_path}", f"WEBHOOK_URL={self.redis_url}/{self.crawls_done_key}", f"CRAWL_ARGS={self.crawl_args}", + f"WACZ_SIGN_URL={self.wacz_sign_url}", + f"WACZ_SIGN_TOKEN={self.wacz_sign_token}", ] labels["btrix.run.schedule"] = schedule diff --git a/backend/storages.py b/backend/storages.py index 1fa1d181..5a807b0e 100644 --- a/backend/storages.py +++ b/backend/storages.py @@ -54,12 +54,15 @@ def init_storages_api(archive_ops, crawl_manager, user_dep): # ============================================================================ @asynccontextmanager -async def get_s3_client(storage): +async def get_s3_client(storage, use_access=False): """ context manager for s3 client""" - if not storage.endpoint_url.endswith("/"): - storage.endpoint_url += "/" + endpoint_url = ( + storage.endpoint_url if not use_access else storage.access_endpoint_url + ) + if not endpoint_url.endswith("/"): + endpoint_url += "/" - parts = urlsplit(storage.endpoint_url) + parts = urlsplit(endpoint_url) bucket, key = parts.path[1:].split("/", 1) endpoint_url = parts.scheme + "://" + parts.netloc @@ -100,7 +103,7 @@ async def get_presigned_url(archive, crawlfile, crawl_manager, duration=3600): else: raise Exception("No Default Storage Found, Invalid Storage Type") - async with get_s3_client(s3storage) as (client, bucket, key): + async with get_s3_client(s3storage, True) as (client, bucket, key): key += crawlfile.filename presigned_url = await client.generate_presigned_url( diff --git a/configs/config.sample.env b/configs/config.sample.env index 8f02a84c..af0fa70c 100644 --- a/configs/config.sample.env +++ b/configs/config.sample.env @@ -38,4 +38,14 @@ REGISTRATION_ENABLED=1 JWT_TOKEN_LIFETIME_MINUTES=60 +# auth sign -- uncomment to enable signing +# WACZ_SIGN_URL="http://authsign:8080/sign" + +# optional token for signing (useful if using remote signing server) +# WACZ_SIGN_TOKEN="" + + + + + diff --git a/configs/signing.sample.yaml b/configs/signing.sample.yaml new file mode 100644 index 00000000..bb7593ce --- /dev/null +++ b/configs/signing.sample.yaml @@ -0,0 +1,21 @@ +signing: + domain: example.com # domain to retrieve a cert for (passed to ACME servers, required) + email: test@example.com # email for acme auth (passed to ACME servers, required) + port: 80 # local port for acme domain check (should be 80, change if running behind a proxy) + + output: /data # dir to store the keys and certs (for internal use) + + staging: False # generate staging certs + + # optional: set a 'cross-singing' CA and private key + # this will be used along with ACME (Lets Encrypt) to sign the same CSR + # csca_cert: + # csca_private_key: + + # rfc3161 timestamp authority cert chain + timestamp urls + # at least one required, if multiple, one is selected at random + timestamping: + # time server cert chain (cert + ca cert) + # pkg:// url to load from python package data + - certfile: pkg://authsign.trusted/ts-chain.pem + url: http://freetsa.org/tsr # timeserver URL diff --git a/docker-compose.yml b/docker-compose.yml index dcad19c9..46031d32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.5' services: backend: build: ./backend - image: registry.digitalocean.com/btrix/webrecorder/browsertrix-api + image: webrecorder/browsertrix-backend volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -20,7 +20,7 @@ services: frontend: build: ./frontend - image: registry.digitalocean.com/btrix/webrecorder/browsertrix-frontend + image: webrecorder/browsertrix-frontend ports: - 9871:80 @@ -50,7 +50,7 @@ services: minio: image: minio/minio - command: server /data --console-address :9001 + command: ["server", "/data", "--console-address", ":9001"] ports: - 9000:9000 - 9001:9001 @@ -77,10 +77,29 @@ services: image: tianon/true container_name: volume_placeholder +# enable to support signing of wacz files +# port 80 must be open to automatically generate cert via LetsEncrypt + authsign: + image: webrecorder/authsign:0.3.1 + + volumes: + - btrix-sign-data:/data + - ./configs/signing.yaml:/app-config/signing.yaml + + environment: + - CONFIG=/app-config/signing.yaml + + env_file: + - ./configs/config.env + + ports: + - 80:80 + volumes: btrix-redis-data: btrix-mongo-data: btrix-minio-data: + btrix-sign-data: networks: