From a9ab17fc61e24450bce7adc9082a94eb87bc6270 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 30 Aug 2023 12:02:02 -0700 Subject: [PATCH] publish helm chart on release (fixes #1114) (#1117) (#1123) - no longer using :latest by default in values.yaml, instead updating version with each release - set chart version to match app version in Chart.yaml - update version in helm chart and values.yaml as part of update-version.sh script - update test.yaml and local-config.yaml to enable using :latest tag images - ci: add ci script for packaging current helm chart - docs: updates docs to indicate deploying directly from GitHub release - docs: add script to fill in latest version for 'VERSION' using custom script - chart: set local_service_port to 30870 by default, but use only if no ingress. - default values.yaml set up for local deployment, local-config.yaml contains additional commented out examples - ci draft: add deployment info to draft with helm install command for current version - test: fix password check test --- .github/workflows/publish-helm-chart.yaml | 47 +++++++++++++++++++++ chart/Chart.yaml | 11 +---- chart/examples/local-config.yaml | 14 +++++-- chart/templates/frontend.yaml | 15 +++---- chart/test/test.yaml | 4 ++ chart/values.yaml | 7 ++-- docs/deploy/local.md | 51 ++++++++++++++++------- docs/js/insertversion.js | 36 ++++++++++++++++ mkdocs.yml | 2 + scripts/check_passwords.py | 3 +- update-version.sh | 5 +++ 11 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/publish-helm-chart.yaml create mode 100644 docs/js/insertversion.js diff --git a/.github/workflows/publish-helm-chart.yaml b/.github/workflows/publish-helm-chart.yaml new file mode 100644 index 00000000..f1425805 --- /dev/null +++ b/.github/workflows/publish-helm-chart.yaml @@ -0,0 +1,47 @@ +name: Publish Helm Chart + +on: + push: + branches: + - main + +jobs: + package_chart: + runs-on: ubuntu-latest + + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: 3.10.2 + + - name: Package Chart + run: | + mkdir .chart-out + helm package chart/ --destination .chart-out + + - name: Get Version + run: | + echo "version=$(cat version.txt)" >> "$GITHUB_ENV" + + - name: Upload Chart To GitHub + uses: softprops/action-gh-release@v1 + with: + body: | + ## Deployment + + This release can be deployed with Helm by running: + + ``` + helm upgrade --install btrix https://github.com/webrecorder/browsertrix-cloud/releases/download/v${{ env.version }}/browsertrix-cloud-${{ env.version }}.tgz + ``` + + See [the development guide](https://docs.browsertrix.cloud/deploy/) for more info how to deploy Browsertrix Cloud. + + files: .chart-out/browsertrix-cloud-${{ env.version }}.tgz + tag_name: v${{ env.version }} + fail_on_unmatched_files: true + draft: true diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 38d0c6fb..50ce66a7 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -4,15 +4,8 @@ description: A chart for running the Webrecorder Browsertrix System type: application icon: https://webrecorder.net/assets/icon.png -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.1.0 +# Browsertrix Cloud and Chart Version +version: 1.7.0-beta.0 dependencies: - name: btrix-admin-logging diff --git a/chart/examples/local-config.yaml b/chart/examples/local-config.yaml index cc4aebdd..11976bf5 100644 --- a/chart/examples/local-config.yaml +++ b/chart/examples/local-config.yaml @@ -4,12 +4,18 @@ # This config is designed for local (non-hosted) use, with either Docker Desktop, minikube or local microk8s. # With this setup, the cluster should be accessed locally via 'localhost' -# run service on localhost on port -local_service_port: 30870 +# use custom port for local access, default is localhost:30870 +# uncomment and change below +# local_service_port: 30870 -default_org: "My Local Organization" +# Uncomment to set custom organization name +# default_org: "My Local Organization" -# overrides to use existing images in local Docker, otherwise will pull from latest +# use version specified in values.yaml, uncomment to use :latest release instead +# backend_image: docker.io/webrecorder/browsertrix-backend:latest +# frontend_image: docker.io/webrecorder/browsertrix-frontend:latest + +# overrides to use existing images in local Docker, otherwise will pull from repository # backend_pull_policy: "Never" # frontend_pull_policy: "Never" # crawler_pull_policy: "Never" diff --git a/chart/templates/frontend.yaml b/chart/templates/frontend.yaml index 8564ad7e..1bee7fc4 100644 --- a/chart/templates/frontend.yaml +++ b/chart/templates/frontend.yaml @@ -95,15 +95,16 @@ spec: app: {{ .Values.name }} role: frontend - {{- if .Values.local_service_port }} - type: NodePort - {{- end }} - ports: - protocol: TCP port: 80 name: frontend - {{- if .Values.local_service_port }} - nodePort: {{ .Values.local_service_port }} - {{- end }} + + {{- if and .Values.local_service_port (not .Values.ingress.host) }} + nodePort: {{ .Values.local_service_port }} + + type: NodePort + {{- end }} + + diff --git a/chart/test/test.yaml b/chart/test/test.yaml index dc605d5e..86f8bea4 100644 --- a/chart/test/test.yaml +++ b/chart/test/test.yaml @@ -1,6 +1,10 @@ # test overrides # -------------- +# use local images built to :latest tag +backend_image: docker.io/webrecorder/browsertrix-backend:latest +frontend_image: docker.io/webrecorder/browsertrix-frontend:latest + backend_pull_policy: "Never" frontend_pull_policy: "Never" diff --git a/chart/values.yaml b/chart/values.yaml index 2448d1e8..4aaa83f0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -88,7 +88,7 @@ default_org: "My Organization" # API Image # ========================================= -backend_image: "docker.io/webrecorder/browsertrix-backend:latest" +backend_image: "docker.io/webrecorder/browsertrix-backend:1.7.0-beta.0" backend_pull_policy: "Always" backend_password_secret: "PASSWORD!" @@ -114,7 +114,7 @@ profile_browser_idle_seconds: 60 # Nginx Image # ========================================= -frontend_image: "docker.io/webrecorder/browsertrix-frontend:latest" +frontend_image: "docker.io/webrecorder/browsertrix-frontend:1.7.0-beta.0" frontend_pull_policy: "Always" frontend_cpu: "5m" @@ -124,7 +124,8 @@ frontend_memory: "36Mi" # if set, maps nginx to a fixed port on host machine # must be between 30000 - 32767 # use for deployments on localhost when not using ingress -# local_service_port: 30870 +# if using ingress, this value is ignored +local_service_port: 30870 # MongoDB Image diff --git a/docs/deploy/local.md b/docs/deploy/local.md index 777c446e..f7f516b1 100644 --- a/docs/deploy/local.md +++ b/docs/deploy/local.md @@ -8,7 +8,10 @@ Before running Browsertrix Cloud, you'll need to set up a running [Kubernetes](h Today, there are numerous ways to deploy Kubernetes fairly easily, and we recommend trying one of the single-node options, which include Docker Desktop, microk8s, minikube and k3s. -The instructions below assume you have cloned the [https://github.com/webrecorder/browsertrix-cloud](https://github.com/webrecorder/browsertrix-cloud) repository locally, and have local package managers for your platform (eg. `brew` for macOS, `choco` for Windows, etc...) already installed. +The instructions below assume you have the local package managers for your platform (eg. `brew` for macOS, `choco` for Windows, etc...) already installed. + +Cloning the repository at [https://github.com/webrecorder/browsertrix-cloud](https://github.com/webrecorder/browsertrix-cloud) is only needed +to access additional configuration files. Here are some environment specific instructions for setting up a local cluster from different Kubernetes vendors: @@ -54,34 +57,53 @@ Here are some environment specific instructions for setting up a local cluster f ## Launching Browsertrix Cloud with Helm -Once you have a running Kubernetes cluster with one of the options above, and Helm 3 installed, you can then run from the Browsertrix Cloud repo directory: +Once you have a running Kubernetes cluster with one of the options above, and Helm 3 installed, you can install the latest release of Browsertrix Cloud +directly from the latest GitHub release. -```shell -helm upgrade --install -f ./chart/values.yaml \ --f ./chart/examples/local-config.yaml btrix ./chart/ + + + +``` +helm upgrade --install btrix \ +https://github.com/webrecorder/browsertrix-cloud/releases/download/VERSION/browsertrix-cloud-VERSION.tgz ``` ??? info "MicroK8S" If using microk8s, the command will be: + + ```sh - microk8s helm3 upgrade --install -f ./chart/values.yaml \ - -f ./chart/examples/local-config.yaml btrix ./chart/ + microk8s helm3 upgrade --install btrix \ + https://github.com/webrecorder/browsertrix-cloud/releases/download/VERSION/browsertrix-cloud-VERSION.tgz ``` Subsequent commands will also use `microk8s helm3` instead of `helm`. -The local setup includes the full Browsertrix Cloud system, with frontend, backend api, db (via MongoDB) and storage (via Minio) +The default setup includes the full Browsertrix Cloud system, with frontend, backend api, db (via MongoDB) and storage (via Minio) An admin user with name `admin@example.com` and password `PASSW0RD!` will be automatically created. -This config uses the standard config (`./chart/values.yaml`) with a couple additional settings for local deployment (`./chart/examples/local-config.yaml`). With Helm, additional YAML files can be added to further override previous settings. +With Helm, additional YAML files can be added to further override previous settings. -These settings can be changed in [charts/examples/local-config.yaml](https://github.com/webrecorder/browsertrix-cloud/blob/main/chart/examples/local-config.yaml). +Some possible settings can be changed are found in [chart/examples/local-config.yaml](https://github.com/webrecorder/browsertrix-cloud/blob/main/chart/examples/local-config.yaml). + +For example, to change the default superadmin, uncomment the `superadmin` block in `local-config.yaml`, and then change the username (`admin@example.com`) and password (`PASSW0RD!`) to different values. (The admin username and password will be updated with each deployment). +To change the local port, change `local_service_port` setting. + +You can then redeploy with these additional settings by running: + + + +``` +helm upgrade --install btrix https://github.com/webrecorder/browsertrix-cloud/releases/download/VERSION/browsertrix-cloud-VERSION.tgz \ +-f ./chart/examples/local-config.yaml +``` + +The above examples assumes running from a cloned Browsertrix Cloud repo, however the config file can be saved anywhere and specified with `-f `. -For example, to change the default superadmin, uncomment the `superadmin` block in `local-config.yaml`, and then change the username (`admin@example.com`) and password (`PASSW0RD!`) to different values. (The admin username and password will be updated with each deployment) ## Waiting for Cluster to Start @@ -128,10 +150,11 @@ The outputs of these commands will be helpful if you'd like to report an issue [ ## Updating the Cluster -To update the cluster, re-run the same command again, which will pull the latest images. In this way, you can upgrade to the latest release of Browsertrix Cloud. The upgrade will preserve the database and current archives. +To update the cluster, for example to update to new version `NEWVERSION`, re-run the same command again, which will pull the latest images. In this way, you can upgrade to the latest release of Browsertrix Cloud. The upgrade will preserve the database and current archives. ```shell -helm upgrade --install -f ./chart/values.yaml -f ./chart/examples/local-config.yaml btrix ./chart/ +helm upgrade --install btrix https://github.com/webrecorder/browsertrix-cloud/releases/download/NEWVERSION/browsertrix-cloud-NEWVERSION.tgz + ``` ## Uninstalling @@ -148,4 +171,4 @@ To fully delete all persistent data (db + archives) created in the cluster, also ## Deploying for Local Development -These instructions are intended for deploying the cluster from the latest release. See [setting up cluster for local development](../develop/local-dev-setup.md) for additional customizations related to developing Browsertrix Cloud and deploying from local images. +These instructions are intended for deploying the cluster from the latest releases published on GitHub. See [setting up cluster for local development](../develop/local-dev-setup.md) for additional customizations related to developing Browsertrix Cloud and deploying from local images. diff --git a/docs/js/insertversion.js b/docs/js/insertversion.js new file mode 100644 index 00000000..c9a8f2e4 --- /dev/null +++ b/docs/js/insertversion.js @@ -0,0 +1,36 @@ +const KEY = "/.__source"; +let retries = 0; + +function loadVersion() { + const value = self.sessionStorage.getItem(KEY); + if (value) { + parseVersion(value); + } else if (retries++ < 10) { + setTimeout(loadVersion, 500); + } +} + +function parseVersion(string) { + const version = JSON.parse(string).version; + if (!version) { + return; + } + + const elems = document.querySelectorAll("insert-version"); + for (const elem of elems) { + try { + const code = elem.parentElement.nextElementSibling.querySelector("code"); + code.childNodes.forEach((node) => { + if (node.nodeType === Node.TEXT_NODE) { + node.nodeValue = node.nodeValue.replaceAll("VERSION", version); + } + }); + } catch (e) { + + } + } +} + +if (window.location.pathname.startsWith("/deploy/local")) { + window.addEventListener("load", () => loadVersion()); +} diff --git a/mkdocs.yml b/mkdocs.yml index 33985325..378fa6e2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,8 @@ repo_name: Browsertrix Cloud edit_uri: edit/main/docs/ extra_css: - stylesheets/extra.css +extra_javascript: + - js/insertversion.js theme: name: material custom_dir: docs/overrides diff --git a/scripts/check_passwords.py b/scripts/check_passwords.py index 4ed4978e..e7cf5d41 100644 --- a/scripts/check_passwords.py +++ b/scripts/check_passwords.py @@ -1,6 +1,7 @@ "A small dirty script to check that none of the password config options have been set to real passwords" from collections.abc import Generator import yaml +from yaml.parser import ParserError from yaml.scanner import ScannerError from yaml.constructor import ConstructorError import sys @@ -45,7 +46,7 @@ for file in changed_files: else: print(f"top level key '{password_keys[0]}' with subkey '{password_keys[1]}' in {file} contains a real password!") WE_DUN_GOOFED = True - except ScannerError: + except (ScannerError, ParserError): print(f"Couldn't parse yaml file for: {file}") pass except ConstructorError: diff --git a/update-version.sh b/update-version.sh index 51e4682f..f9455f55 100755 --- a/update-version.sh +++ b/update-version.sh @@ -6,3 +6,8 @@ mv ./tmp-package.json ./frontend/package.json echo '""" current version """' > ./backend/btrixcloud/version.py echo "__version__ = \"$version\"" >> ./backend/btrixcloud/version.py + +sed -E -i "" "s/^version:.*$/version: $version/" chart/Chart.yaml + +sed -E -i "" "s/\/browsertrix-backend:[[:alnum:].-]+/\/browsertrix-backend:$version/" chart/values.yaml +sed -E -i "" "s/\/browsertrix-frontend:[[:alnum:].-]+/\/browsertrix-frontend:$version/" chart/values.yaml