0c8a5a49b4
- use python-on-whale to use docker cli api directly, creating docker stack for each crawl or profile browser - configure storages via storages.yaml secret - add crawl_job, profile_job, splitting into base and k8s/swarm implementations - split manager into base crawlmanager and k8s/swarm implementations - swarm: load initial scale from db to avoid modifying fixed configs, in k8s, load from configmap - swarm: support scheduled jobs via swarm-cronjob service - remove docker dependencies (aiodocker, apscheduler, scheduling) - swarm: when using local minio, expose via /data/ route in nginx via extra include (in k8s, include dir is empty and routing handled via ingress) - k8s: cleanup minio chart: move init containers to minio.yaml - swarm: stateful set implementation to be consistent with k8s scaling: - don't use service replicas, - create a unique service with '-N' appended and allocate unique volume for each replica - allows crawl containers to be restarted w/o losing data - add volume pruning background service, as volumes can be deleted only after service shuts down fully - watch: fully simplify routing, route via replica index instead of ip for both k8s and swarm - rename network btrix-cloud-net -> btrix-net to avoid conflict with compose network
147 lines
2.8 KiB
YAML
147 lines
2.8 KiB
YAML
version: '3.8'
|
|
|
|
services:
|
|
backend:
|
|
build: ./backend
|
|
image: ${REGISTRY}webrecorder/browsertrix-backend:latest
|
|
volumes:
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
|
|
configs:
|
|
- shared_job_config
|
|
|
|
secrets:
|
|
- storages
|
|
|
|
env_file:
|
|
- ./configs/config.env
|
|
|
|
depends_on:
|
|
- minio
|
|
- mongo
|
|
|
|
# enable to expose backend api container directly
|
|
# ports:
|
|
# - 8000:8000
|
|
|
|
environment:
|
|
- CRAWLER_FQDN_SUFFIX=
|
|
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.role == manager
|
|
|
|
frontend:
|
|
build: ./frontend
|
|
image: ${REGISTRY}webrecorder/browsertrix-frontend:latest
|
|
ports:
|
|
- 9871:80
|
|
|
|
depends_on:
|
|
- backend
|
|
- minio
|
|
|
|
environment:
|
|
- BACKEND_HOST=backend
|
|
- CRAWLER_FQDN_SUFFIX=
|
|
- CRAWLER_SVC_SUFFIX=
|
|
|
|
swarm-cronjob:
|
|
image: crazymax/swarm-cronjob
|
|
volumes:
|
|
- "/var/run/docker.sock:/var/run/docker.sock"
|
|
|
|
environment:
|
|
- "TZ=UTC"
|
|
- "LOG_LEVEL=debug"
|
|
- "LOG_JSON=false"
|
|
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.role == manager
|
|
|
|
prune-resources:
|
|
image: docker
|
|
command: ["docker", "volumes", "prune", "-f", "--filter", "label=crawl"]
|
|
volumes:
|
|
- "/var/run/docker.sock:/var/run/docker.sock"
|
|
deploy:
|
|
mode: global
|
|
labels:
|
|
- "swarm.cronjob.enable=true"
|
|
- "swarm.cronjob.schedule=0 */5 * * *"
|
|
- "swarm.cronjob.skip-running=false"
|
|
restart_policy:
|
|
condition: none
|
|
|
|
mongo:
|
|
image: mongo
|
|
volumes:
|
|
- btrix-mongo-data:/data/db
|
|
|
|
env_file:
|
|
- ./configs/config.env
|
|
|
|
minio:
|
|
image: minio/minio
|
|
#command: ["server", "/data", "--console-address", ":9001"]
|
|
|
|
entrypoint: "/bin/sh"
|
|
# auto-create bucket via existing dir (only for single-node mode)
|
|
command: ["-c", "mkdir -p /data/$$MINIO_BUCKET; /usr/bin/docker-entrypoint.sh server /data --console-address :9001"]
|
|
|
|
volumes:
|
|
- btrix-minio-data:/data
|
|
|
|
env_file:
|
|
- ./configs/config.env
|
|
|
|
# enable below to access minio console directly
|
|
# ports:
|
|
# - 9001:9001
|
|
#
|
|
# environment:
|
|
# - MINIO_SERVER_URL=http://127.0.0.1:9000
|
|
|
|
|
|
# enable to support signing of wacz files
|
|
# port 80 must be open to automatically generate cert via LetsEncrypt
|
|
# authsign:
|
|
# image: webrecorder/authsign:0.4.0
|
|
|
|
# volumes:
|
|
# - btrix-sign-data:/data
|
|
# - ./configs/signing.yaml:/app-config/signing.yaml
|
|
|
|
# environment:
|
|
# - CONFIG=/app-config/signing.yaml
|
|
|
|
# env_file:
|
|
# - ./configs/config.env
|
|
|
|
# ports:
|
|
# - 80:80
|
|
|
|
volumes:
|
|
btrix-redis-data:
|
|
btrix-mongo-data:
|
|
btrix-minio-data:
|
|
btrix-sign-data:
|
|
|
|
networks:
|
|
default:
|
|
name: btrix-net
|
|
|
|
configs:
|
|
shared_job_config:
|
|
file: ./configs/config.yaml
|
|
|
|
secrets:
|
|
storages:
|
|
file: ./configs/storages.yaml
|
|
|
|
|
|
|