57a4b6b46f
- collections defined by name per archive
- can update collections with additional metadata (currently just description)
- crawl config api accepts a list of collections by name, resolved to collection uids and stored in config
- finished crawls also associated with collection list
- /archives/{aid}/collections/{name} can list all crawl artifacts (wacz files) from a named collection (in frictionless data package-ish format)
- /archives/{aid}/collections/$all lists all crawled artifacts for the archive
readiness check: add /healthz endpoints for app and nginx
ingress: add /data/ route to local bucket
storage improvements:
- for default storages, store path only, and prepend default storage access endpoint
- collections api returns the paths using the storage access endpoint
- define default storages as secrets in k8s (can support multiple), hard-coded in docker (only one for now)
117 lines
2.2 KiB
YAML
117 lines
2.2 KiB
YAML
name: browsertrix-cloud
|
|
|
|
# API Image
|
|
# =========================================
|
|
api_image: "webrecorder/browsertrix-api"
|
|
api_pull_policy: "Never"
|
|
|
|
api_password_secret: "c9085f33ecce4347aa1d69339e16c499"
|
|
|
|
api_num_replicas: 1
|
|
|
|
api_requests_cpu: "25m"
|
|
api_limits_cpu: "100m"
|
|
|
|
api_requests_memory: "100M"
|
|
api_limits_memory: "256M"
|
|
|
|
|
|
# Nginx Image
|
|
# =========================================
|
|
nginx_image: "nginx"
|
|
nginx_pull_policy: "IfNotPresent"
|
|
|
|
nginx_limit_cpu: "100m"
|
|
nginx_requests_cpu: "25m"
|
|
|
|
|
|
# MongoDB Image
|
|
# =========================================
|
|
mongo_local: true
|
|
|
|
mongo_host: "local-mongo"
|
|
|
|
mongo_image: "mongo"
|
|
mongo_pull_policy: "IfNotPresent"
|
|
|
|
mongo_limit_cpu: "100m"
|
|
mongo_requests_cpu: "25m"
|
|
|
|
mongo_auth:
|
|
username: root
|
|
password: example
|
|
|
|
|
|
# Redis Image
|
|
# =========================================
|
|
redis_local: true
|
|
|
|
redis_image: "redis"
|
|
redis_pull_policy: "IfNotPresent"
|
|
|
|
redis_url: "redis://local-redis.default:6379/1"
|
|
|
|
|
|
|
|
# Crawler Image
|
|
# =========================================
|
|
|
|
crawler_image: "webrecorder/browsertrix-crawler:latest"
|
|
crawler_pull_policy: "Never"
|
|
|
|
crawler_namespace: "crawlers"
|
|
|
|
# num retries
|
|
crawl_retries: 1
|
|
|
|
# browsertrix-crawler args:
|
|
crawler_args: "--timeout 90 --logging stats,behaviors,debug --generateWACZ --screencastPort 9037"
|
|
|
|
|
|
|
|
# Storage
|
|
# =========================================
|
|
|
|
storages:
|
|
- name: "default"
|
|
access_key: "ADMIN"
|
|
secret_key: "PASSW0RD"
|
|
bucket_name: "test-bucket"
|
|
|
|
endpoint_url: "http://local-minio.default:9000/"
|
|
|
|
|
|
# Local Minio Pod (optional)
|
|
# =========================================
|
|
# set to true to use a local minio image
|
|
minio_local: true
|
|
|
|
minio_scheme: "http"
|
|
minio_host: "local-minio.default:9000"
|
|
|
|
minio_image: minio/minio
|
|
minio_mc_image: minio/mc
|
|
minio_pull_policy: "IfNotPresent"
|
|
|
|
|
|
# Deployment options
|
|
# =========================================
|
|
|
|
# Ingress (Optional)
|
|
# Optional: if 'host' is set, a publicly accessible Ingress controller is created with an SSL cert (using letsencrypt)
|
|
ingress:
|
|
host: "btrix.cloud"
|
|
cert_email: "test@example.com"
|
|
scheme: "http"
|
|
tls: false
|
|
|
|
|
|
|
|
|
|
# Optional: configure load balancing
|
|
service:
|
|
type: NodePort
|
|
# annotations:
|
|
# service.beta.kubernetes.io/aws-load-balancer-internal: "true"
|
|
# helm.sh/resource-policy: keep
|