6dc452ebad
- Refactors storage to support replicas + custom storages on the Org. - There is a default primary + replica storage, while an Org can also have primary and replica storages. - StorageRef object is used to store references to default and custom storage. - CrawlFile has been updated to contain a StorageRef instead of a def_storage_name, which references either a default storage (in StorageOps) or custom storage (in Organization) - There is also a 'replicas' Optional[List[StorageRef]] which contains replicas, if any. - CrawlFileOut contain a numReplicas for how many replicas exist for a given file. - Migration: migration 0020 added to migrate existing Orgs, CrawlFile and ProfileFile objects to new storage system (CrawlFile and ProfileFile now extend BaseFile) Part of #1262 --------- Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>
81 lines
1.8 KiB
YAML
81 lines
1.8 KiB
YAML
# test overrides
|
|
# --------------
|
|
|
|
# use local images built to :latest tag
|
|
backend_image: docker.io/webrecorder/browsertrix-backend:latest
|
|
frontend_image: docker.io/webrecorder/browsertrix-frontend:latest
|
|
|
|
crawler_iamge: docker.io/webrecorder/browsertrix-crawler:0.12.0-beta.1
|
|
|
|
backend_pull_policy: "Never"
|
|
frontend_pull_policy: "Never"
|
|
|
|
default_crawl_filename_template: "@ts-testing-@hostsuffix.wacz"
|
|
|
|
operator_resync_seconds: 3
|
|
|
|
# for testing only
|
|
crawler_extra_cpu_per_browser: 300m
|
|
|
|
crawler_extra_memory_per_browser: 256Mi
|
|
|
|
mongo_auth:
|
|
# specify either username + password (for local mongo)
|
|
username: root
|
|
password: PASSWORD@
|
|
|
|
|
|
superuser:
|
|
# set this to enable a superuser admin
|
|
email: admin@example.com
|
|
|
|
# optional: if not set, automatically generated
|
|
# change or remove this
|
|
password: PASSW0RD!
|
|
|
|
|
|
# additional storages
|
|
minio_local_bucket_name: &local_bucket_name "btrix-test-data"
|
|
|
|
|
|
storages:
|
|
- name: "replica-0"
|
|
access_key: "ADMIN"
|
|
secret_key: "PASSW0RD"
|
|
|
|
endpoint_url: "http://local-minio.default:9000/btrix-test-data/replica-0/"
|
|
|
|
- name: "replica-1"
|
|
access_key: "ADMIN"
|
|
secret_key: "PASSW0RD"
|
|
|
|
endpoint_url: "http://local-minio.default:9000/btrix-test-data/replica-1/"
|
|
|
|
- name: "default"
|
|
access_key: "ADMIN"
|
|
secret_key: "PASSW0RD"
|
|
bucket_name: *local_bucket_name
|
|
|
|
endpoint_url: "http://local-minio.default:9000/"
|
|
is_default_primary: True
|
|
|
|
- name: "alt-storage"
|
|
access_key: "ADMIN"
|
|
secret_key: "PASSW0RD"
|
|
|
|
endpoint_url: "http://local-minio.default:9000/alt-storage/"
|
|
|
|
|
|
local_service_port: 30870
|
|
|
|
# test max pages per crawl global limit
|
|
max_pages_per_crawl: 4
|
|
|
|
registration_enabled: "0"
|
|
|
|
# log failed crawl pods to operator backend
|
|
log_failed_crawl_lines: 200
|
|
|
|
# disable for tests
|
|
disk_utilization_threshold: 0
|