b9d7907ab3
* simplify back to single config.env! - back to good ole env vars! - remove shared secret, which made it difficult to have scheduled crawls, since secrets are immutable, so could not update config if a scheduled crawl existed :/ - all env vars unified in configs/config.env - run-swarm.sh and run-pod.sh 'source' this config - remove config.sample.yaml - customize minio volume dir via config.env - customize redis port via config.env - include authsign ports in debug-ports config
105 lines
2.2 KiB
YAML
105 lines
2.2 KiB
YAML
version: '3.9'
|
|
|
|
services:
|
|
crawler:
|
|
image: {{ env.CRAWLER_IMAGE }}
|
|
command:
|
|
- crawl
|
|
- --config
|
|
- /var/run/secrets/crawl-config-{{ cid }}
|
|
- --redisStoreUrl
|
|
- {{ redis_url }}
|
|
{%- if env.PROFILE_FILENAME %}
|
|
- --profile
|
|
- "@profiles/{{ env.PROFILE_FILENAME }}"
|
|
{%- endif %}
|
|
|
|
hostname: "crawl-{{ id }}-{{ index }}_crawler"
|
|
container_name: "crawl-{{ id }}-{{ index }}_crawler"
|
|
|
|
networks:
|
|
- btrix
|
|
|
|
secrets:
|
|
- crawl-config-{{ cid }}
|
|
|
|
volumes:
|
|
- crawl-data:/crawls
|
|
|
|
stop_grace_period: 1000s
|
|
|
|
restart: always
|
|
|
|
deploy:
|
|
endpoint_mode: dnsrr
|
|
replicas: 1
|
|
labels:
|
|
crawl: {{ id }}
|
|
role: crawler
|
|
|
|
{% if env.CRAWLER_LIMITS_CPU and env.CRAWLER_LIMITS_MEMORY and
|
|
env.CRAWLER_REQUESTS_CPU and env.CRAWLER_REQUESTS_MEMORY %}
|
|
resources:
|
|
limits:
|
|
cpus: "{{ env.CRAWLER_LIMITS_CPU }}"
|
|
memory: "{{ env.CRAWLER_LIMITS_MEMORY }}"
|
|
reservations:
|
|
cpus: "{{ env.CRAWLER_REQUESTS_CPU }}"
|
|
memory: "{{ env.CRAWLER_REQUESTS_MEMORY }}"
|
|
{% endif %}
|
|
|
|
environment:
|
|
CRAWL_ID: {{ id }}
|
|
|
|
STORE_ENDPOINT_URL: "{{ env.STORE_ENDPOINT_URL }}"
|
|
STORE_ACCESS_KEY: "{{ env.STORE_ACCESS_KEY }}"
|
|
STORE_SECRET_KEY: "{{ env.STORE_SECRET_KEY }}"
|
|
|
|
STORE_PATH: "{{ env.STORE_PATH }}"
|
|
STORE_FILENAME: "{{ env.STORE_FILENAME }}"
|
|
STORE_USER: "{{ env.USER_ID }}"
|
|
|
|
{%- if env.WACZ_SIGN_URL %}
|
|
WACZ_SIGN_TOKEN: "{{ env.WACZ_SIGN_TOKEN }}"
|
|
WACZ_SIGN_URL: "{{ env.WACZ_SIGN_URL }}"
|
|
{%- endif %}
|
|
|
|
WEBHOOK_URL: "{{ redis_url }}/crawls-done"
|
|
CRAWL_ARGS: "{{ env.CRAWL_ARGS }}"
|
|
|
|
{% if index == 0 %}
|
|
redis:
|
|
image: {{ env.REDIS_IMAGE }}
|
|
command: ["redis-server", "--appendonly", "yes"]
|
|
container_name: "crawl-{{ id }}-{{ index }}_redis"
|
|
restart: always
|
|
|
|
deploy:
|
|
endpoint_mode: dnsrr
|
|
replicas: 1
|
|
labels:
|
|
crawl: {{ id }}
|
|
role: redis
|
|
|
|
networks:
|
|
- btrix
|
|
|
|
{% endif %}
|
|
|
|
networks:
|
|
btrix:
|
|
external:
|
|
name: btrix-net
|
|
|
|
secrets:
|
|
crawl-config-{{ cid }}:
|
|
external: true
|
|
|
|
volumes:
|
|
crawl-data:
|
|
name: "crawl-{{ id }}-{{ index }}"
|
|
labels:
|
|
btrix.crawl: {{ id }}
|
|
|
|
|