0c8a5a49b4
- use python-on-whale to use docker cli api directly, creating docker stack for each crawl or profile browser - configure storages via storages.yaml secret - add crawl_job, profile_job, splitting into base and k8s/swarm implementations - split manager into base crawlmanager and k8s/swarm implementations - swarm: load initial scale from db to avoid modifying fixed configs, in k8s, load from configmap - swarm: support scheduled jobs via swarm-cronjob service - remove docker dependencies (aiodocker, apscheduler, scheduling) - swarm: when using local minio, expose via /data/ route in nginx via extra include (in k8s, include dir is empty and routing handled via ingress) - k8s: cleanup minio chart: move init containers to minio.yaml - swarm: stateful set implementation to be consistent with k8s scaling: - don't use service replicas, - create a unique service with '-N' appended and allocate unique volume for each replica - allows crawl containers to be restarted w/o losing data - add volume pruning background service, as volumes can be deleted only after service shuts down fully - watch: fully simplify routing, route via replica index instead of ip for both k8s and swarm - rename network btrix-cloud-net -> btrix-net to avoid conflict with compose network
98 lines
2.9 KiB
Plaintext
98 lines
2.9 KiB
Plaintext
include ./resolvers/resolvers.conf;
|
|
|
|
server {
|
|
listen 80 default_server;
|
|
server_name _;
|
|
proxy_buffering off;
|
|
proxy_buffers 16 64k;
|
|
proxy_buffer_size 64k;
|
|
root /usr/share/nginx/html;
|
|
index index.html index.htm;
|
|
|
|
error_page 500 501 502 503 504 /50x.html;
|
|
|
|
merge_slashes off;
|
|
location = /50x.html {
|
|
root /usr/share/nginx/html;
|
|
}
|
|
|
|
# fallback to index for any page
|
|
error_page 404 /index.html;
|
|
|
|
location / {
|
|
root /usr/share/nginx/html;
|
|
index index.html index.htm;
|
|
}
|
|
|
|
# used in both k8s and docker: RWP_BASE_URL set in Dockerfile
|
|
location /replay/sw.js {
|
|
add_header Content-Type application/javascript;
|
|
return 200 'importScripts("${RWP_BASE_URL}sw.js");';
|
|
}
|
|
|
|
# used by docker only: k8s deployment handles /api directly via ingress
|
|
location /api/ {
|
|
proxy_pass http://${BACKEND_HOST}:8000;
|
|
proxy_set_header Host $http_host;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
}
|
|
|
|
location ~* /watch/([^/]+)/([^/]+)/([^/]+)/ws {
|
|
set $archive $1;
|
|
set $crawl $2;
|
|
set $num $3;
|
|
set $auth_bearer $arg_auth_bearer;
|
|
auth_request /access_check;
|
|
|
|
proxy_pass http://crawl-$crawl-$num${CRAWLER_SVC_SUFFIX}${CRAWLER_FQDN_SUFFIX}:9037/ws;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
|
|
location = /access_check {
|
|
internal;
|
|
proxy_pass http://${BACKEND_HOST}:8000/api/archives/$archive/crawls/$crawl/access?auth_bearer=$auth_bearer;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
}
|
|
|
|
location ~* /loadbrowser/([^/]+)/(.*)/page/(.*)$ {
|
|
set $browserid $1;
|
|
set $auth_bearer $arg_auth_bearer;
|
|
set $archive $arg_aid;
|
|
auth_request /access_check_profiles;
|
|
|
|
#proxy_pass http://$browser_ip:9222/$2/page/$3;
|
|
proxy_pass http://browser-$browserid-0.browser-$browserid${CRAWLER_FQDN_SUFFIX}:9222/$2/page/$3;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
|
|
location = /access_check_profiles {
|
|
internal;
|
|
proxy_pass http://${BACKEND_HOST}:8000/api/archives/$archive/profiles/browser/$browserid/access?auth_bearer=$auth_bearer;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
}
|
|
|
|
location ~* /loadbrowser/([^/]+)/(.*) {
|
|
set $browserid $1;
|
|
#proxy_pass http://$browser_ip:9222/$2;
|
|
proxy_pass http://browser-$browserid-0.browser-$browserid${CRAWLER_FQDN_SUFFIX}:9222/$2;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
|
|
include ./includes/*.conf;
|
|
}
|
|
|