2f63c7dcf8
* add profile creation, list endpoints at /archives/<aid>/profiles
* add profile browser creation, get, ping, commit, delete endpoints at /archives/<aid>/profiles/browser
* support creation of profile browser using browsertrix-crawler 'create-login-profile' in docker and k8s
* ensure profile browser expires after set time, k8s job or docker container automatically deleted on exit
* profile browser creation returns temporary browser id, or `{"detail": "waiting_for_browser"}` while waiting for browser container init
* nginx frontend: proxy /loadbrowser/ to port 9223 in browsertrix-crawler, connecting directly to chrome devtools
* profile api auth: use redis for auth
- store browserid->archiveid and browserid->browser ip mapping in redis
- browser apis: ensure profile browser is associated with specified archive
- browser ws: pass arcchiveid and browserid to ws query args, browserid is part of archive, and browserid corresponds to specified ip
* store profiles in /profiles/ directory in default storage, include profileid in profile tar.gz filename
* support profile in crawlconfig:
- add profileid to CrawlConfig, and profileName to CrawlConfigOut
- support resolving profile path via profileid, setting '--profile @{path/to/profile.tar.gz}' for crawler (assuming same storage for profile as output for now) in both docker and k8s setups
- docker: support out_filename, custom wacz output filename missing functionality
95 lines
2.6 KiB
Plaintext
95 lines
2.6 KiB
Plaintext
include ./resolvers/resolvers.conf;
|
|
|
|
server {
|
|
listen 80 default_server;
|
|
server_name _;
|
|
proxy_buffering off;
|
|
proxy_buffers 16 64k;
|
|
proxy_buffer_size 64k;
|
|
root /usr/share/nginx/html;
|
|
index index.html index.htm;
|
|
|
|
error_page 500 501 502 503 504 /50x.html;
|
|
|
|
merge_slashes off;
|
|
location = /50x.html {
|
|
root /usr/share/nginx/html;
|
|
}
|
|
|
|
# fallback to index for any page
|
|
error_page 404 /index.html;
|
|
|
|
location / {
|
|
root /usr/share/nginx/html;
|
|
index index.html index.htm;
|
|
}
|
|
|
|
# used in both k8s and docker: RWP_BASE_URL set in Dockerfile
|
|
location /replay/sw.js {
|
|
add_header Content-Type application/javascript;
|
|
return 200 'importScripts("${RWP_BASE_URL}sw.js");';
|
|
}
|
|
|
|
# used by docker only: k8s deployment handles /api directly via ingress
|
|
location /api/ {
|
|
proxy_pass http://${BACKEND_HOST}:8000/;
|
|
proxy_set_header Host $http_host;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
}
|
|
|
|
location ~* /watch/([^/]+)/([^/]+)/([^/]+)/ws {
|
|
set $archive $1;
|
|
set $crawl $2;
|
|
set $crawlerip $3;
|
|
set $auth_bearer $arg_auth_bearer;
|
|
auth_request /ipaccess_crawl;
|
|
|
|
proxy_pass http://$crawlerip:9037/ws;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
|
|
location = /ipaccess_crawl {
|
|
internal;
|
|
proxy_pass http://${BACKEND_HOST}:8000/archives/$archive/crawls/$crawl/ipaccess/$crawlerip?auth_bearer=$auth_bearer;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
}
|
|
|
|
location ~* /loadbrowser/([^/]+)/(.*)/page/(.*)$ {
|
|
set $browser_ip $1;
|
|
set $auth_bearer $arg_auth_bearer;
|
|
set $browserid $arg_browserid;
|
|
set $archive $arg_aid;
|
|
auth_request /ipaccess_profiles;
|
|
|
|
proxy_pass http://$browser_ip:9222/$2/page/$3;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
|
|
location = /ipaccess_profiles {
|
|
internal;
|
|
proxy_pass http://${BACKEND_HOST}:8000/archives/$archive/profiles/browser/$browserid/ipaccess/$browser_ip?auth_bearer=$auth_bearer;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
}
|
|
|
|
location ~* /loadbrowser/([^/]+)/(.*) {
|
|
set $browser_ip $1;
|
|
proxy_pass http://$browser_ip:9222/$2;
|
|
proxy_set_header Host "localhost";
|
|
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Upgrade $http_upgrade;
|
|
proxy_set_header Connection $http_connection;
|
|
}
|
|
}
|
|
|