00fb8ac048
concurrent crawl limits: (addresses #866) - support limits on concurrent crawls that can be run within a single org - change 'waiting' state to 'waiting_org_limit' for concurrent crawl limit and 'waiting_capacity' for capacity-based limits orgs: - add 'maxConcurrentCrawl' to new 'quotas' object on orgs - add /quotas endpoint for updating quotas object operator: - add all crawljobs as related, appear to be returned in creation order - operator: if concurrent crawl limit set, ensures current job is in the first N set of crawljobs (as provided via 'related' list of crawljob objects) before it can proceed to 'starting', otherwise set to 'waiting_org_limit' - api: add org /quotas endpoint for configuring quotas - remove 'new' state, always start with 'starting' - crawljob: add 'oid' to crawljob spec and label for easier querying - more stringent state transitions: add allowed_from to set_state() - ensure state transitions only happened from allowed states, while failed/canceled can happen from any state - ensure finished and state synched from db if transition not allowed - add crawl indices by oid and cid frontend: - show different waiting states on frontend: 'Waiting (Crawl Limit) and 'Waiting (At Capacity)' - add gear icon on orgs admin page - and initial popup for setting org quotas, showing all properties from org 'quotas' object tests: - add concurrent crawl limit nightly tests - fix state waiting -> waiting_capacity - ci: add logging of operator output on test failure
94 lines
2.5 KiB
YAML
94 lines
2.5 KiB
YAML
name: Cluster Run (K3d)
|
|
|
|
on:
|
|
push:
|
|
paths:
|
|
- 'backend/**'
|
|
- 'chart/**'
|
|
|
|
pull_request:
|
|
paths:
|
|
- 'backend/**'
|
|
- 'chart/**'
|
|
|
|
jobs:
|
|
btrix-k3d-test:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Create k3d Cluster
|
|
uses: AbsaOSS/k3d-action@v2
|
|
with:
|
|
cluster-name: btrix-1
|
|
args: >-
|
|
-p "30870:30870@agent:0:direct"
|
|
--agents 1
|
|
--no-lb
|
|
--k3s-arg "--no-deploy=traefik,servicelb,metrics-server@server:*"
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v2
|
|
with:
|
|
driver-opts: network=host
|
|
|
|
- name: Build Backend
|
|
uses: docker/build-push-action@v3
|
|
with:
|
|
context: backend
|
|
load: true
|
|
#outputs: type=tar,dest=backend.tar
|
|
tags: webrecorder/browsertrix-backend:latest
|
|
cache-from: type=gha,scope=backend
|
|
cache-to: type=gha,scope=backend,mode=max
|
|
|
|
- name: Build Frontend
|
|
uses: docker/build-push-action@v3
|
|
with:
|
|
context: frontend
|
|
load: true
|
|
#outputs: type=tar,dest=frontend.tar
|
|
tags: webrecorder/browsertrix-frontend:latest
|
|
cache-from: type=gha,scope=frontend
|
|
cache-to: type=gha,scope=frontend,mode=max
|
|
|
|
- name: 'Import Images'
|
|
run: |
|
|
k3d image import webrecorder/browsertrix-backend:latest -m direct -c btrix-1 --verbose
|
|
k3d image import webrecorder/browsertrix-frontend:latest -m direct -c btrix-1 --verbose
|
|
|
|
- name: Install Kubectl
|
|
uses: azure/setup-kubectl@v3
|
|
|
|
- name: Install Helm
|
|
uses: azure/setup-helm@v3
|
|
with:
|
|
version: 3.10.2
|
|
|
|
- name: Start Cluster with Helm
|
|
run: |
|
|
helm upgrade --install -f ./chart/values.yaml -f ./chart/test/test.yaml btrix ./chart/
|
|
|
|
- name: Install Python
|
|
uses: actions/setup-python@v3
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Install Python Libs
|
|
run: pip install pytest requests
|
|
|
|
- name: Wait for all pods to be ready
|
|
run: kubectl wait --for=condition=ready pod --all --timeout=240s
|
|
|
|
- name: Run Tests
|
|
run: pytest -vv ./backend/test/*.py
|
|
|
|
- name: Print Backend Logs (API)
|
|
if: ${{ failure() }}
|
|
run: kubectl logs svc/browsertrix-cloud-backend -c api
|
|
|
|
- name: Print Backend Logs (Operator)
|
|
if: ${{ failure() }}
|
|
run: kubectl logs svc/browsertrix-cloud-backend -c op
|