fb3d88291f
Fixes #1252 Supports a generic background job system, with two background jobs, CreateReplicaJob and DeleteReplicaJob. - CreateReplicaJob runs on new crawls, uploads, profiles and updates the `replicas` array with the info about the replica after the job succeeds. - DeleteReplicaJob deletes the replica. - Both jobs are created from the new `replica_job.yaml` template. The CreateReplicaJob sets secrets for primary storage + replica storage, while DeleteReplicaJob only needs the replica storage. - The job is processed in the operator when the job is finalized (deleted), which should happen immediately when the job is done, either because it succeeds or because the backoffLimit is reached (currently set to 3). - /jobs/ api lists all jobs using a paginated response, including filtering and sorting - /jobs/<job id> returns details for a particular job - tests: nightly tests updated to check create + delete replica jobs for crawls as well as uploads, job api endpoints - tests: also fixes to timeouts in nightly tests to avoid crawls finishing too quickly. --------- Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>
101 lines
2.6 KiB
Python
101 lines
2.6 KiB
Python
"""background jobs tests, named to run after everything else has finished"""
|
|
|
|
import requests
|
|
|
|
import pytest
|
|
|
|
from .conftest import API_PREFIX
|
|
|
|
|
|
job_id = None
|
|
|
|
|
|
def test_background_jobs_list(admin_auth_headers, default_org_id):
|
|
r = requests.get(
|
|
f"{API_PREFIX}/orgs/{default_org_id}/jobs/", headers=admin_auth_headers
|
|
)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
items = data["items"]
|
|
|
|
assert items
|
|
assert len(items) == data["total"]
|
|
|
|
for item in items:
|
|
assert item["id"]
|
|
assert item["type"]
|
|
assert item["oid"]
|
|
assert item["success"] in (True, False, None)
|
|
assert item["started"]
|
|
finished = item["finished"]
|
|
assert finished or finished is None
|
|
|
|
global job_id
|
|
job_id = [item for item in items if item["finished"] and item["success"]][0]["id"]
|
|
assert job_id
|
|
|
|
|
|
@pytest.mark.parametrize("job_type", [("create-replica"), ("delete-replica")])
|
|
def test_background_jobs_list_filter_by_type(
|
|
admin_auth_headers, default_org_id, job_type
|
|
):
|
|
r = requests.get(
|
|
f"{API_PREFIX}/orgs/{default_org_id}/jobs/?jobType={job_type}",
|
|
headers=admin_auth_headers,
|
|
)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
items = data["items"]
|
|
|
|
assert items
|
|
assert len(items) == data["total"]
|
|
|
|
for item in items:
|
|
assert item["type"] == job_type
|
|
|
|
|
|
def test_background_jobs_list_filter_by_success(admin_auth_headers, default_org_id):
|
|
r = requests.get(
|
|
f"{API_PREFIX}/orgs/{default_org_id}/jobs/?success=True",
|
|
headers=admin_auth_headers,
|
|
)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
items = data["items"]
|
|
|
|
assert items
|
|
assert len(items) == data["total"]
|
|
|
|
for item in items:
|
|
assert item["success"]
|
|
|
|
|
|
def test_background_jobs_no_failures(admin_auth_headers, default_org_id):
|
|
r = requests.get(
|
|
f"{API_PREFIX}/orgs/{default_org_id}/jobs/?success=False",
|
|
headers=admin_auth_headers,
|
|
)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
assert data["items"] == []
|
|
assert data["total"] == 0
|
|
|
|
|
|
def test_get_background_job(admin_auth_headers, default_org_id):
|
|
r = requests.get(
|
|
f"{API_PREFIX}/orgs/{default_org_id}/jobs/{job_id}", headers=admin_auth_headers
|
|
)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
|
|
assert data["id"]
|
|
assert data["type"] in ("create-replica", "delete-replica")
|
|
assert data["oid"] == default_org_id
|
|
assert data["success"]
|
|
assert data["started"]
|
|
assert data["finished"]
|
|
assert data["file_path"]
|
|
assert data["object_type"]
|
|
assert data["object_id"]
|
|
assert data["replica_storage"]
|