57a4b6b46f
- collections defined by name per archive
- can update collections with additional metadata (currently just description)
- crawl config api accepts a list of collections by name, resolved to collection uids and stored in config
- finished crawls also associated with collection list
- /archives/{aid}/collections/{name} can list all crawl artifacts (wacz files) from a named collection (in frictionless data package-ish format)
- /archives/{aid}/collections/$all lists all crawled artifacts for the archive
readiness check: add /healthz endpoints for app and nginx
ingress: add /data/ route to local bucket
storage improvements:
- for default storages, store path only, and prepend default storage access endpoint
- collections api returns the paths using the storage access endpoint
- define default storages as secrets in k8s (can support multiple), hard-coded in docker (only one for now)
40 lines
1.0 KiB
YAML
40 lines
1.0 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: auth-secrets
|
|
namespace: {{ .Release.Namespace }}
|
|
|
|
type: Opaque
|
|
stringData:
|
|
PASSWORD_SECRET: "{{ .Values.api_password_secret }}"
|
|
|
|
{{- if .Values.minio_local }}
|
|
{{- with (first .Values.storages) }}
|
|
MINIO_ROOT_USER: "{{ .access_key }}"
|
|
MINIO_ROOT_PASSWORD: "{{ .secret_key }}"
|
|
|
|
MC_HOST: "{{ $.Values.minio_scheme }}://{{ .access_key }}:{{ .secret_key }}@{{ $.Values.minio_host }}"
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
|
|
{{- range $storage := .Values.storages }}
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: storage-{{ $storage.name }}
|
|
namespace: {{ $.Values.crawler_namespace }}
|
|
|
|
type: Opaque
|
|
stringData:
|
|
STORE_ACCESS_KEY: "{{ $storage.access_key }}"
|
|
STORE_SECRET_KEY: "{{ $storage.secret_key }}"
|
|
STORE_ENDPOINT_URL: "{{ $storage.endpoint_url }}{{ $storage.bucket_name }}/"
|
|
{{- if and $.Values.ingress.host $.Values.minio_local }}
|
|
STORE_ACCESS_ENDPOINT_URL: {{ $.Values.ingress.scheme | default "https" }}://{{ $.Values.ingress.host }}/data/{{ $storage.bucket_name }}/
|
|
{{- end }}
|
|
|
|
{{- end }}
|