* Btrixjobs Operator - Phase 1 (#679) - add metacontroller and custom crds - add main_op entrypoint for operator * Btrix Operator Crawl Management (#767) * operator backend: - run operator api in separate container but in same pod, with WEB_CONCURRENCY=1 - operator creates statefulsets and services for CrawlJob and ProfileJob - operator: use service hook endpoint, set port in values.yaml * crawls working with CrawlJob - jobs start with 'crawljob-' prefix - update status to reflect current crawl state - set sync time to 10 seconds by default, overridable with 'operator_resync_seconds' - mark crawl as running, failed, complete when finished - store finished status when crawl is complete - support updating scale, forcing rollover, stop via patching CrawlJob - support cancel via deletion - requires hack to content-length for patching custom resources - auto-delete of CrawlJob via 'ttlSecondsAfterFinished' - also delete pvcs until autodelete supported via statefulset (k8s >1.27) - ensure filesAdded always set correctly, keep counter in redis, add to status display - optimization: attempt to reduce automerging, by reusing volumeClaimTemplates from existing children, as these may have additional props added - add add_crawl_errors_to_db() for storing crawl errors from redis '<crawl>:e' key to mongodb when crawl is finished/failed/canceled - add .status.size to display human-readable crawl size, if available (from webrecorder/browsertrix-crawler#291) - support new page size, >0.9.0 and old page size key (changed in webrecorder/browsertrix-crawler#284) * support for scheduled jobs! - add main_scheduled_job entrypoint to run scheduled jobs - add crawl_cron_job.yaml template for declaring CronJob - CronJobs moved to default namespace * operator manages ProfileJobs: - jobs start with 'profilejob-' - update expiry time by updating ProfileJob object 'expireTime' while profile is active * refactor/cleanup: - remove k8s package - merge k8sman and basecrawlmanager into crawlmanager - move templates, k8sapi, utils into root package - delete all *_job.py files - remove dt_now, ts_now from crawls, now in utils - all db operations happen in crawl/crawlconfig/org files - move shared crawl/crawlconfig/org functions that use the db to be importable directly, including get_crawl_config, add_new_crawl, inc_crawl_stats * role binding: more secure setup, don't allow crawler namespace any k8s permissions - move cronjobs to be created in default namespace - grant default namespace access to create cronjobs in default namespace - remove role binding from crawler namespace * additional tweaks to templates: - templates: split crawler and redis statefulset into separate yaml file (in case need to load one or other separately) * stats / redis optimization: - don't update stats in mongodb on every operator sync, only when crawl is finished - for api access, read stats directly from redis to get up-to-date stats - move get_page_stats() to utils, add get_redis_url() to k8sapi to unify access * Add migration for operator changes - Update configmap for crawl configs with scale > 1 or crawlTimeout > 0 and schedule exists to recreate CronJobs - add option to rerun last migration, enabled via env var and by running helm with --set=rerun_last_migration=1 * subcharts: move crawljob and profilejob crds to separate subchart, as this seems best way to guarantee proper install order with + update on upgrade with helm, add built btrix-crds-0.1.0.tgz subchart - metacontroller: use release from ghcr, add metacontroller-helm-v4.10.1.tgz subchart * backend api fixes - ensure changing scale of crawl also updates it in the db - crawlconfigs: add 'currCrawlSize' and 'lastCrawlSize' to crawlconfig api --------- Co-authored-by: D. Lee <leepro@gmail.com> Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>
		
			
				
	
	
		
			194 lines
		
	
	
		
			4.7 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
			
		
		
	
	
			194 lines
		
	
	
		
			4.7 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
| ---
 | |
| apiVersion: apps/v1
 | |
| kind: Deployment
 | |
| metadata:
 | |
|   name: {{ .Values.name }}-backend
 | |
|   namespace: {{ .Release.Namespace }}
 | |
| 
 | |
| spec:
 | |
|   selector:
 | |
|     matchLabels:
 | |
|       app: {{ .Values.name }}
 | |
|       role: backend
 | |
|   replicas: {{ .Values.backend_num_replicas }}
 | |
|   template:
 | |
|     metadata:
 | |
|       labels:
 | |
|         app: {{ .Values.name }}
 | |
|         role: backend
 | |
| 
 | |
|       annotations:
 | |
|         # force helm to update the deployment each time
 | |
|         {{- if not .Values.frontend_only }}
 | |
|         "helm.update": {{ randAlphaNum 5 | quote }}
 | |
|         {{- end }}
 | |
| 
 | |
|     spec:
 | |
|       {{- if .Values.main_node_type }}
 | |
|       nodeSelector:
 | |
|         nodeType: {{ .Values.main_node_type }}
 | |
|       {{- end }}
 | |
| 
 | |
|       volumes:
 | |
|         - name: config-volume
 | |
|           configMap:
 | |
|             name: shared-job-config
 | |
|             items:
 | |
|               - key: config.yaml
 | |
|                 path: config.yaml
 | |
| 
 | |
|       containers:
 | |
|         - name: api
 | |
|           image: {{ .Values.backend_image }}
 | |
|           imagePullPolicy: {{ .Values.backend_pull_policy }}
 | |
|           envFrom:
 | |
|             - configMapRef:
 | |
|                 name: {{ .Values.name }}-env-config
 | |
|             - secretRef:
 | |
|                 name: auth-secrets
 | |
|             - secretRef:
 | |
|                 name: mongo-auth
 | |
| 
 | |
|           env:
 | |
|             - name: WEB_CONCURRENCY
 | |
|               value: "{{ .Values.backend_workers | default 4 }}"
 | |
| 
 | |
|           resources:
 | |
|             limits:
 | |
|               cpu: {{ .Values.backend_limits_cpu }}
 | |
|               memory: {{ .Values.backend_limits_memory }}
 | |
| 
 | |
|             requests:
 | |
|               cpu: {{ .Values.backend_requests_cpu }}
 | |
|               memory: {{ .Values.backend_requests_memory }}
 | |
| 
 | |
|           startupProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: 8000
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 5
 | |
|             failureThreshold: 30
 | |
|             successThreshold: 1
 | |
| 
 | |
|           readinessProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: 8000
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 30
 | |
|             failureThreshold: 5
 | |
|             successThreshold: 1
 | |
| 
 | |
|           livenessProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: 8000
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 30
 | |
|             failureThreshold: 5
 | |
|             successThreshold: 1
 | |
| 
 | |
|         - name: op
 | |
|           image: {{ .Values.backend_image }}
 | |
|           imagePullPolicy: {{ .Values.backend_pull_policy }}
 | |
|           command:
 | |
|               - uvicorn
 | |
|               - btrixcloud.main_op:app_root
 | |
|               - --host
 | |
|               - 0.0.0.0
 | |
|               - --port
 | |
|               - "{{ .Values.opPort }}"
 | |
|               - --access-log
 | |
|               - --log-level
 | |
|               - info
 | |
| 
 | |
|           envFrom:
 | |
|             - configMapRef:
 | |
|                 name: {{ .Values.name }}-env-config
 | |
|             - secretRef:
 | |
|                 name: auth-secrets
 | |
|             - secretRef:
 | |
|                 name: mongo-auth
 | |
| 
 | |
|           env:
 | |
|             - name: WEB_CONCURRENCY
 | |
|               value: "{{ .Values.operator_workers | default 1 }}"
 | |
| 
 | |
|           volumeMounts:
 | |
|             - name: config-volume
 | |
|               mountPath: /config
 | |
| 
 | |
|           resources:
 | |
|             limits:
 | |
|               cpu: {{ .Values.backend_limits_cpu }}
 | |
|               memory: {{ .Values.backend_limits_memory }}
 | |
| 
 | |
|             requests:
 | |
|               cpu: {{ .Values.backend_requests_cpu }}
 | |
|               memory: {{ .Values.backend_requests_memory }}
 | |
| 
 | |
|           startupProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: {{ .Values.opPort }}
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 5
 | |
|             failureThreshold: 30
 | |
|             successThreshold: 1
 | |
| 
 | |
|           readinessProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: {{ .Values.opPort }}
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 30
 | |
|             failureThreshold: 5
 | |
|             successThreshold: 1
 | |
| 
 | |
|           livenessProbe:
 | |
|             httpGet:
 | |
|               path: /healthz
 | |
|               port: {{ .Values.opPort }}
 | |
|             initialDelaySeconds: 5
 | |
|             periodSeconds: 30
 | |
|             failureThreshold: 5
 | |
|             successThreshold: 1
 | |
| 
 | |
| 
 | |
| 
 | |
| ---
 | |
| 
 | |
| apiVersion: v1
 | |
| kind: Service
 | |
| 
 | |
| metadata:
 | |
|   namespace: {{ .Release.Namespace }}
 | |
|   name: {{ .Values.name }}-backend
 | |
|   labels:
 | |
|     app: {{ .Values.name }}
 | |
|     role: backend
 | |
| 
 | |
| {{- if .Values.service }}
 | |
|   {{- if .Values.service.annotations }}
 | |
|   annotations:
 | |
|   {{- range $key, $val := .Values.service.annotations }}
 | |
|     {{ $key }}: {{ $val | quote }}
 | |
|   {{- end }}
 | |
|   {{- end }}
 | |
| {{- end }}
 | |
| 
 | |
| spec:
 | |
|   selector:
 | |
|     app: {{ .Values.name }}
 | |
|     role: backend
 | |
| 
 | |
|   ports:
 | |
|     - protocol: TCP
 | |
|       port: 8000
 | |
|       name: api
 | |
| 
 | |
|     - protocol: TCP
 | |
|       port: {{ .Values.opPort }}
 | |
|       name: operator
 |