0c8a5a49b4
- use python-on-whale to use docker cli api directly, creating docker stack for each crawl or profile browser - configure storages via storages.yaml secret - add crawl_job, profile_job, splitting into base and k8s/swarm implementations - split manager into base crawlmanager and k8s/swarm implementations - swarm: load initial scale from db to avoid modifying fixed configs, in k8s, load from configmap - swarm: support scheduled jobs via swarm-cronjob service - remove docker dependencies (aiodocker, apscheduler, scheduling) - swarm: when using local minio, expose via /data/ route in nginx via extra include (in k8s, include dir is empty and routing handled via ingress) - k8s: cleanup minio chart: move init containers to minio.yaml - swarm: stateful set implementation to be consistent with k8s scaling: - don't use service replicas, - create a unique service with '-N' appended and allocate unique volume for each replica - allows crawl containers to be restarted w/o losing data - add volume pruning background service, as volumes can be deleted only after service shuts down fully - watch: fully simplify routing, route via replica index instead of ip for both k8s and swarm - rename network btrix-cloud-net -> btrix-net to avoid conflict with compose network
106 lines
2.0 KiB
YAML
106 lines
2.0 KiB
YAML
{{- if .Values.minio_local }}
|
|
|
|
---
|
|
kind: PersistentVolumeClaim
|
|
apiVersion: v1
|
|
metadata:
|
|
name: minio-storage-pvc
|
|
annotations:
|
|
"helm.sh/resource-policy": keep
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|
|
|
|
{{- if .Values.volume_storage_class }}
|
|
storageClassName: {{ .Values.volume_storage_class }}
|
|
{{- end }}
|
|
|
|
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: local-minio
|
|
namespace: {{ .Release.Namespace }}
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: local-minio
|
|
|
|
replicas: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: local-minio
|
|
|
|
spec:
|
|
{{- if .Values.main_node_type }}
|
|
nodeSelector:
|
|
nodeType: {{ .Values.main_node_type }}
|
|
{{- end }}
|
|
|
|
volumes:
|
|
- name: data-minio
|
|
persistentVolumeClaim:
|
|
claimName: minio-storage-pvc
|
|
|
|
initContainers:
|
|
- name: init-bucket
|
|
image: {{ .Values.minio_mc_image }}
|
|
imagePullPolicy: {{ .Values.minio_pull_policy }}
|
|
env:
|
|
- name: MC_HOST_local
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: auth-secrets
|
|
key: MC_HOST
|
|
|
|
command: ['/bin/sh']
|
|
args: ['-c', 'mc mb --ignore-existing local/{{ .Values.minio_local_bucket_name }}' ]
|
|
|
|
containers:
|
|
- name: minio
|
|
image: {{ .Values.minio_image }}
|
|
imagePullPolicy: {{ .Values.minio_pull_policy }}
|
|
args: ["server", "/data", "--console-address", ":9001"]
|
|
envFrom:
|
|
- secretRef:
|
|
name: auth-secrets
|
|
|
|
volumeMounts:
|
|
- name: data-minio
|
|
mountPath: /data
|
|
subPath: minio
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
|
|
metadata:
|
|
namespace: {{ .Release.Namespace }}
|
|
name: local-minio
|
|
labels:
|
|
app: local-minio
|
|
|
|
spec:
|
|
type: NodePort
|
|
selector:
|
|
app: local-minio
|
|
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9000
|
|
name: minio
|
|
|
|
- protocol: TCP
|
|
port: 9001
|
|
name: minio-console
|
|
|
|
{{- end }}
|
|
|
|
|