5b6aa3bc95
* k8s: add tolerations for 'nodeType=crawling:NoSchedule' to allow scheduling crawling on designated nodes for crawler and profiles jobs and statefulsets * add affinity for 'nodeType=crawling' on crawling and profile browser statefulsets * refactor crawljob: combine crawl_updater logic into base crawl_job * increment new 'crawlAttemptCount' counter crawlconfig when crawl is started, not necessarily finished, to avoid deleting configs that had attempted but not finished crawls. * better external mongodb support: use MONGO_DB_URL to set custom url directly, otherwise build from username, password and mongo host
136 lines
2.8 KiB
YAML
136 lines
2.8 KiB
YAML
|
|
# mongo secrets used with backend and with local mongo, if any
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: mongo-auth
|
|
namespace: {{ .Values.crawler_namespace }}
|
|
|
|
type: Opaque
|
|
stringData:
|
|
MONGO_INITDB_ROOT_USERNAME: {{ .Values.mongo_auth.username | quote }}
|
|
MONGO_INITDB_ROOT_PASSWORD: {{ .Values.mongo_auth.password | quote }}
|
|
MONGO_HOST: "{{ .Values.mongo_host }}"
|
|
MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}"
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: mongo-auth
|
|
namespace: {{ .Release.Namespace }}
|
|
|
|
type: Opaque
|
|
stringData:
|
|
MONGO_INITDB_ROOT_USERNAME: {{ .Values.mongo_auth.username | quote }}
|
|
MONGO_INITDB_ROOT_PASSWORD: {{ .Values.mongo_auth.password | quote }}
|
|
MONGO_HOST: "{{ .Values.mongo_host }}"
|
|
MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}"
|
|
|
|
|
|
{{- if .Values.mongo_local }}
|
|
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: local-mongo
|
|
namespace: {{ .Release.Namespace }}
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: local-mongo
|
|
serviceName: local-mongo
|
|
replicas: {{ .Values.api_num_replicas }}
|
|
podManagementPolicy: Parallel
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data-db
|
|
annotations:
|
|
helm.sh/resource-policy: keep
|
|
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
|
|
resources:
|
|
requests:
|
|
storage: 2Gi
|
|
|
|
{{- if .Values.volume_storage_class }}
|
|
storageClassName: {{ .Values.volume_storage_class }}
|
|
{{- end }}
|
|
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: local-mongo
|
|
|
|
spec:
|
|
{{- if .Values.main_node_type }}
|
|
nodeSelector:
|
|
nodeType: {{ .Values.main_node_type }}
|
|
{{- end }}
|
|
|
|
containers:
|
|
- name: mongo
|
|
image: {{ .Values.mongo_image }}
|
|
|
|
{{- if .Values.mongo_run_repair }}
|
|
command: ["mongod", "--repair"]
|
|
{{- end }}
|
|
|
|
imagePullPolicy: {{ .Values.mongo_pull_policy }}
|
|
envFrom:
|
|
- secretRef:
|
|
name: mongo-auth
|
|
|
|
volumeMounts:
|
|
- name: data-db
|
|
mountPath: /data/db
|
|
subPath: mongo
|
|
|
|
resources:
|
|
limits:
|
|
cpu: {{ .Values.mongo_limits_cpu }}
|
|
memory: {{ .Values.mongo_limits_memory }}
|
|
|
|
requests:
|
|
cpu: {{ .Values.mongo_requests_cpu }}
|
|
memory: {{ .Values.mongo_requests_memory }}
|
|
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- mongo
|
|
- --eval
|
|
- db.adminCommand('ping')
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
|
|
metadata:
|
|
namespace: {{ .Release.Namespace }}
|
|
name: local-mongo
|
|
labels:
|
|
app: local-mongo
|
|
|
|
spec:
|
|
#type: ClusterIP
|
|
clusterIP: None
|
|
selector:
|
|
app: local-mongo
|
|
|
|
ports:
|
|
- protocol: TCP
|
|
port: 27017
|
|
targetPort: 27017
|
|
name: mongo
|
|
|
|
{{- end }}
|
|
|
|
|