4bea7565bc
Operator: Modified init behavior to only load redis when at least one crawler pod available: - waits for at least one crawler pod to be available before starting redis pod, to avoid situation where many crawler pods are in pending mode, but redis pods are still running. - redis statefulset starts at scale of 0 - once crawler pod becomes available, redis sts is scaled to 1 (via `initRedis==true` status) - crawl remains in 'starting' or 'waiting_capacity' state until pod becomes available without redis pod running - set to 'running' state only after redis and at least one crawler pod is available - if no crawler pods available after running, or, if stuck in starting for >60 seconds, switch to 'waiting_capacity' state - when switching to 'waiting_capacity', also scale down redis to 0, wait for crawler pod to become available, only then scale up redis to 1, and get back to 'running' other tweaks: - add new status field 'initRedis', default to false, not displayed - crawler pod: consider 'ContainerCreating' state as available, as container will not be blocked by resource limits - add a resync after 3 seconds when waiting for crawler pod or redis pod to become available, configurable via 'operator_fast_resync_secs' - set_state: if not updating state, ensure state reflects actual value in db
150 lines
4.7 KiB
YAML
150 lines
4.7 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ .Values.name }}-env-config
|
|
namespace: {{ .Release.Namespace }}
|
|
|
|
data:
|
|
APP_ORIGIN: {{.Values.ingress.scheme }}://{{ .Values.ingress.host | default "localhost:9870" }}
|
|
|
|
CRON_NAMESPACE: {{ .Release.Namespace }}
|
|
|
|
CRAWLER_NAMESPACE: {{ .Values.crawler_namespace }}
|
|
CRAWLER_IMAGE: {{ .Values.crawler_image }}
|
|
CRAWLER_PULL_POLICY: {{ .Values.crawler_pull_policy }}
|
|
|
|
CRAWLER_FQDN_SUFFIX: ".{{ .Values.crawler_namespace }}.svc.cluster.local"
|
|
|
|
CRAWLER_TIMEOUT: "{{ .Values.crawl_timeout }}"
|
|
CRAWLER_RETRIES: "{{ .Values.crawl_retries }}"
|
|
|
|
CRAWLER_REQUESTS_CPU: "{{ .Values.crawler_requests_cpu }}"
|
|
CRAWLER_LIMITS_CPU: "{{ .Values.crawler_limits_cpu }}"
|
|
|
|
CRAWLER_REQUESTS_MEM: "{{ .Values.crawler_requests_memory }}"
|
|
CRAWLER_LIMITS_MEM: "{{ .Values.crawler_limits_memory }}"
|
|
|
|
CRAWLER_LIVENESS_PORT: "{{ .Values.crawler_liveness_port | default 0 }}"
|
|
|
|
DEFAULT_ORG: "{{ .Values.default_org }}"
|
|
|
|
INVITE_EXPIRE_SECONDS: "{{ .Values.invite_expire_seconds }}"
|
|
|
|
JOB_IMAGE: "{{ .Values.backend_image }}"
|
|
JOB_PULL_POLICY: "{{ .Values.backend_pull_policy }}"
|
|
|
|
{{- if .Values.crawler_pv_claim }}
|
|
CRAWLER_PV_CLAIM: "{{ .Values.crawler_pv_claim }}"
|
|
{{- end }}
|
|
|
|
REDIS_URL: "{{ .Values.redis_url }}"
|
|
|
|
REDIS_CRAWLS_DONE_KEY: "crawls-done"
|
|
|
|
GRACE_PERIOD_SECS: "{{ .Values.grace_period_secs | default 600 }}"
|
|
|
|
REGISTRATION_ENABLED: "{{ .Values.registration_enabled | default 0 }}"
|
|
|
|
ALLOW_DUPE_INVITES: "{{ .Values.allow_dupe_invites | default 0 }}"
|
|
|
|
JWT_TOKEN_LIFETIME_MINUTES: "{{ .Values.jwt_token_lifetime_minutes | default 60 }}"
|
|
|
|
DEFAULT_BEHAVIOR_TIME_SECONDS: "{{ .Values.default_behavior_time_seconds }}"
|
|
|
|
DEFAULT_PAGE_LOAD_TIME_SECONDS: "{{ .Values.default_page_load_time_seconds }}"
|
|
|
|
DEFAULT_CRAWL_FILENAME_TEMPLATE: "{{ .Values.default_crawl_filename_template }}"
|
|
|
|
MAX_PAGES_PER_CRAWL: "{{ .Values.max_pages_per_crawl | default 0 }}"
|
|
|
|
IDLE_TIMEOUT: "{{ .Values.profile_browser_idle_seconds | default 60 }}"
|
|
|
|
RERUN_LAST_MIGRATION: "{{ .Values.rerun_last_migration }}"
|
|
|
|
PRESIGN_DURATION_MINUTES: "{{ .Values.storage_presign_duration_minutes | default 60 }}"
|
|
|
|
FAST_RETRY_SECS: "{{ .Values.operator_fast_resync_secs | default 3 }}"
|
|
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: shared-crawler-config
|
|
namespace: {{ .Values.crawler_namespace }}
|
|
|
|
data:
|
|
CRAWL_ARGS: >-
|
|
--workers {{ .Values.crawler_browser_instances | default 1 }} --sizeLimit {{ .Values.crawler_session_size_limit_bytes }} --timeLimit {{ .Values.crawler_session_time_limit_seconds }} --maxPageLimit {{ .Values.max_pages_per_crawl | default 0 }} --healthCheckPort {{ .Values.crawler_liveness_port }} --diskUtilization {{ .Values.disk_utilization_threshold }} --userAgentSuffix {{ .Values.user_agent_suffix | quote }} --userAgent {{ .Values.user_agent | quote }} --logging {{ .Values.crawler_logging_opts }} --text {{ .Values.crawler_extract_full_text }} --generateWACZ --waitOnDone --collection thecrawl --screencastPort 9037 --logErrorsToRedis {{ .Values.crawler_extra_args }}
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: shared-job-config
|
|
#namespace: {{ .Values.crawler_namespace }}
|
|
namespace: {{ .Release.Namespace }}
|
|
|
|
|
|
data:
|
|
config.yaml: |
|
|
namespace: {{ .Values.crawler_namespace }}
|
|
termination_grace_secs: "{{ .Values.grace_period_secs | default 600 }}"
|
|
|
|
volume_storage_class: "{{ .Values.volume_storage_class }}"
|
|
|
|
requests_hd: "{{ .Values.crawler_requests_storage }}"
|
|
|
|
# redis
|
|
redis_image: {{ .Values.redis_image }}
|
|
redis_image_pull_policy: {{ .Values.redis_pull_policy }}
|
|
|
|
redis_requests_cpu: "{{ .Values.redis_requests_cpu }}"
|
|
redis_limits_cpu: "{{ .Values.redis_limits_cpu }}"
|
|
|
|
redis_requests_memory: "{{ .Values.redis_requests_memory }}"
|
|
redis_limits_memory: "{{ .Values.redis_limits_memory }}"
|
|
|
|
|
|
# crawler
|
|
crawler_image: {{ .Values.crawler_image }}
|
|
crawler_image_pull_policy: {{ .Values.crawler_pull_policy }}
|
|
|
|
crawler_requests_cpu: "{{ .Values.crawler_requests_cpu }}"
|
|
crawler_limits_cpu: "{{ .Values.crawler_limits_cpu }}"
|
|
|
|
crawler_requests_memory: "{{ .Values.crawler_requests_memory }}"
|
|
crawler_limits_memory: "{{ .Values.crawler_limits_memory }}"
|
|
|
|
crawler_liveness_port: "{{ .Values.crawler_liveness_port | default 0 }}"
|
|
|
|
crawler_node_type: "{{ .Values.crawler_node_type }}"
|
|
redis_node_type: "{{ .Values.redis_node_type }}"
|
|
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: shared-redis-conf
|
|
namespace: {{ .Values.crawler_namespace }}
|
|
|
|
data:
|
|
redis.conf: |
|
|
appendonly yes
|
|
dir /data
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: nginx-config
|
|
namespace: {{ .Release.Namespace }}
|
|
|
|
data:
|
|
{{ (.Files.Glob "*.conf").AsConfig | indent 2 }}
|
|
|
|
#{{ (.Files.Glob "frontend/*.*").AsConfig | indent 2 }}
|
|
|