a6a78c9ef2
Previously, the crawler pods use preferred node affinity, instead of required node affinity. This results in crawler nodes running on the main node pool. Instead, we want to ensure crawler nodes are running on dedicated node pool (if configured). - Converts 'preferred node affinity' to 'required node affinity' for the node pool, while keeping preferred pod affinity for keeping all crawler / redis pods together. - For profiles, updates to same node affinity, and also adds resource constraint to match a single crawler for profile browser, which did not have resource constraints.
86 lines
1.9 KiB
YAML
86 lines
1.9 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: browser-{{ id }}
|
|
namespace: {{ namespace }}
|
|
labels:
|
|
browser: {{ id }}
|
|
role: browser
|
|
|
|
spec:
|
|
hostname: browser-{{ id }}
|
|
subdomain: browser
|
|
|
|
{% if priorityClassName %}
|
|
priorityClassName: {{ priorityClassName }}
|
|
{% endif %}
|
|
|
|
restartPolicy: OnFailure
|
|
|
|
{% if crawler_node_type %}
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: nodeType
|
|
operator: In
|
|
values:
|
|
- "{{ crawler_node_type }}"
|
|
{% endif %}
|
|
|
|
tolerations:
|
|
- key: nodeType
|
|
operator: Equal
|
|
value: crawling
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/not-ready
|
|
operator: Exists
|
|
tolerationSeconds: 300
|
|
effect: NoExecute
|
|
- key: node.kubernetes.io/unreachable
|
|
operator: Exists
|
|
effect: NoExecute
|
|
tolerationSeconds: 300
|
|
|
|
containers:
|
|
- name: browser
|
|
image: {{ crawler_image }}
|
|
imagePullPolicy: {{ crawler_image_pull_policy }}
|
|
command:
|
|
- create-login-profile
|
|
- --interactive
|
|
- --filename
|
|
- /tmp/profile.tar.gz
|
|
- --url
|
|
- {{ url }}
|
|
{%- if profile_filename %}
|
|
- --profile
|
|
- "@{{ profile_filename }}"
|
|
{%- endif %}
|
|
|
|
envFrom:
|
|
- secretRef:
|
|
name: {{ storage_secret }}
|
|
|
|
env:
|
|
- name: STORE_PATH
|
|
value: {{ storage_path }}
|
|
|
|
- name: VNC_PASS
|
|
value: {{ vnc_password }}
|
|
|
|
{% if crawler_socks_proxy_host %}
|
|
- name: CHROME_FLAGS
|
|
value: "--proxy-server=socks5://{{ crawler_socks_proxy_host }}:{{ crawler_socks_proxy_port | default('9050') }}"
|
|
{% endif %}
|
|
|
|
resources:
|
|
limits:
|
|
memory: "{{ crawler_memory }}"
|
|
|
|
requests:
|
|
cpu: "{{ crawler_cpu }}"
|
|
memory: "{{ crawler_memory }}"
|