node affinity: set to required instead of preferred to keep crawlers on dedicated infrastructure (#1366)

Previously, the crawler pods use preferred node affinity, instead of
required node affinity. This results in crawler nodes running on the
main node pool. Instead, we want to ensure crawler nodes are running on
dedicated node pool (if configured).
- Converts 'preferred node affinity' to 'required node affinity' for
the node pool, while keeping preferred pod affinity for keeping all
crawler / redis pods together.
- For profiles, updates to same node affinity, and also adds
resource constraint to match a single crawler for profile browser,
which did not have resource constraints.

chart/app-templates/crawler.yaml

@@ -60,24 +60,28 @@ spec:
         claimName: {{ name }}
 
   affinity:
+{% if crawler_node_type %}
     nodeAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
+      requiredDuringSchedulingIgnoredDuringExecution:
-        - weight: 1
+        nodeSelectorTerms:
-          preference:
+          - matchExpressions:
-            matchExpressions:
             - key: nodeType
               operator: In
               values:
                 - "{{ crawler_node_type }}"
+{% endif %}
 
     podAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
-        - weight: 2
+        - weight: 10
           podAffinityTerm:
-            topologyKey: "failure-domain.beta.kubernetes.io/zone"
+            topologyKey: "kubernetes.io/hostname"
             labelSelector:
-              matchLabels:
+              matchExpressions:
-                crawl: {{ id }}
+              - key: crawl
+                operator: In
+                values:
+                - {{ id }}
 
   tolerations:
     - key: nodeType

chart/app-templates/profilebrowser.yaml

@@ -18,16 +18,17 @@ spec:
 
   restartPolicy: OnFailure
 
+{% if crawler_node_type %}
   affinity:
     nodeAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
+      requiredDuringSchedulingIgnoredDuringExecution:
-        - weight: 1
+        nodeSelectorTerms:
-          preference:
+          - matchExpressions:
-            matchExpressions:
             - key: nodeType
               operator: In
               values:
                 - "{{ crawler_node_type }}"
+{% endif %}
 
   tolerations:
     - key: nodeType
@@ -74,3 +75,11 @@ spec:
         - name: CHROME_FLAGS
           value: "--proxy-server=socks5://{{ crawler_socks_proxy_host }}:{{ crawler_socks_proxy_port | default('9050') }}"
       {% endif %}
+
+      resources:
+        limits:
+          memory: "{{ crawler_memory }}"
+
+        requests:
+          cpu: "{{ crawler_cpu }}"
+          memory: "{{ crawler_memory }}"