Crawler network policy (#1727)

Limit egress traffic from crawler/profilebrowser pods to the internet
and limited internal services like dns, redis, frontend, auth-signer on certain ports

---------
Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>

chart/app-templates/crawler.yaml

@@ -38,6 +38,7 @@ metadata:
   labels:
     crawl: {{ id }}
     role: crawler
+    network-policy: limit-crawler-egress
 
 spec:
   hostname: {{ name }}

chart/app-templates/profilebrowser.yaml

@@ -7,6 +7,7 @@ metadata:
   labels:
     browser: {{ id }}
     role: browser
+    network-policy: limit-crawler-egress
 
 spec:
   hostname: browser-{{ id }}

chart/templates/networkpolicies.yaml

@@ -0,0 +1,101 @@
+{{- if .Values.crawler_enable_network_policy -}}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: crawler-limit-egress
+  namespace: {{ .Values.crawler_namespace }}
+spec:
+  podSelector:
+    matchLabels:
+      network-policy: limit-crawler-egress
+  policyTypes:
+    - Egress
+  egress:
+  {{- if .Values.crawler_network_policy_egress | default false -}}
+  {{- .Values.crawler_network_policy_egress | toYaml | nindent 4 -}}
+  {{- else }}
+    # allow WWW
+    - to:
+      - ipBlock:
+          cidr: 0.0.0.0/0
+          except: # Exclude traffic to Kubernetes service IPs and pods
+          - 10.0.0.0/8
+          - 172.16.0.0/12
+          - 192.168.0.0/16
+
+    # allow frontend access for QA runs
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        podSelector:
+          matchLabels:
+            role: frontend
+
+      ports:
+        - port: 80
+          protocol: TCP
+
+      # allow DNS
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: kube-system
+        podSelector:
+          matchLabels:
+            k8s-app: kube-dns
+      ports:
+        - port: 53
+          protocol: UDP
+        - port: 53
+          protocol: TCP
+
+
+    # allow other redis
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: {{ .Values.crawler_namespace }}
+        podSelector:
+          matchLabels:
+            role: redis
+
+      ports:
+        - port: 6379
+          protocol: TCP
+
+
+    {{ if .Values.minio_local }}
+    # allow minio
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        podSelector:
+          matchLabels:
+            app: local-minio
+
+      ports:
+        - port: 9000
+          protocol: TCP
+
+    {{- end -}}
+
+
+    {{ if .Values.signer.enabled }}
+    # allow auth signer
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: {{ .Release.Namespace }}
+        podSelector:
+          matchLabels:
+            app: auth-signer
+
+      ports:
+        - port: {{ .Values.signer.port | default "5053" }}
+          protocol: TCP
+
+    {{- end -}}
+  {{- end -}}
+{{- end -}}

chart/values.yaml

@@ -298,6 +298,12 @@ crawler_liveness_port: 6065
 # crawler_fsgroup: 201400007
 
 
+# optional: enable/disable crawler network policy
+crawler_enable_network_policy: true
+
+# optional: replace the default crawler egress policy with your own
+# see chart/templates/networkpolicies.yaml for an example
+# crawler_network_policy_egress: {}
 
 # time to wait for graceful stop
 grace_period: 1000