Backend mem usage fix - use fixed MOTOR_MAX_WORKERS + switch to gunicorn (#1468)

Refactors backend deployment to:
- Use MOTOR_MAX_WORKERS (defaulting to 1) to reduce threads used by
mongodb connections
- Also sets backend workers to 1 by default to reduce default memory
usage
- Switches to gunicorn with uvloop worker for production use instead of
uvicorn (as recommended by uvicorn)

Lower thread count should address memory leak/increased usage, which
resulted in 5x thread x cpus x workers, eg. potentially 20 or 40 threads
just for mongodb connections. Lower default number of workers should
make it easier to scale backend with HPA if additional capacity.

Fixes #1467

backend/Dockerfile

@@ -8,6 +8,4 @@ RUN pip install -r requirements.txt
 
 ADD btrixcloud/ /app/btrixcloud/
 
-CMD uvicorn btrixcloud.main:app_root --host 0.0.0.0 --access-log --log-level info
-
 EXPOSE 8000

backend/requirements.txt

@@ -1,4 +1,5 @@
-uvicorn
+gunicorn
+uvicorn[standard]
 fastapi==0.103.2
 motor==3.3.1
 passlib

chart/templates/backend.yaml

@@ -53,6 +53,18 @@ spec:
         - name: api
           image: {{ .Values.backend_image }}
           imagePullPolicy: {{ .Values.backend_pull_policy }}
+          command:
+              - gunicorn
+              - btrixcloud.main:app_root
+              - --bind
+              - "0.0.0.0:8000"
+              - --access-logfile
+              - "-"
+              - --workers
+              - "{{ .Values.backend_workers | default 1 }}"
+              - --worker-class
+              - uvicorn.workers.UvicornWorker
+
           envFrom:
             - configMapRef:
                 name: backend-env-config
@@ -62,8 +74,8 @@ spec:
                 name: mongo-auth
 
           env:
-            - name: WEB_CONCURRENCY
-              value: "{{ .Values.backend_workers | default 4 }}"
+            - name: MOTOR_MAX_WORKERS
+              value: "{{ .Values.backend_mongodb_workers | default 1 }}"
 
           volumeMounts:
             - name: ops-configs
@@ -114,15 +126,16 @@ spec:
           image: {{ .Values.backend_image }}
           imagePullPolicy: {{ .Values.backend_pull_policy }}
           command:
-              - uvicorn
+              - gunicorn
               - btrixcloud.main_op:app_root
-              - --host
-              - 0.0.0.0
-              - --port
-              - "{{ .Values.opPort }}"
-              - --access-log
-              - --log-level
-              - info
+              - --bind
+              - "0.0.0.0:{{ .Values.opPort }}"
+              - --access-logfile
+              - "-"
+              - --workers
+              - "{{ .Values.backend_workers | default 1 }}"
+              - --worker-class
+              - uvicorn.workers.UvicornWorker
 
           envFrom:
             - configMapRef:
@@ -133,8 +146,8 @@ spec:
                 name: mongo-auth
 
           env:
-            - name: WEB_CONCURRENCY
-              value: "{{ .Values.operator_workers | default 1 }}"
+            - name: MOTOR_MAX_WORKERS
+              value: "{{ .Values.backend_mongodb_workers | default 1 }}"
 
           volumeMounts:
             - name: config-volume

chart/values.yaml

@@ -102,7 +102,7 @@ backend_password_secret: "PASSWORD!"
 backend_num_replicas: 1
 
 # number of workers per pod
-backend_workers: 2
+backend_workers: 1
 
 backend_cpu: "25m"