[k8s] Change the required resources

Reducing the required resources, adding a shared redis storate, adding RuntimeError exception to retry for in the query schedd.

k8s/deployments/spider-flower.yaml

@@ -25,6 +25,12 @@ spec:
         name: spider-flower
         ports:
         - containerPort: 8888
-        resources: {}
+        resources:
+          requests:
+             cpu: 100m
+             memory: 500Mi
+          limits:
+            cpu: 300m
+            memory: 1Gi
       restartPolicy: Always
 status: {}

k8s/deployments/spider-redis-cp.yaml

@@ -30,11 +30,11 @@ spec:
         - containerPort: 6379
         resources: 
           requests:
-            cpu: 500m
+            cpu: 100m
             memory: 500Mi
           limits:
             cpu: 1000m
-            memory: 4Gi
+            memory: 1Gi
         volumeMounts:
         - mountPath: /data
           name: redis-checkpoint-claim0

k8s/deployments/spider-redis.yaml

@@ -37,5 +37,6 @@ spec:
       restartPolicy: Always
       volumes:
       - name: redis-claim0
-        emptyDir: {}
+        persistentVolumeClaim:
+          claimName: redis-cephfs-claim
 status: {}

k8s/deployments/spider-worker.yaml

@@ -28,10 +28,10 @@ spec:
         resources:
           requests:
              cpu: 100m
-             memory: 200Mi
+             memory: 500Mi
           limits:
             cpu: 2000m
-            memory: 1Gi
+            memory: 2Gi
         env: &spider_env
         - name: AFFILIATION_DIR_LOCATION
           value: /cms_shared/affiliation_dir.json

k8s/storages/shared_redis.yml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: redis-cephfs-claim
+  namespace: spider
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 1G
+  storageClassName: meyrin-cephfs

src/htcondor_es/celery/tasks.py

@@ -51,7 +51,7 @@
 
 
 # ---Tasks----
-@app.task(max_retries=3, serializer="pickle")
+@app.task(max_retries=3, autoretry_for=(RuntimeError,) serializer="pickle")
 def query_schedd(
     schedd_ad,
     start_time=None,