[k8s] Trying the 32bit redis to decrease memory requirements

dmwm · Jun 3, 2020 · b493954 · b493954
1 parent aa5d280
commit b493954
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 12 deletions.
diff --git a/k8s/cronjobs/spider-cron-affiliation.yaml b/k8s/cronjobs/spider-cron-affiliation.yaml
@@ -46,8 +46,9 @@ spec:
                 name: shared-spider
           volumes:
             - name: shared-spider
-              persistentVolumeClaim:
-                claimName: affilations-cephfs-claim
+              emptyDir: {}
+        #PROD#persistentVolumeClaim:
+        #PROD#  claimName: affilations-cephfs-claim
           restartPolicy: Never
           securityContext:
             fsGroup: 1414
diff --git a/k8s/deployments/spider-redis-cp.yaml b/k8s/deployments/spider-redis-cp.yaml
@@ -31,7 +31,7 @@ spec:
         resources: 
           requests:
             cpu: 100m
-            memory: 500Mi
+            memory: 200Mi
           limits:
             cpu: 1000m
             memory: 1Gi

diff --git a/k8s/deployments/spider-redis.yaml b/k8s/deployments/spider-redis.yaml
@@ -20,14 +20,14 @@ spec:
         app: redis
     spec:
       containers:
-      - image: redis:alpine
+      - image: redis:5.0.9-32bit
         name: redis
         ports:
         - containerPort: 6379
         resources: 
           requests:
             cpu: 200m
-            memory: 1Gi
+            memory: 3Gi
           limits:
             cpu: 500m
             memory: 4Gi

diff --git a/k8s/deployments/spider-worker.yaml b/k8s/deployments/spider-worker.yaml
@@ -28,11 +28,11 @@ spec:
         - worker
         resources:
           requests:
-             cpu: 100m
+             cpu: 300m
              memory: 500Mi
           limits:
-            cpu: 2000m
-            memory: 2Gi
+            cpu: 1000m
+            memory: 1Gi
         env: &spider_env
         - name: AFFILIATION_DIR_LOCATION
           value: /cms_shared/affiliation_dir.json
@@ -103,5 +103,6 @@ spec:
             path: collectors
           secretName: collectors
       - name: shared-spider
-        persistentVolumeClaim:
-          claimName: affilations-cephfs-claim
+        emptyDir: {}
+        #PROD#persistentVolumeClaim:
+        #PROD#  claimName: affilations-cephfs-claim
diff --git a/src/htcondor_es/celery/tasks.py b/src/htcondor_es/celery/tasks.py
@@ -10,7 +10,6 @@
 import htcondor
 import classad
 from celery import group
-from celery.signals import celeryd_init
 from itertools import zip_longest, islice
 import collections
 from htcondor_es.convert_to_json import (
@@ -191,7 +190,7 @@ def post_ads_es(es_docs, es_index, metadata=None):
         traceback.print_exc()
 
 
-@app.task()
+@app.task(ignore_result=True)
 def create_affiliation_dir(days=1):
     try:
         output_file = os.getenv(