[Celery]Using multiple queues

Using multiple queues the tasks from different types can be served in parallel.
dmwm · Jun 4, 2020 · 42fe429 · 42fe429
1 parent 74e63fa
commit 42fe429
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,7 +27,7 @@ services:
       - CMS_ES_CONF_FILE=/run/secrets/es-conf
       - REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt
       - LOGLEVEL=DEBUG
-    entrypoint: ['celery', "-A", "htcondor_es.celery.celery", "worker"]
+    entrypoint: ['celery', "-A", "htcondor_es.celery.celery", "worker","-Q","default,es_post,convert"]
     secrets: &secrets
       - amq-username
       - amq-password

diff --git a/src/htcondor_es/celery/tasks.py b/src/htcondor_es/celery/tasks.py
@@ -117,7 +117,7 @@ def query_schedd(
     return (schedd_ad["name"], n_tasks)
 
 
-@app.task(ignore_result=True)
+@app.task(ignore_result=True,queue="convert")
 def process_docs(
     docs,
     reduce_data=True,
@@ -169,7 +169,7 @@ def process_docs(
     return post_ads(converted_docs) if converted_docs else []
 
 
-@app.task(ignore_result=True)
+@app.task(ignore_result=True, queue="es_post")
 def post_ads_es(es_docs, es_index, metadata=None):
     """
     Send the messages to ES.