From 42fe42941337334e78b1edba308f66d5287e304f Mon Sep 17 00:00:00 2001 From: Christian Ariza Date: Thu, 4 Jun 2020 15:48:25 +0200 Subject: [PATCH] [Celery]Using multiple queues Using multiple queues the tasks from different types can be served in parallel. --- docker-compose.yml | 2 +- src/htcondor_es/celery/tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a51081..594a33b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - CMS_ES_CONF_FILE=/run/secrets/es-conf - REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt - LOGLEVEL=DEBUG - entrypoint: ['celery', "-A", "htcondor_es.celery.celery", "worker"] + entrypoint: ['celery', "-A", "htcondor_es.celery.celery", "worker","-Q","default,es_post,convert"] secrets: &secrets - amq-username - amq-password diff --git a/src/htcondor_es/celery/tasks.py b/src/htcondor_es/celery/tasks.py index 663225b..d5cb010 100644 --- a/src/htcondor_es/celery/tasks.py +++ b/src/htcondor_es/celery/tasks.py @@ -117,7 +117,7 @@ def query_schedd( return (schedd_ad["name"], n_tasks) -@app.task(ignore_result=True) +@app.task(ignore_result=True,queue="convert") def process_docs( docs, reduce_data=True, @@ -169,7 +169,7 @@ def process_docs( return post_ads(converted_docs) if converted_docs else [] -@app.task(ignore_result=True) +@app.task(ignore_result=True, queue="es_post") def post_ads_es(es_docs, es_index, metadata=None): """ Send the messages to ES.