diff --git a/README.md b/README.md index cd26d1c..a34b8db 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,17 @@ To change the endpoints, update the code in [api/app.py](api/app.py) Task changes should happen in [celery-queue/tasks.py](celery-queue/tasks.py) + +### Setup considerations + +From: [Redis administration](https://redis.io/docs/management/admin/) + +``` +Make sure to set Linux kernel overcommit memory setting to 1. + +This can be done by adding vm.overcommit_memory=1 to /etc/sysctl.conf. Then, reboot or run the command sysctl vm.overcommit_memory=1 to activate the setting. +``` + --- adapted from [https://github.com/itsrifat/flask-celery-docker-scale](https://github.com/itsrifat/flask-celery-docker-scale) diff --git a/api/Dockerfile b/api/Dockerfile index e158921..ba2ceab 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,8 +1,7 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/Dockerfile.dev b/api/Dockerfile.dev index b0fd2e3..3f0e7df 100644 --- a/api/Dockerfile.dev +++ b/api/Dockerfile.dev @@ -1,8 +1,7 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/requirements.txt b/api/requirements.txt index d0b24fd..900d1cf 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,3 +1,3 @@ -Flask==2.0.1 -celery==5.1.2 -redis==3.5.3 \ No newline at end of file +Flask==2.3.2 +celery==5.3.1 +redis==4.6.0 \ No newline at end of file diff --git a/api/worker.py b/api/worker.py index 0ed39d7..ffd7a06 100644 --- a/api/worker.py +++ b/api/worker.py @@ -5,5 +5,10 @@ CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') - -celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +celery = Celery(__name__, + broker=CELERY_BROKER_URL, + backend=CELERY_RESULT_BACKEND, + broker_connection_retry=True, + broker_connection_retry_on_startup=True, + broker_connection_max_retries=10, + ) diff --git a/celery-queue/Dockerfile b/celery-queue/Dockerfile index 3d3621b..a1bed50 100644 --- a/celery-queue/Dockerfile +++ b/celery-queue/Dockerfile @@ -1,8 +1,7 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue diff --git a/celery-queue/Dockerfile.dev b/celery-queue/Dockerfile.dev index e3ab9ab..45bc42f 100644 --- a/celery-queue/Dockerfile.dev +++ b/celery-queue/Dockerfile.dev @@ -1,8 +1,7 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue @@ -11,4 +10,4 @@ RUN pip install -U setuptools pip RUN pip install -r requirements.txt # hot code reloading -CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO -E +CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO --uid=nobody --gid=nogroup -E diff --git a/celery-queue/requirements.txt b/celery-queue/requirements.txt index edd1ecf..9f06c78 100644 --- a/celery-queue/requirements.txt +++ b/celery-queue/requirements.txt @@ -1,5 +1,4 @@ -celery==5.1.2 -flower==1.0.0 -redis==3.5.3 -watchdog==2.1.5 - +celery==5.3.1 +flower==2.0.0 +redis==4.6.0 +watchdog==3.0.0 \ No newline at end of file diff --git a/celery-queue/tasks.py b/celery-queue/tasks.py index 987cee6..579e3d8 100644 --- a/celery-queue/tasks.py +++ b/celery-queue/tasks.py @@ -6,7 +6,13 @@ CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +celery = Celery(__name__, + broker=CELERY_BROKER_URL, + backend=CELERY_RESULT_BACKEND, + broker_connection_retry=True, + broker_connection_retry_on_startup=True, + broker_connection_max_retries=10, + ) @celery.task(name='tasks.add') diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9aa0265..237db6a 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -17,7 +17,7 @@ services: build: context: ./celery-queue dockerfile: Dockerfile.dev - command: celery -A tasks worker -l info -E + command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E environment: CELERY_BROKER_URL: redis://redis CELERY_RESULT_BACKEND: redis://redis @@ -30,7 +30,7 @@ services: dockerfile: Dockerfile.dev ports: - "5555:5555" - command: ['celery', 'flower', '-A', 'tasks'] + command: ['celery', '-A', 'tasks', 'flower'] environment: CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index 17ead9e..77eaed8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: build: context: ./celery-queue dockerfile: Dockerfile - command: celery -A tasks worker -l info -E + command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E environment: CELERY_BROKER_URL: redis://redis CELERY_RESULT_BACKEND: redis://redis @@ -27,7 +27,7 @@ services: dockerfile: Dockerfile ports: - "5555:5555" - command: ['celery', 'flower', '-A', 'tasks'] + command: ['celery', '-A', 'tasks', 'flower'] environment: CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0