From fe0f0819e05573ac9abc05ad416860a7ec9668c5 Mon Sep 17 00:00:00 2001 From: wolf1996 Date: Mon, 21 May 2018 04:04:40 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=80=D0=B0=D0=B7=D0=B2=D1=91=D1=80=D1=82=D1=8B=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=B2=20=D0=BE=D0=B4=D0=BD=D1=83=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BC=D0=B0=D0=BD=D0=B4=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 113 +++++++++++++----- python/data/Makefile | 39 +++++- python/data/VERSION | 1 + .../data/kuber}/data-deployment.yaml | 2 +- .../data/kuber}/data-service.yaml | 0 python/gateway/Makefile | 37 ++++++ python/gateway/VERSION | 1 + .../gateway/kuber}/gateway-deployment.yaml | 0 .../gateway/kuber}/gateway-service.yaml | 0 python/receiver/Makefile | 37 +++++- python/receiver/VERSION | 1 + .../receiver/app/data_broker/data_broker.py | 17 ++- .../receiver/kuber}/reciever-deployment.yaml | 0 .../receiver/kuber}/reciever-service.yaml | 0 python/stats/Makefile | 38 +++++- .../stats/kuber}/stats-deployment.yaml | 0 .../stats/kuber}/stats-service.yaml | 0 17 files changed, 243 insertions(+), 43 deletions(-) create mode 100644 python/data/VERSION rename {kuber => python/data/kuber}/data-deployment.yaml (92%) rename {kuber => python/data/kuber}/data-service.yaml (100%) create mode 100644 python/gateway/VERSION rename {kuber => python/gateway/kuber}/gateway-deployment.yaml (100%) rename {kuber => python/gateway/kuber}/gateway-service.yaml (100%) create mode 100644 python/receiver/VERSION rename {kuber => python/receiver/kuber}/reciever-deployment.yaml (100%) rename {kuber => python/receiver/kuber}/reciever-service.yaml (100%) rename {kuber => python/stats/kuber}/stats-deployment.yaml (100%) rename {kuber => python/stats/kuber}/stats-service.yaml (100%) diff --git a/Makefile b/Makefile index 207585f..ea44af2 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,58 @@ build_all: \ build_stats \ build_gateway \ +# не работает, надо лапками вводить команду у себя для работы use_kubernetes_docker: - eval $(minikube docker-env) + @eval $$(minikube docker-env) +# выкатываем с тегом +docker_image_build_gateway_minikube: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/gateway docker_build + +docker_image_build_data_minikube: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/data docker_build + +docker_image_build_stats_minikube: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/stats docker_build + +docker_image_build_receiver_minikube: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/receiver docker_build + +docker_images_minikube_build: \ + docker_image_build_gateway_minikube \ + docker_image_build_data_minikube \ + docker_image_build_stats_minikube \ + docker_image_build_receiver_minikube + +#выкатываем без перетеггирования +docker_image_build_gateway_minikube_curr: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/gateway docker_build_curr + +docker_image_build_data_minikube_curr: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/data docker_build_curr + +docker_image_build_stats_minikube_curr: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/stats docker_build_curr + +docker_image_build_receiver_minikube_curr: + @eval $$(minikube docker-env);\ + $(MAKE) -C python/receiver docker_build_curr + + +docker_images_minikube_build_curr: \ + docker_image_build_gateway_minikube_curr \ + docker_image_build_data_minikube_curr \ + docker_image_build_stats_minikube_curr \ + docker_image_build_receiver_minikube_curr + +# просто выкатываем образы docker_image_build_gateway: $(MAKE) -C python/gateway docker_build @@ -88,68 +137,76 @@ rebuild_protobuf: # data kubernetes kubernetes_data_service: - kubectl apply -f kuber/data-service.yaml + $(MAKE) -C python/data kubernetes_service kubernetes_data_deployment: - kubectl apply -f kuber/data-deployment.yaml + $(MAKE) -C python/data kubernetes_deployment kubernetes_data_deployment_remove: - kubectl delete deployment -l app=data + $(MAKE) -C python/data kubernetes_deployment_remove kubernetes_data_service_remove: - kubectl delete service -l name=data + $(MAKE) -C python/data kubernetes_service_remove + + +#Собираем новый образ и запускаем +kubernetes_data_buildnload: + @eval $$(minikube docker-env); \ + $(MAKE) -C python/data kubernetes_buildnload # stats kubernetes kubernetes_stats_service: - kubectl apply -f kuber/stats-service.yaml + $(MAKE) -C python/stats kubernetes_service kubernetes_stats_deployment: - kubectl apply -f kuber/stats-deployment.yaml + $(MAKE) -C python/stats kubernetes_deployment kubernetes_stats_deployment_remove: - kubectl delete deployment -l app=stats + $(MAKE) -C python/stats kubernetes_deployment_remove kubernetes_stats_service_remove: - kubectl delete service -l name=stats + $(MAKE) -C python/stats kubernetes_service_remove + +kubernetes_stats_buildnload: + @eval $$(minikube docker-env); \ + $(MAKE) -C python/stats kubernetes_buildnload + # gateway kubernetes kubernetes_gateway_service: - kubectl apply -f kuber/gateway-service.yaml + $(MAKE) -C python/gateway kubernetes_service kubernetes_gateway_deployment: - kubectl apply -f kuber/gateway-deployment.yaml + $(MAKE) -C python/gateway kubernetes_deployment kubernetes_gateway_deployment_remove: - kubectl delete deployment -l app=gateway + $(MAKE) -C python/gateway kubernetes_deployment_remove kubernetes_gateway_service_remove: - kubectl delete service -l name=gateway + $(MAKE) -C python/gateway kubernetes_service_remove + +kubernetes_gateway_buildnload: + @eval $$(minikube docker-env); \ + $(MAKE) -C python/gateway kubernetes_buildnload + # reciever kubernetes kubernetes_reciever_service: - kubectl apply -f kuber/reciever-service.yaml + $(MAKE) -C python/receiver kubernetes_service kubernetes_reciever_deployment: - kubectl apply -f kuber/reciever-deployment.yaml + $(MAKE) -C python/receiver kubernetes_deployment kubernetes_reciever_deployment_remove: - kubectl delete deployment -l app=reciever + $(MAKE) -C python/receiver kubernetes_deployment_remove kubernetes_reciever_service_remove: - kubectl delete service -l name=reciever + $(MAKE) -C python/receiver kubernetes_service_remove -# # rabbitmq -# kubernetes_rabbitmq_service: -# kubectl apply -f kuber/rabbitmq-service.yaml - -# kubernetes_rabbitmq_controller: -# kubectl apply -f kuber/rabbitmq-controller.yaml - -# kubernetes_rabbitmq_controller_remove: -# kubectl delete pod -l component=rabbitmq +kubernetes_receiver_buildnload: + @eval $$(minikube docker-env); \ + $(MAKE) -C python/receiver kubernetes_buildnload -# kubernetes_rabbitmq_service_remove: -# kubectl delete service -l name=rabbitmq # устанавливаем dns kubernetes_install_dns: diff --git a/python/data/Makefile b/python/data/Makefile index c8e6b19..c5918ae 100644 --- a/python/data/Makefile +++ b/python/data/Makefile @@ -30,10 +30,41 @@ docker_container_remove: docker_run_container: docker run -p 5000:5000 -d --name $(CONTAINERNAME) $(IMAGENAME) -docker_build: - docker build -t $(IMAGENAME) . - docker_run: \ docker_stop \ docker_container_remove \ - docker_run_container \ No newline at end of file + docker_run_container + +docker_build: \ + version_file_increment \ + docker_build_curr + +docker_build_curr: + o=`cut -f1 VERSION`; \ + docker build -t $(IMAGENAME):$$o . ; + +kubernetes_service: + kubectl apply -f kuber/data-service.yaml + +kubernetes_deployment: + kubectl apply -f kuber/data-deployment.yaml + +kubernetes_deployment_remove: + kubectl delete deployment -l app=data + +kubernetes_service_remove: + kubectl delete service -l name=data + +version_file_increment: + o=`cut -f1 VERSION`; \ + n=`expr $$o + 1`; \ + echo $$n > VERSION; \ + +kubernetes_buildnload: \ + docker_build \ + kubernetes_update_image + +kubernetes_update_image: + o=`cut -f1 VERSION`; \ + echo "image version is ", $$o; \ + kubectl set image deployment/data-deployment $(IMAGENAME)=$(IMAGENAME):$$o \ No newline at end of file diff --git a/python/data/VERSION b/python/data/VERSION new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/python/data/VERSION @@ -0,0 +1 @@ +5 diff --git a/kuber/data-deployment.yaml b/python/data/kuber/data-deployment.yaml similarity index 92% rename from kuber/data-deployment.yaml rename to python/data/kuber/data-deployment.yaml index 2294b3c..d14292c 100644 --- a/kuber/data-deployment.yaml +++ b/python/data/kuber/data-deployment.yaml @@ -16,6 +16,6 @@ spec: containers: - name: data imagePullPolicy: Never - image: data:latest + image: data ports: - containerPort: 5000 diff --git a/kuber/data-service.yaml b/python/data/kuber/data-service.yaml similarity index 100% rename from kuber/data-service.yaml rename to python/data/kuber/data-service.yaml diff --git a/python/gateway/Makefile b/python/gateway/Makefile index a11681e..f6adb50 100644 --- a/python/gateway/Makefile +++ b/python/gateway/Makefile @@ -48,3 +48,40 @@ stop: build: cp -r ../../depends/python depends + +kubernetes_service: + kubectl apply -f kuber/reciever-service.yaml + +kubernetes_deployment: + kubectl apply -f kuber/reciever-deployment.yaml + +kubernetes_deployment_remove: + kubectl delete deployment -l app=gateway + +kubernetes_service_remove: + kubectl delete service -l name=gateway + +docker_build: \ + version_file_increment \ + docker_build_curr + +docker_build_curr: + o=`cut -f1 VERSION`; \ + docker build -t $(IMAGENAME):$$o . ; + +kubernetes_service_remove: + kubectl delete service -l name=gateway + +version_file_increment: + o=`cut -f1 VERSION`; \ + n=`expr $$o + 1`; \ + echo $$n > VERSION; \ + +kubernetes_buildnload: \ + docker_build \ + kubernetes_update_image + +kubernetes_update_image: + o=`cut -f1 VERSION`; \ + echo "image version is ", $$o; \ + kubectl set image deployment/gateway-deployment $(IMAGENAME)=$(IMAGENAME):$$o \ No newline at end of file diff --git a/python/gateway/VERSION b/python/gateway/VERSION new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/python/gateway/VERSION @@ -0,0 +1 @@ +6 diff --git a/kuber/gateway-deployment.yaml b/python/gateway/kuber/gateway-deployment.yaml similarity index 100% rename from kuber/gateway-deployment.yaml rename to python/gateway/kuber/gateway-deployment.yaml diff --git a/kuber/gateway-service.yaml b/python/gateway/kuber/gateway-service.yaml similarity index 100% rename from kuber/gateway-service.yaml rename to python/gateway/kuber/gateway-service.yaml diff --git a/python/receiver/Makefile b/python/receiver/Makefile index 35ca05b..4f9d355 100644 --- a/python/receiver/Makefile +++ b/python/receiver/Makefile @@ -31,10 +31,41 @@ docker_container_remove: docker_run_container: docker run -p 5000:5000 -d --name $(CONTAINERNAME) $(IMAGENAME) -docker_build: - docker build -t $(IMAGENAME) . +docker_build: \ + version_file_increment \ + docker_build_curr + +docker_build_curr: + o=`cut -f1 VERSION`; \ + docker build -t $(IMAGENAME):$$o . ; docker_run: \ docker_stop \ docker_container_remove \ - docker_run_container \ No newline at end of file + docker_run_container + +kubernetes_service: + kubectl apply -f kuber/reciever-service.yaml + +kubernetes_deployment: + kubectl apply -f kuber/reciever-deployment.yaml + +kubernetes_deployment_remove: + kubectl delete deployment -l app=reciever + +kubernetes_service_remove: + kubectl delete service -l name=reciever + +version_file_increment: + o=`cut -f1 VERSION`; \ + n=`expr $$o + 1`; \ + echo $$n > VERSION; \ + +kubernetes_buildnload: \ + docker_build \ + kubernetes_update_image + +kubernetes_update_image: + o=`cut -f1 VERSION`; \ + echo "image version is ", $$o; \ + kubectl set image deployment/reciever-deployment $(IMAGENAME)=$(IMAGENAME):$$o \ No newline at end of file diff --git a/python/receiver/VERSION b/python/receiver/VERSION new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/python/receiver/VERSION @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/python/receiver/app/data_broker/data_broker.py b/python/receiver/app/data_broker/data_broker.py index f88be47..d19d4f3 100644 --- a/python/receiver/app/data_broker/data_broker.py +++ b/python/receiver/app/data_broker/data_broker.py @@ -1,15 +1,17 @@ import pika import json import time +import logging + +log = logging.getLogger("flask.app") class DataBroker: _SENSOR_DATA_QUEUE = 'sensor_data' def __init__(self, host, port=5672): credentials = pika.PlainCredentials('user','user') - self._connection = pika.BlockingConnection(pika.ConnectionParameters(host=host, port=port,credentials=credentials )) + self._connection = pika.BlockingConnection(pika.ConnectionParameters(host=host, port=port,credentials=credentials, heartbeat=0)) self._chanel = self._connection.channel() - self._declare_sensor_data_queue() def declare_queue(self, queue_cls): @@ -18,8 +20,15 @@ def declare_queue(self, queue_cls): def _declare_sensor_data_queue(self): self._chanel.queue_declare(queue=DataBroker._SENSOR_DATA_QUEUE) - def send_sensor_data(self, sensor_data): - sensor_data['timestamp'] = sensor_data['timestamp'].strftime("%Y-%m-%dT%H:%M:%S") + def _send_data(self, sensor_data): self._chanel.basic_publish(exchange='', routing_key='sensor_data', body=json.dumps(sensor_data)) + + def send_sensor_data(self, sensor_data): + sensor_data['timestamp'] = sensor_data['timestamp'].strftime("%Y-%m-%dT%H:%M:%S") + try: + self._send_data(sensor_data) + except pika.exceptions.ConnectionClosed as e: + log.error("failed to connect {}".format(str(e))) + self._send_data(sensor_data) \ No newline at end of file diff --git a/kuber/reciever-deployment.yaml b/python/receiver/kuber/reciever-deployment.yaml similarity index 100% rename from kuber/reciever-deployment.yaml rename to python/receiver/kuber/reciever-deployment.yaml diff --git a/kuber/reciever-service.yaml b/python/receiver/kuber/reciever-service.yaml similarity index 100% rename from kuber/reciever-service.yaml rename to python/receiver/kuber/reciever-service.yaml diff --git a/python/stats/Makefile b/python/stats/Makefile index d5f1803..e5c442e 100644 --- a/python/stats/Makefile +++ b/python/stats/Makefile @@ -30,10 +30,42 @@ docker_container_remove: docker_run_container: docker run -p 5000:5000 -d --name $(CONTAINERNAME) $(IMAGENAME) -docker_build: - docker build -t $(IMAGENAME) . +docker_build: \ + version_file_increment \ + docker_build_curr + +docker_build_curr: + o=`cut -f1 VERSION`; \ + docker build -t $(IMAGENAME):$$o . ; docker_run: \ docker_stop \ docker_container_remove \ - docker_run_container \ No newline at end of file + docker_run_container + +kubernetes_service: + kubectl apply -f kuber/stats-service.yaml + +kubernetes_deployment: + kubectl apply -f kuber/stats-deployment.yaml + +kubernetes_deployment_remove: + kubectl delete deployment -l app=stats + +kubernetes_service_remove: + kubectl delete service -l name=stats + + +version_file_increment: + o=`cut -f1 VERSION`; \ + n=`expr $$o + 1`; \ + echo $$n > VERSION; \ + +kubernetes_buildnload: \ + docker_build \ + kubernetes_update_image + +kubernetes_update_image: + o=`cut -f1 VERSION`; \ + echo "image version is ", $$o; \ + kubectl set image deployment/stats-deployment $(IMAGENAME)=$(IMAGENAME):$$o \ No newline at end of file diff --git a/kuber/stats-deployment.yaml b/python/stats/kuber/stats-deployment.yaml similarity index 100% rename from kuber/stats-deployment.yaml rename to python/stats/kuber/stats-deployment.yaml diff --git a/kuber/stats-service.yaml b/python/stats/kuber/stats-service.yaml similarity index 100% rename from kuber/stats-service.yaml rename to python/stats/kuber/stats-service.yaml