Pull request update/231213

.github/workflows/test_bi_scheduler.yaml

@@ -17,4 +17,6 @@ jobs:
       - name: Build image
         run: bash -x build.sh bi_scheduler build
       - name: Build test image and run tests
-        run: bash -x docker_images/bi_scheduler/run_test.sh
+        run: bash -x docker_images/run_test.sh bi_scheduler
+      - name: Build test image and run unit tests
+        run: bash -x docker_images/bi_scheduler/run_test.sh

.github/workflows/test_booking_observer.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh booking_observer build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh booking_observer

.github/workflows/test_calendar_observer.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh calendar_observer build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh calendar_observer

.github/workflows/test_cleaninfluxdb.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh cleaninfluxdb build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh cleaninfluxdb *.py

.github/workflows/test_cleanmongodb.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh cleanmongodb build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh cleanmongodb *.py

.github/workflows/test_configurator.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh configurator build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh configurator

.github/workflows/test_demo_org_cleanup.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh demo_org_cleanup build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh demo_org_cleanup

.github/workflows/test_failed_imports_dataset_generator.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh failed_imports_dataset_generator build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh failed_imports_dataset_generator

.github/workflows/test_herald_executor.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh herald_executor build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh herald_executor

.github/workflows/test_keeper_executor.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh keeper_executor build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh keeper_executor

.github/workflows/test_live_demo_generator.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh live_demo_generator build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh live_demo_generator

.github/workflows/test_ohsu.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh ohsu build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh ohsu

.github/workflows/test_organization_violations.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh organization_violations build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh organization_violations

.github/workflows/test_resource_discovery.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh resource_discovery build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh resource_discovery

.github/workflows/test_resource_observer.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh resource_observer build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh resource_observer

.github/workflows/test_resource_violations.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh resource_violations build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh resource_violations

.github/workflows/test_slacker_executor.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh slacker_executor build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh slacker_executor

.github/workflows/test_users_dataset_generator.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh users_dataset_generator build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh users_dataset_generator

.github/workflows/test_webhook_executor.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh webhook_executor build
+      - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh webhook_executor

docker_images/.pylintrc

@@ -0,0 +1,10 @@
+[REPORTS]
+reports=no
+
+[MESSAGES CONTROL]
+disable=R,C0111,C0103
+
+[BASIC]
+good-names=i, db, id
+generated-members=deleted, id, state, metadata
+

docker_images/Dockerfile_tests

@@ -0,0 +1,7 @@
+ARG BUILDTAG=build
+ARG IMAGE=
+FROM ${IMAGE}:${BUILDTAG}
+LABEL org.opencontainers.image.authors="Hystax"
+
+COPY docker_images/test-requirements.txt docker_images/.pylintrc ./
+RUN pip install --no-cache-dir -r test-requirements.txt

docker_images/booking_observer/worker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import os
-import requests
 import time
 from pymongo import MongoClient, UpdateOne
 from datetime import datetime
@@ -11,7 +10,7 @@
 from kombu.utils.debug import setup_logging
 from kombu import Connection as QConnection, Exchange, Queue
 from kombu.pools import producers
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -119,8 +118,8 @@ def _process(self, organization_id, observe_time):
 
     def _publish_activities_tasks(self, tasks):
         queue_conn = QConnection('amqp://{user}:{pass}@{host}:{port}'.format(
-                **self.config_cl.read_branch('/rabbit')),
-                transport_options=RETRY_POLICY)
+            **self.config_cl.read_branch('/rabbit')),
+            transport_options=RETRY_POLICY)
         task_exchange = Exchange(ACTIVITIES_EXCHANGE_NAME, type='topic')
         with producers[queue_conn].acquire(block=True) as producer:
             for task_params in tasks:
@@ -149,7 +148,7 @@ def heartbeat(self):
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'INFO' if not debug else 'DEBUG'
     setup_logging(loglevel=log_level, loggers=[''])

docker_images/calendar_observer/worker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import os
-import requests
 import time
 
 from threading import Thread
@@ -10,7 +9,7 @@
 from kombu import Connection
 from kombu.utils.debug import setup_logging
 from kombu import Exchange, Queue
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -69,7 +68,7 @@ def heartbeat(self):
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'DEBUG' if debug else 'INFO'
     setup_logging(loglevel=log_level, loggers=[''])

docker_images/cleanmongodb/clean-mongo-db.py

@@ -81,7 +81,7 @@ def update_cleaned_at(self, cloud_account_id):
             session.close()
 
     @staticmethod
-    def delete_rows(collection, cloud_account_id,  chunk_size, rows_limit):
+    def delete_rows(collection, cloud_account_id, chunk_size, rows_limit):
         row_ids = list(collection.find(
             {'cloud_account_id': cloud_account_id}, ['_id']).limit(rows_limit))
         for j in range(0, len(row_ids), chunk_size):

docker_images/failed_imports_dataset_generator/failed_imports_dataset_generator.py

@@ -104,7 +104,7 @@ def _get_failed_cloud_accounts(mydb):
             ca_t.last_import_at, ca_t.last_import_attempt_error,
             ca_t.organization_id, org_t.name
         FROM (
-            SELECT id, name, organization_id, type, created_at, last_import_at, 
+            SELECT id, name, organization_id, type, created_at, last_import_at,
                 last_import_attempt_error
             FROM cloudaccount
             WHERE deleted_at=0

docker_images/herald_executor/worker.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import calendar
 import os
-import requests
 import time
 import uuid
 from enum import Enum
@@ -13,7 +12,7 @@
 from kombu import Connection
 from kombu.utils.debug import setup_logging
 from kombu import Exchange, Queue, binding
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -673,8 +672,8 @@ def _get_org_constraint_link(self, constraint, created_at, filters):
             end_date = datetime.combine(created, created.time().max) + timedelta(
                 days=1)
         elif constraint['type'] == 'expiring_budget':
-            start_date = datetime.utcfromtimestamp(constraint['definition'][
-                                                       'start_date'])
+            start_date = datetime.utcfromtimestamp(
+                constraint['definition']['start_date'])
             end_date = None
         elif constraint['type'] == 'recurring_budget':
             start_date = created.replace(day=1, hour=0, minute=0, second=0)
@@ -741,21 +740,21 @@ def _get_org_constraint_template_params(self, organization, constraint,
                  if constraint['type'] not in resource_types
                  else round(latest_hit['value']))
         params = {
-                'texts': {
-                    'title': title,
-                    'organization': self._get_organization_params(organization),
-                    'organization_constraint': {**constraint_data},
-                    'limit_hit': {
-                        'created_at': hit_date,
-                        'value': value,
-                        'link': link,
-                        'constraint_limit': constraint_limit
-                    },
-                    'user': {
-                        'user_display_name': user_info.get('display_name')
-                    },
-                }
+            'texts': {
+                'title': title,
+                'organization': self._get_organization_params(organization),
+                'organization_constraint': {**constraint_data},
+                'limit_hit': {
+                    'created_at': hit_date,
+                    'value': value,
+                    'link': link,
+                    'constraint_limit': constraint_limit
+                },
+                'user': {
+                    'user_display_name': user_info.get('display_name')
+                },
             }
+        }
         if constraint['type'] == 'tagging_policy':
             tag = constraint['definition']['conditions'].get('tag')
             without_tag = constraint['definition']['conditions'].get('without_tag')
@@ -966,7 +965,7 @@ def heartbeat(self):
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'DEBUG' if debug else 'INFO'
     setup_logging(loglevel=log_level, loggers=[''])

docker_images/keeper_executor/worker.py

@@ -8,7 +8,7 @@
 from kombu.log import get_logger
 from kombu.utils.debug import setup_logging
 from kombu import Exchange, Queue, binding, Connection
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -626,7 +626,7 @@ def heartbeat(self):
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'DEBUG' if debug else 'INFO'
     setup_logging(loglevel=log_level, loggers=[''])

docker_images/live_demo_generator/scheduler.py

@@ -44,7 +44,7 @@ def main(config_cl):
     })
     if count > 0:
         publish_tasks(count)
-        LOG.info('Published %s tasks' % count )
+        LOG.info('Published %s tasks' % count)
     deleted = live_demos_collection.delete_many({
         'created_at': {'$lt': int(dt.timestamp())}
     }).deleted_count

docker_images/live_demo_generator/worker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import os
-import requests
 import time
 
 from threading import Thread
@@ -10,7 +9,7 @@
 from kombu import Connection
 from kombu.utils.debug import setup_logging
 from kombu import Exchange, Queue
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 from datetime import datetime
 
 from optscale_client.config_client.client import Client as ConfigClient
@@ -72,11 +71,11 @@ def generate_live_demo(self):
         response['created_at'] = int(d_start.timestamp())
         self.mongo_cl.restapi.live_demos.insert_one(response)
         LOG.info('Live demo generated in %s seconds' % (
-                datetime.utcnow() - d_start).total_seconds())
+            datetime.utcnow() - d_start).total_seconds())
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'INFO' if not debug else 'DEBUG'
     setup_logging(loglevel=log_level, loggers=[''])

docker_images/ohsu/handlers/v2/base.py

@@ -30,7 +30,7 @@ def initialize(self, config):
         self._controller = None
 
     def raise405(self):
-        raise OptHTTPError(405, Err.OH0002, [self.request.method])
+        raise OptHTTPError(405, Err.OHE0002, [self.request.method])
 
     def head(self, *args, **kwargs):
         self.raise405()

docker_images/organization_violations/worker.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import calendar
 import os
-import requests
 import time
 import uuid
 from copy import deepcopy
@@ -14,7 +13,7 @@
 from kombu import Connection as QConnection
 from kombu import Exchange, Queue
 from kombu.pools import producers
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import urllib3
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -517,7 +516,7 @@ def process_organization_constraints(self, task):
                 self.month_start(date), start.time())).days + 1,
             # hit_days for resource_quota and anomalies constraints
             (start - datetime.combine(
-                        date - timedelta(days=1), start.time())).days + 1]
+                date - timedelta(days=1), start.time())).days + 1]
         _, response = self.rest_cl.organization_constraint_list(
             org_id, hit_days=max(hit_days))
         constraints = response.get('organization_constraints')
@@ -536,8 +535,8 @@ def process_organization_constraints(self, task):
                               'Error: %s' % (constr['id'], str(exc)))
         self.publish_activities_tasks(notif_tasks)
         LOG.info('Organization violation process for organization %s completed'
-                 ' in %s seconds' % (
-                    org_id, int(datetime.utcnow().timestamp()) - start_ts))
+                 ' in %s seconds' %
+                 (org_id, int(datetime.utcnow().timestamp()) - start_ts))
 
     def process_task(self, body, message):
         try:
@@ -553,7 +552,7 @@ def heartbeat(self):
 
 
 if __name__ == '__main__':
-    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+    urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
     debug = os.environ.get('DEBUG', False)
     log_level = 'INFO' if not debug else 'DEBUG'
     setup_logging(loglevel=log_level, loggers=[''])