Pull request update/231226

316b123 OS-7200: Fixed broken Selector component
942964a OS-7187: Fixed community docs route mismatches
c4437f1 OS-7199. Check alembic down revisions
6534d77 OS-7188: Fixed false error blink in docs panel when switching
pages
95d2bb4 Merge feature/ml_leaderboard into integration
291d87d OS-7196. Fixed migration tree in restapi
030be74 OS-6765: Add Learn panel
461af7b OS-7138. Replace pep8 with pycodestyle and pylint upgrade
2e767d4 OS-7099. Make pylint tests work: pharos_backend
3b341a1 OS-7100. Make pylint tests work: rest_api
70d8d9d OS-7179: Updated TS and packages
37d10f2 OS-7097. Make pylint tests work: keeper
f470c67 OS-7095. Make pylint tests work: jira_bus
b0ed4ec OS-7069. Bulldozer pylint
713379f OS-7093. Make pylint tests work: herald

.github/workflows/test_bulldozer.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh bulldozer build
+      - name: Build test image and run tests
+        run: bash -x bulldozer/run_test.sh

.github/workflows/test_jira_bus.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened, synchronize]
     paths:
-    - 'jira_bus/jira_bus_server/**'
+    - 'jira_bus/**'
   workflow_dispatch:
 
 

.github/workflows/test_layout_cleaner.yaml

@@ -0,0 +1,20 @@
+name: Build layout_cleaner test image and run tests on it
+run-name: Build layout_cleaner test image and run tests on it - started by ${{ github.actor }}
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+    - 'docker_images/layout_cleaner/**'
+  workflow_dispatch:
+
+
+jobs:
+  build_image:
+    runs-on: self-hosted
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Build image
+        run: bash -x build.sh layout_cleaner build
+      - name: Build test image and run tests
+        run: bash -x docker_images/layout_cleaner/run_test.sh

.github/workflows/test_pharos_worker.yaml

@@ -16,3 +16,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Build image
         run: bash -x build.sh pharos_worker build
+      - name: Build test image and run tests
+        run: bash -x pharos_backend/pharos_worker/run_test.sh

.github/workflows/test_power_schedule.yaml

@@ -17,4 +17,6 @@ jobs:
       - name: Build image
         run: bash -x build.sh power_schedule build
       - name: Build test image and run tests
+        run: bash -x docker_images/run_test.sh power_schedule
+      - name: Build test image and run unit tests
         run: bash -x docker_images/power_schedule/run_test.sh

arcee/Dockerfile

@@ -10,6 +10,7 @@ COPY optscale_client/aconfig_cl optscale_client/aconfig_cl
 RUN pip install --no-cache-dir -r /usr/src/app/arcee_receiver/requirements.txt
 
 COPY arcee/arcee_receiver/migrations ./migrations
+COPY arcee/arcee_receiver/modules ./modules
 COPY arcee/arcee_receiver/*.py ./
 
 CMD ["python", "-u","/usr/src/app/arcee_receiver/server.py"]

arcee/arcee_receiver/migrations/20231016161400_added_decription_to_applications.py

@@ -0,0 +1,9 @@
+from mongodb_migrations.base import BaseMigration
+
+
+class Migration(BaseMigration):
+    def upgrade(self):
+        self.db.application.update_many({}, {"$set": {"description": None}})
+
+    def downgrade(self):
+        self.db.application.update_many({}, {"$unset": {"description": ""}})

arcee/arcee_receiver/migrations/20231024174700_changed_runs_datasets.py

@@ -0,0 +1,28 @@
+from mongodb_migrations.base import BaseMigration
+from pymongo import UpdateOne
+
+BULK_SIZE = 5000
+
+
+class Migration(BaseMigration):
+    def upgrade(self):
+        run_dataset_map = {}
+        for run in self.db.run.find({'dataset_ids': {'$exists': True}}):
+            run_dataset_map[run['_id']] = run['dataset_ids'][-1]
+        update_operations = []
+        for run_id, dataset_id in run_dataset_map.items():
+            update_operations.append(UpdateOne(
+                filter={
+                    '_id': run_id
+                },
+                update={'$set': {'dataset_id': dataset_id}},
+            ))
+        if update_operations:
+            self.db.run.update_many({}, {'$unset': {'dataset_ids': ""}})
+            for i in range(0, len(update_operations), BULK_SIZE):
+                bulk = update_operations[i:i + BULK_SIZE]
+            self.db.run.bulk_write(bulk)
+
+    def downgrade(self):
+        # there is no need to revert dataset_ids -> dataset_id transition
+        pass

arcee/arcee_receiver/migrations/20231031154932_added_dataset_optional_fields.py

@@ -0,0 +1,19 @@
+from mongodb_migrations.base import BaseMigration
+
+
+class Migration(BaseMigration):
+    def upgrade(self):
+        self.db.dataset.update_many({}, {"$set": {
+            "training_set": None,
+            "validation_set": None,
+            "timespan_from": None,
+            "timespan_to": None
+        }})
+
+    def downgrade(self):
+        self.db.dataset.update_many({}, {"$unset": {
+            "training_set": "",
+            "validation_set": "",
+            "timespan_from": "",
+            "timespan_to": ""
+        }})