Pull request update/240529

f0d1062 OS-7579. Updaded README file in order to use Python 3.9 during
cluste… (#809)
6559239 OS-7575. RI/SP breakdown performance improvement
3fa1885 OS-7507. nebius_migration archive
7ebcb54 OS-7578. Handle missing AttachedInstanceId for Alibaba cloud

README.md

@@ -128,7 +128,7 @@ _The current installation process does not work on Ubuntu 22.04_
 Run the following commands:
 
 ```
-sudo apt update; sudo apt install git python3-venv python3-dev sshpass python3.9
+sudo apt update; sudo apt install python3-pip sshpass git python3-virtualenv python3.9
 ```
 
 #### Pulling optscale-deploy scripts
@@ -150,8 +150,8 @@ cd optscale/optscale-deploy
 Run the following commands:
 
 ```
-python3 -m venv .venv
-source .venv/bin/activate
+virtualenv -p python3.9 venv
+source venv/bin/activate
 pip install -r requirements.txt
 ```
 

bumiworker/bumiworker/modules/archive/nebius_migration.py

@@ -0,0 +1,39 @@
+from collections import defaultdict
+
+from bumiworker.bumiworker.consts import ArchiveReason
+from bumiworker.bumiworker.modules.base import ArchiveBase
+from bumiworker.bumiworker.modules.recommendations.nebius_migration import (
+    NebiusMigration as NebiusMigrationRecommendation
+)
+
+
+class NebiusMigration(ArchiveBase, NebiusMigrationRecommendation):
+
+    @property
+    def supported_cloud_types(self):
+        return list(self.get_cloud_funcs_map().keys())
+
+    def _get(self, previous_options, optimizations, cloud_accounts_map,
+             **kwargs):
+        result = []
+        if not self._is_nebius_option_enabled():
+            return result
+
+        account_optimizations_map = defaultdict(list)
+        for optimization in optimizations:
+            account_optimizations_map[optimization['cloud_account_id']].append(
+                optimization)
+
+        for cloud_account_id, optimizations_ in account_optimizations_map.items():
+            reason = ArchiveReason.OPTIONS_CHANGED if (
+                cloud_account_id in cloud_accounts_map
+            ) else ArchiveReason.CLOUD_ACCOUNT_DELETED
+            for optimization in optimizations_:
+                self._set_reason_properties(optimization, reason)
+                result.append(optimization)
+        return result
+
+
+def main(organization_id, config_client, created_at, **kwargs):
+    return NebiusMigration(
+        organization_id, config_client, created_at).get()

bumiworker/bumiworker/modules/recommendations/nebius_migration.py

@@ -202,16 +202,19 @@ def _is_nebius_option_enabled(self):
         feature_options = json.loads(response['value'])
         return feature_options.get('nebius_connection_enabled', 0)
 
-    def _get(self):
-        if not self._is_nebius_option_enabled():
-            return []
-        (days_threshold, skip_cloud_accounts) = self.get_options_values()
-        cloud_func_map = {
+    def get_cloud_funcs_map(self):
+        return {
             'aws_cnr': self._get_aws_usages,
             'azure_cnr': self._get_azure_usages,
             'alibaba_cnr': self._get_alibaba_usages,
             'gcp_cnr': self._get_gcp_usages
         }
+
+    def _get(self):
+        if not self._is_nebius_option_enabled():
+            return []
+        (days_threshold, skip_cloud_accounts) = self.get_options_values()
+        cloud_func_map = self.get_cloud_funcs_map()
         cloud_account_map = self.get_cloud_accounts(
             supported_cloud_types=list(cloud_func_map.keys()),
             skip_cloud_accounts=skip_cloud_accounts)

diworker/diworker/importers/alibaba.py

@@ -39,7 +39,8 @@ def _get_system_disk_ids(self, current_day):
             item['AttachedInstanceId']: item['InstanceId']
             for item in self.cloud_adapter.get_raw_usage(
                 'YunDisk', 'Hour', start_time, end_time)
-            if item['Portable'] == '0'
+            if (item['Portable'] == '0' and 'AttachedInstanceId' in item
+                and 'InstanceId' in item)
         }
 
     def _get_snapshot_chain_usage(self, current_day):

diworker/diworker/migrations/2024052311450000_added_aws_lineitemtype_index.py

@@ -0,0 +1,40 @@
+import logging
+from diworker.diworker.migrations.base import BaseMigration
+
+"""
+Added aws LineItemType index
+"""
+
+LOG = logging.getLogger(__name__)
+NEW_INDEXES = {
+    'AWSLineItemType': (
+        ['cloud_account_id', 'lineItem/LineItemType', 'start_date'],
+        {'lineItem/LineItemType': {'$exists': True}}
+    )
+}
+
+
+class Migration(BaseMigration):
+    @property
+    def raw_collection(self):
+        return self.db.raw_expenses
+
+    def _get_existing_index_names(self):
+        return [i['name'] for i in self.raw_collection.list_indexes()]
+
+    def upgrade(self):
+        existing_indexes = self._get_existing_index_names()
+        for index_name, (index_keys, partial_exp) in NEW_INDEXES.items():
+            if index_name in existing_indexes:
+                continue
+            self.raw_collection.create_index(
+                [(f, 1) for f in index_keys],
+                name=index_name,
+                background=True,
+                partialFilterExpression=partial_exp)
+
+    def downgrade(self):
+        existing_indexes = self._get_existing_index_names()
+        for index_name in NEW_INDEXES.keys():
+            if index_name in existing_indexes:
+                self.raw_collection.drop_index(index_name)

rest_api/rest_api_server/controllers/ri_breakdown.py

@@ -120,7 +120,6 @@ def get_flavors(self, cloud_account_ids):
                 'start_date': {
                     '$gte': datetime.fromtimestamp(self.start_date),
                     '$lte': datetime.fromtimestamp(self.end_date)},
-                'box_usage': True,
                 'lineItem/LineItemType': 'DiscountedUsage'
             }
         )

rest_api/rest_api_server/controllers/sp_breakdown.py

@@ -38,18 +38,28 @@ def get_usage_breakdown(self, start_date, end_date, cloud_account_ids):
 
     def get_flavors(self, cloud_account_ids):
         flavor_rate_map = defaultdict(float)
-        expenses = self.raw_expenses_collection.find({
-                    'cloud_account_id': {'$in': cloud_account_ids},
-                    'start_date': {
-                        '$gte': datetime.fromtimestamp(self.start_date),
-                        '$lt': datetime.fromtimestamp(self.end_date)},
-                    'box_usage': True,
-                    'lineItem/LineItemType': 'SavingsPlanCoveredUsage'
-                })
+        expenses = self.raw_expenses_collection.aggregate([
+            {'$match': {
+                'cloud_account_id': {'$in': cloud_account_ids},
+                'start_date': {
+                    '$gte': datetime.fromtimestamp(self.start_date),
+                    '$lt': datetime.fromtimestamp(self.end_date)
+                },
+                'lineItem/LineItemType': 'SavingsPlanCoveredUsage'
+            }},
+            {'$group': {
+                '_id': {
+                    'instance_type': '$product/instanceType',
+                    'description': '$lineItem/LineItemDescription'
+                },
+                'rate': {'$last': '$savingsPlan/SavingsPlanRate'}
+            }}
+        ])
+        # todo: it could be two different rates for flavor
         for expense in expenses:
-            flavor_name = (expense.get('product/instanceType') or
-                           expense['lineItem/LineItemDescription'])
-            sp_rate = float(expense.get('savingsPlan/SavingsPlanRate', 0))
+            _id = expense['_id']
+            flavor_name = _id.get('instance_type') or _id.get('description')
+            sp_rate = float(expense.get('rate', 0))
             if sp_rate:
                 flavor_rate_map[flavor_name] = sp_rate
         return flavor_rate_map

rest_api/rest_api_server/controllers/ttl_analysis.py

@@ -25,7 +25,7 @@ def _get_model_type(self):
 
     def _get_resource_expenses(self, pool_id, start_date, end_date):
         resources = list(self.resources_collection.find(
-            {'pool_id': pool_id}, []))
+            {'pool_id': pool_id}, ['_id']))
         query = """
             SELECT resource_id, SUM(sign * cost)
             FROM expenses

rest_api/rest_api_server/tests/unittests/test_cloud_resources.py

@@ -329,7 +329,7 @@ def test_patch_environment(self):
         params = {'env_properties': {'field': 'value'}}
         code, response = self.client.cloud_resource_update(resource['id'], params)
         self.assertEqual(code, 200)
-        history_len = self.property_history_collection.count()
+        history_len = self.property_history_collection.count_documents({})
         self.assertEqual(history_len, 1)
 
     def test_properties_with_dots(self):
@@ -347,7 +347,7 @@ def test_properties_with_dots(self):
         params = {'env_properties': {'field.1': 'value'}}
         code, response = self.client.cloud_resource_update(resource['id'], params)
         self.assertEqual(code, 200)
-        history_len = self.property_history_collection.count()
+        history_len = self.property_history_collection.count_documents({})
         self.assertEqual(history_len, 1)
 
     def test_patch_shareable_with_bookings(self):

rest_api/test-requirements.txt

@@ -4,7 +4,7 @@ pylint==3.0.2
 freezegun==0.3.8
 pytest-xdist==3.5.0
 pytest==8.1.0
-mongomock==3.22.1
+mongomock==4.1.2
 pyyaml==6.0.1
 
 # OptScale packages