Merge branch 'fix/add_storage_url_tests' into 'main'

fix: ignore local storage urls when generating partial mirror

See merge request espressif/idf-component-manager!448

ci/diff_docs.py

@@ -21,6 +21,7 @@ def main(base_folder: str, preview_folder: str, output_dir: str):
     preview_folder_html_rel_paths = set(
         glob.glob('**/*.html', root_dir=preview_folder, recursive=True)
     )
+    has_diff = False
     for item in sorted(base_folder_html_rel_paths | preview_folder_html_rel_paths):
         base_name = item if item in base_folder_html_rel_paths else ''
         preview_name = item if item in preview_folder_html_rel_paths else ''
@@ -49,9 +50,13 @@ def main(base_folder: str, preview_folder: str, output_dir: str):
                     fw.write(diff)
 
                 diff_url = f'{os.getenv("GITLAB_PAGE_HTTPS_URL")}/-/idf-component-manager/-/jobs/{os.getenv("CI_JOB_ID")}/artifacts/{output_file}'
+                has_diff = True
 
         table.add_row([base_name, preview_name, diff_url])
 
+    if not has_diff:
+        table = 'No differences found'
+
     print(table)
     if os.getenv('CI_JOB_ID'):
         post_mr_comment(table)

idf_component_tools/constants.py

@@ -18,6 +18,7 @@
 DEFAULT_NAMESPACE = 'espressif'
 IDF_COMPONENT_REGISTRY_URL = 'https://components.espressif.com/'
 IDF_COMPONENT_STORAGE_URL = 'https://components-file.espressif.com/'
+IDF_COMPONENT_STAGING_REGISTRY_URL = 'https://components-staging.espressif.com'
 
 UPDATE_SUGGESTION = """
 SUGGESTION: This component may be using a newer version of the component manager.

idf_component_tools/registry/multi_storage_client.py

@@ -70,10 +70,20 @@ def registry_storage_client(self) -> t.Optional[StorageClient]:
 
         return None
 
+    @property
+    @lru_cache(1)
+    def local_storage_clients(self):
+        return [StorageClient(url) for url in self.local_storage_urls]
+
+    @property
+    @lru_cache(1)
+    def profile_storage_clients(self):
+        return [StorageClient(url) for url in self.storage_urls]
+
     @property
     @lru_cache(1)
     def storage_clients(self):
-        clients = [StorageClient(url) for url in [*self.local_storage_urls, *self.storage_urls]]
+        clients = [*self.local_storage_clients, *self.profile_storage_clients]
         if self.registry_storage_client:
             clients.append(self.registry_storage_client)
 
@@ -111,8 +121,20 @@ def component(self, component_name: str, version: t.Optional[str] = None) -> t.D
         raise VersionNotFound(error_message)
 
     def get_component_info(self, component_name, spec):
+        """
+        Get component info from all storage clients.
+
+        .. note::
+
+            This function is only used for while partial mirror creation.
+        """
+        # local_storage_urls shall not be used when create partial mirror
+        _clients = self.profile_storage_clients.copy()
+        if self.registry_storage_client:
+            _clients.append(self.registry_storage_client)
+
         error_message = ''
-        for storage_client in self.storage_clients:
+        for storage_client in _clients:
             try:
                 info = storage_client.get_component_info(component_name=component_name, spec=spec)
                 return ComponentInfo(info, storage_client.storage_url)

tests/test_api_client.py

@@ -121,15 +121,15 @@ def test_file_adapter(self, fixtures_path):
         )
 
     def test_no_registry_token_error(self, monkeypatch, tmp_path):
-        monkeypatch.setenv('IDF_COMPONENT_STORAGE_URL', 'http://localhost:9000/test-public')
+        monkeypatch.setenv('IDF_COMPONENT_REGISTRY_URL', 'http://localhost:9000')
 
-        client = APIClient(registry_url=get_registry_url(), api_token='test')
+        client = APIClient(registry_url=get_registry_url())
 
         file_path = str(tmp_path / 'cmp.tgz')
         with open(file_path, 'w+') as f:
             f.write('a')
 
-        with pytest.raises(APIClientError, match='Token not found.'):
+        with pytest.raises(APIClientError, match='API token is required'):
             client.upload_version(component_name='example/cmp', file_path=file_path)
 
     def test_env_var_for_upload_empty(self, monkeypatch):

tests/test_profile.py

@@ -0,0 +1,182 @@
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+
+from jsonref import requests
+from pytest import fixture, raises, warns
+
+from idf_component_tools.config import Config
+from idf_component_tools.constants import (
+    DEFAULT_NAMESPACE,
+    IDF_COMPONENT_STAGING_REGISTRY_URL,
+)
+from idf_component_tools.messages import UserDeprecationWarning
+from idf_component_tools.registry.client_errors import APIClientError
+from idf_component_tools.registry.service_details import (
+    NoSuchProfile,
+    get_api_client,
+    get_profile,
+    get_storage_client,
+)
+
+
+@fixture()
+def service_config(tmp_path):
+    return Config().profiles.get('default', {})
+
+
+@fixture()
+def config_path(tmp_path):
+    config_path = os.path.join(str(tmp_path), 'idf_component_manager.yml')
+    with open(config_path, 'w+') as f:
+        f.write(
+            json.dumps({
+                'profiles': {
+                    'default': {
+                        'registry_url': 'https://default.com/',
+                        'default_namespace': 'default_ns',
+                        'api_token': None,
+                    },
+                    'test': {
+                        'registry_url': 'https://example.com/',
+                        'default_namespace': 'test',
+                        'api_token': 'token',
+                    },
+                    'test2': {
+                        'registry_url': 'https://example2.com/',
+                        'default_namespace': 'test2',
+                        'api_token': 'token',
+                    },
+                    'test3': {
+                        'registry_url': 'https://example3.com/',
+                        'default_namespace': 'test3',
+                        'api_token': 'token',
+                    },
+                    'emptyprofile': None,
+                }
+            })
+        )
+    return config_path
+
+
+class TestGetProfile:
+    def test_get_profile_success(self, config_path):
+        profile = get_profile('test', config_path)
+        assert profile.registry_url == 'https://example.com/'
+        assert profile.default_namespace == 'test'
+        assert profile.api_token == 'token'
+
+    def test_get_service_profile_env_dep(self, config_path, monkeypatch):
+        monkeypatch.setenv('IDF_COMPONENT_SERVICE_PROFILE', 'test')
+        with warns(UserDeprecationWarning):
+            assert get_profile(None, config_path=config_path).default_namespace == 'test'
+
+    def test_get_registry_profile_env_dep(self, config_path, monkeypatch):
+        monkeypatch.setenv('IDF_COMPONENT_REGISTRY_PROFILE', 'test2')
+        with warns(UserDeprecationWarning):
+            assert get_profile(None, config_path=config_path).default_namespace == 'test2'
+
+    def test_get_profile_env(self, config_path, monkeypatch):
+        monkeypatch.setenv('IDF_COMPONENT_PROFILE', 'test3')
+        assert get_profile(None, config_path=config_path).default_namespace == 'test3'
+
+    def test_get_profile_not_exist(self, config_path):
+        assert get_profile('not_exists', config_path) is None
+
+    def test_get_profile_with_default_name(self, config_path):
+        profile = get_profile('default', config_path)
+        assert profile.registry_url == 'https://default.com/'
+        assert profile.default_namespace == 'default_ns'
+        assert profile.api_token is None
+
+
+class TestApiClient:
+    def test_get_namespace_with_namespace(self):
+        assert get_api_client(namespace='example').default_namespace == 'example'
+
+    def test_get_namespace_from_profile(self, config_path):
+        api_client = get_api_client(profile_name='test', config_path=config_path)
+        assert api_client.default_namespace == 'test'
+
+    def test_get_token_env(self, monkeypatch):
+        monkeypatch.setenv('IDF_COMPONENT_API_TOKEN', 'some_token')
+
+        assert get_api_client().api_token == 'some_token'
+
+    def test_empty_env_profile(self, monkeypatch):
+        monkeypatch.setenv('IDF_COMPONENT_PROFILE', '')
+        with raises(
+            NoSuchProfile,
+            match='Profile "not_exists" not found in the idf_component_manager.yml config file',
+        ):
+            get_api_client(profile_name='not_exists')
+
+    def test_get_token_profile(self, config_path, monkeypatch):
+        api_client = get_api_client(profile_name='test', config_path=config_path)
+        assert api_client.api_token == 'token'
+
+    def test_service_details_without_token(self, tmp_path):
+        client = get_api_client(config_path=str(tmp_path), namespace='test')
+
+        with raises(APIClientError, match='API token is required'):
+            client.upload_version('file', 'component', '1.0.0')
+
+    def test_service_details_with_empty_profile(self, config_path):
+        client = get_api_client(config_path=config_path, profile_name='emptyprofile')
+        with raises(APIClientError, match='API token is required'):
+            client.upload_version('file', 'component', '1.0.0')
+
+
+class TestMultiStorageClient:
+    def test_get_namespace_default(self):
+        assert get_storage_client().default_namespace == DEFAULT_NAMESPACE
+
+    def test_service_details_success(self, config_path):
+        client = get_storage_client(profile_name='test', namespace='test', config_path=config_path)
+        assert client.registry_url == 'https://example.com/'
+        assert client.default_namespace == 'test'
+
+    def test_service_details_namespace_not_exist(self, tmp_path):
+        with raises(NoSuchProfile):
+            get_storage_client(config_path=str(tmp_path), profile_name='not_exists')
+
+    def test_service_details_without_profile(self, tmp_path):
+        with raises(NoSuchProfile, match='Profile "test" not found*'):
+            get_storage_client(config_path=str(tmp_path), profile_name='test', namespace='test')
+
+    def test_get_component_registry_url_with_profile(self, monkeypatch, config_path):
+        monkeypatch.setenv('IDF_COMPONENT_PROFILE', 'test')
+
+        client = get_storage_client(config_path=config_path)
+
+        assert client.registry_url == 'https://example.com/'
+        assert client.storage_urls == []
+
+    def test_registry_storage_url(self):
+        client = get_storage_client(registry_url=IDF_COMPONENT_STAGING_REGISTRY_URL)
+
+        assert (
+            client.registry_storage_client.storage_url
+            == requests.get(IDF_COMPONENT_STAGING_REGISTRY_URL + '/api').json()[
+                'components_base_url'
+            ]
+        )
+
+    def test_storage_clients_precedence(self):
+        client = get_storage_client(
+            registry_url=IDF_COMPONENT_STAGING_REGISTRY_URL,
+            storage_urls=['https://something.else'],
+            local_storage_urls=['file://local1', 'file://local2'],
+        )
+
+        assert client.storage_clients[0].storage_url == 'file://local1'
+        assert client.storage_clients[1].storage_url == 'file://local2'
+        assert client.storage_clients[2].storage_url == 'https://something.else'
+        assert (
+            client.storage_clients[3].storage_url
+            == requests.get(IDF_COMPONENT_STAGING_REGISTRY_URL + '/api').json()[
+                'components_base_url'
+            ]
+        )