From c8a24e049e2c350c4d4a3d2a2d8f2d26e30550d4 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Sun, 17 Sep 2023 21:33:29 +0200 Subject: [PATCH 01/42] refactor IMLNetworkManager import Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 2 +- XYZHubConnector/xyz_qgis/gui/iml/iml_space_info_dialog.py | 2 +- XYZHubConnector/xyz_qgis/iml/loader/iml_auth_loader.py | 2 +- XYZHubConnector/xyz_qgis/iml/loader/iml_space_loader.py | 2 +- XYZHubConnector/xyz_qgis/iml/network/__init__.py | 2 -- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index 9d51948..dddd294 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -75,7 +75,7 @@ IMLEditSyncController, ) from .xyz_qgis.iml.loader.iml_auth_loader import HomeProjectNotFound, AuthenticationError -from .xyz_qgis.iml.network import IMLNetworkManager +from .xyz_qgis.iml.network.network import IMLNetworkManager from .xyz_qgis.iml.models import IMLServerTokenConfig from .xyz_qgis import basemap diff --git a/XYZHubConnector/xyz_qgis/gui/iml/iml_space_info_dialog.py b/XYZHubConnector/xyz_qgis/gui/iml/iml_space_info_dialog.py index 329a46a..9e84f61 100644 --- a/XYZHubConnector/xyz_qgis/gui/iml/iml_space_info_dialog.py +++ b/XYZHubConnector/xyz_qgis/gui/iml/iml_space_info_dialog.py @@ -13,7 +13,7 @@ from qgis.PyQt.QtWidgets import QDialog, QInputDialog from .. import get_ui_class -from ...iml.network import IMLNetworkManager +from ...iml.network.network import IMLNetworkManager LAYER_TYPES = [ "interactivemap", diff --git a/XYZHubConnector/xyz_qgis/iml/loader/iml_auth_loader.py b/XYZHubConnector/xyz_qgis/iml/loader/iml_auth_loader.py index d6908b3..a882e01 100644 --- a/XYZHubConnector/xyz_qgis/iml/loader/iml_auth_loader.py +++ b/XYZHubConnector/xyz_qgis/iml/loader/iml_auth_loader.py @@ -21,7 +21,7 @@ ) from ...network.net_handler import NetworkError -from ...iml.network import IMLNetworkManager +from ...iml.network.network import IMLNetworkManager from ...common.signal import make_print_qgis diff --git a/XYZHubConnector/xyz_qgis/iml/loader/iml_space_loader.py b/XYZHubConnector/xyz_qgis/iml/loader/iml_space_loader.py index 187e659..11d71d4 100644 --- a/XYZHubConnector/xyz_qgis/iml/loader/iml_space_loader.py +++ b/XYZHubConnector/xyz_qgis/iml/loader/iml_space_loader.py @@ -9,7 +9,7 @@ ############################################################################### from .iml_auth_loader import IMLAuthLoader, IMLProjectScopedSemiAuthLoader -from ..network import IMLNetworkManager +from ..network.network import IMLNetworkManager from ...common.signal import make_fun_args, make_qt_args from ...controller import NetworkFun, WorkerFun, AsyncFun, ChainController, DelayedIdentityFun diff --git a/XYZHubConnector/xyz_qgis/iml/network/__init__.py b/XYZHubConnector/xyz_qgis/iml/network/__init__.py index 9b32aaa..ec79b22 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/__init__.py +++ b/XYZHubConnector/xyz_qgis/iml/network/__init__.py @@ -7,5 +7,3 @@ # License-Filename: LICENSE # ############################################################################### - -from .network import IMLNetworkManager, check_oauth2_token From bbf8c4fc46c842669f65a0323d2c7eada7f25bda Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 7 Sep 2023 23:40:09 +0200 Subject: [PATCH 02/42] add china platform url + commit encoded url + test Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/common/config.py | 4 +- XYZHubConnector/xyz_qgis/common/crypter.py | 23 +++++ .../xyz_qgis/iml/network/login_webengine.py | 43 +++++---- .../xyz_qgis/iml/network/network.py | 41 ++++----- .../xyz_qgis/iml/network/platform_server.py | 56 ++++++++++++ XYZHubConnector/xyz_qgis/models/connection.py | 4 +- test/test_crypter.py | 88 +++++++++++++++++++ 7 files changed, 219 insertions(+), 40 deletions(-) create mode 100644 XYZHubConnector/xyz_qgis/common/crypter.py create mode 100644 XYZHubConnector/xyz_qgis/iml/network/platform_server.py create mode 100644 test/test_crypter.py diff --git a/XYZHubConnector/xyz_qgis/common/config.py b/XYZHubConnector/xyz_qgis/common/config.py index 21fe4cc..5628233 100644 --- a/XYZHubConnector/xyz_qgis/common/config.py +++ b/XYZHubConnector/xyz_qgis/common/config.py @@ -10,8 +10,6 @@ import os -from qgis.PyQt.QtCore import QSettings - from ... import __version__ as version @@ -39,6 +37,8 @@ def get_external_os_lib(self): return lib_path def get_plugin_setting(self, key): + from qgis.PyQt.QtCore import QSettings + key_prefix = "xyz_qgis/settings" key_ = f"{key_prefix}/{key}" return QSettings().value(key_) diff --git a/XYZHubConnector/xyz_qgis/common/crypter.py b/XYZHubConnector/xyz_qgis/common/crypter.py new file mode 100644 index 0000000..3a6f947 --- /dev/null +++ b/XYZHubConnector/xyz_qgis/common/crypter.py @@ -0,0 +1,23 @@ +import base64 +from itertools import cycle + +CRYPTER_STRING = "7JC1bRsq_4UCTZZkoRO5_zEtd48P1lvTvA9xlI_T8WqilSU5FXS51gEawfvsIKvIinE" + + +def encrypt_text(text): + return xor_crypt_string(text, key=CRYPTER_STRING, encode=True) + + +def decrypt_text(text): + return xor_crypt_string(text, key=CRYPTER_STRING, decode=True) + + +def xor_crypt_string(data: str, key="xor_string", encode=False, decode=False): + bdata = data.encode("utf-8") if isinstance(data, str) else data + if decode: + bdata = base64.b64decode(bdata) + xored = bytes(x ^ y for x, y in zip(bdata, cycle(key.encode("utf-8")))) + out = xored + if encode: + out = base64.b64encode(xored) + return out.decode("utf-8").strip() diff --git a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py index fb63ce9..41bb064 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py @@ -11,7 +11,6 @@ import json import os - try: from PyQt5.Qt import PYQT_VERSION_STR @@ -28,6 +27,9 @@ QNetworkAccessManager, ) + +from .platform_server import PlatformServer, PlatformEndpoint +from ...common.crypter import decrypt_text from ...common.signal import BasicSignal from ...common.utils import get_qml_full_path, add_qml_import_path from ...models import SpaceConnectionInfo @@ -72,8 +74,9 @@ def save_access_token(self, conn_info: SpaceConnectionInfo): def open_login_view( self, conn_info: SpaceConnectionInfo, parent=None, cb_login_view_closed=None ): - # TODO: show qml dialog + login_url = PlatformLoginServer.get_login_url(conn_info.get_server()) self.view = self.create_qml_view( + login_url=login_url, title=self._dialog_title(conn_info), cb_login_view_closed=lambda *a: self.cb_login_view_closed( conn_info, cb_login_view_closed, *a @@ -124,7 +127,7 @@ def apply_token(cls, conn_info: SpaceConnectionInfo) -> str: return conn_info @classmethod - def create_qml_view(cls, title="", cb_login_view_closed=None): + def create_qml_view(cls, login_url: str, title="", cb_login_view_closed=None): view = QQuickView() engine = view.engine() @@ -135,10 +138,13 @@ def create_qml_view(cls, title="", cb_login_view_closed=None): # QTWEBENGINE_REMOTE_DEBUGGING: port debugMode = os.environ.get("HERE_QML_DEBUG", "") + view_props = {"loginUrl": login_url} if debugMode: - view.setInitialProperties({"debugMode": debugMode}) + title = title + " debug" + view.setInitialProperties(dict(view_props, debugMode=debugMode)) view.setSource(QUrl.fromLocalFile(get_qml_full_path("web_debug.qml"))) else: + view.setInitialProperties(dict(view_props)) view.setSource(QUrl.fromLocalFile(get_qml_full_path("web.qml"))) errors = [e.toString() for e in view.errors()] @@ -173,13 +179,23 @@ def remove_access_token(cls, conn_info: SpaceConnectionInfo): return conn_info -class PlatformUserAuthentication: - PLATFORM_URL_SIT = "https://platform.in.here.com" - PLATFORM_URL_PRD = "https://platform.here.com" - ENDPOINT_ACCESS_TOKEN = "/api/portal/accessToken" - ENDPOINT_TOKEN_EXCHANGE = "/api/portal/authTokenExchange" - ENDPOINT_SCOPED_TOKEN = "/api/portal/scopedTokenExchange" +class PlatformLoginServer: + PLATFORM_URL_PRD = PlatformServer.PLATFORM_URL_PRD + PLATFORM_SERVERS = { + SpaceConnectionInfo.PLATFORM_PRD: PLATFORM_URL_PRD, + SpaceConnectionInfo.PLATFORM_SIT: decrypt_text(PlatformServer.PLATFORM_URL_SIT), + SpaceConnectionInfo.PLATFORM_KOREA: PLATFORM_URL_PRD, + SpaceConnectionInfo.PLATFORM_CHINA: decrypt_text(PlatformServer.PLATFORM_URL_CHINA), + } + + ENDPOINT_SCOPED_TOKEN = decrypt_text(PlatformEndpoint.ENDPOINT_SCOPED_TOKEN) + + @classmethod + def get_login_url(cls, server): + return cls.PLATFORM_SERVERS.get(server, cls.PLATFORM_URL_PRD) + +class PlatformUserAuthentication: def __init__(self, network: QNetworkAccessManager): self.signal = BasicSignal() self.network = network @@ -190,11 +206,9 @@ def auth_project(self, conn_info: SpaceConnectionInfo): reply_tag = "oauth_project" project_hrn = conn_info.get_("project_hrn") - platform_server = ( - self.PLATFORM_URL_SIT if conn_info.is_platform_sit() else self.PLATFORM_URL_PRD - ) + platform_server = PlatformLoginServer.get_login_url(conn_info.get_server()) url = "{platform_server}{endpoint}".format( - platform_server=platform_server, endpoint=self.ENDPOINT_SCOPED_TOKEN + platform_server=platform_server, endpoint=PlatformLoginServer.ENDPOINT_SCOPED_TOKEN ) payload = {"scope": project_hrn} kw_prop = dict(reply_tag=reply_tag, req_payload=payload) @@ -207,7 +221,6 @@ def auth_project(self, conn_info: SpaceConnectionInfo): return reply def auth(self, conn_info: SpaceConnectionInfo): - reply_tag = "oauth" kw_prop = dict(reply_tag=reply_tag) diff --git a/XYZHubConnector/xyz_qgis/iml/network/network.py b/XYZHubConnector/xyz_qgis/iml/network/network.py index 81854e0..5fda6cf 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/network.py +++ b/XYZHubConnector/xyz_qgis/iml/network/network.py @@ -14,6 +14,8 @@ from .login_webengine import PlatformUserAuthentication, PlatformAuthLoginView from .net_handler import IMLNetworkHandler +from .platform_server import PlatformServer +from ...common.crypter import decrypt_text from ...models import SpaceConnectionInfo from ...network.network import NetManager from ...network.net_utils import ( @@ -31,42 +33,37 @@ class IMLNetworkManager(NetManager): TIMEOUT_COUNT = 10000 - API_PRD_URL = "https://interactive.data.api.platform.here.com/interactive/v1" - API_SIT_URL = "https://interactive-dev-eu-west-1.api-gateway.sit.ls.hereapi.com/interactive/v1" - API_KOREA_URL = "https://ap-northeast-2.interactive.data.api.platform.here.com/interactive/v1" - API_CONFIG_PRD_URL = "https://config.data.api.platform.here.com/config/v1" - API_CONFIG_SIT_URL = "https://config.data.api.platform.sit.here.com/config/v1" - API_OAUTH_PRD_URL = "https://account.api.here.com/oauth2/token" - API_OAUTH_SIT_URL = "https://stg.account.api.here.com/oauth2/token" - API_AUTH_PRD_URL = "https://account.api.here.com/authorization/v1.1" - API_AUTH_SIT_URL = "https://stg.account.api.here.com/authorization/v1.1" - API_GROUP_INTERACTIVE = "interactive" API_GROUP_CONFIG = "config" - API_GROUP_OAUTH = "oauth" API_GROUP_AUTH = "auth" + API_GROUP_OAUTH = "oauth" API_SIT = "SIT" API_PRD = "PRD" API_KOREA = "KOREA" + API_CHINA = "CHINA" API_URL = { API_GROUP_INTERACTIVE: { - API_KOREA: API_KOREA_URL, - API_PRD: API_PRD_URL, - API_SIT: API_SIT_URL, + API_PRD: PlatformServer.API_PRD_URL, + API_SIT: decrypt_text(PlatformServer.API_SIT_URL), + API_KOREA: decrypt_text(PlatformServer.API_KOREA_URL), + API_CHINA: decrypt_text(PlatformServer.API_CHINA_URL), }, API_GROUP_CONFIG: { - API_PRD: API_CONFIG_PRD_URL, - API_SIT: API_CONFIG_SIT_URL, - }, - API_GROUP_OAUTH: { - API_PRD: API_OAUTH_PRD_URL, - API_SIT: API_OAUTH_SIT_URL, + API_PRD: PlatformServer.API_CONFIG_PRD_URL, + API_SIT: decrypt_text(PlatformServer.API_CONFIG_SIT_URL), + API_CHINA: decrypt_text(PlatformServer.API_CONFIG_CHINA_URL), }, API_GROUP_AUTH: { - API_PRD: API_AUTH_PRD_URL, - API_SIT: API_AUTH_SIT_URL, + API_PRD: PlatformServer.API_AUTH_PRD_URL, + API_SIT: decrypt_text(PlatformServer.API_AUTH_SIT_URL), + API_CHINA: decrypt_text(PlatformServer.API_AUTH_CHINA_URL), + }, + API_GROUP_OAUTH: { + API_PRD: PlatformServer.API_OAUTH_PRD_URL, + API_SIT: decrypt_text(PlatformServer.API_OAUTH_SIT_URL), + API_CHINA: decrypt_text(PlatformServer.API_OAUTH_CHINA_URL), }, } diff --git a/XYZHubConnector/xyz_qgis/iml/network/platform_server.py b/XYZHubConnector/xyz_qgis/iml/network/platform_server.py new file mode 100644 index 0000000..8338d24 --- /dev/null +++ b/XYZHubConnector/xyz_qgis/iml/network/platform_server.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Copyright (c) 2023 HERE Europe B.V. +# +# SPDX-License-Identifier: MIT +# License-Filename: LICENSE +# +############################################################################### + + +class PlatformServer: + API_PRD_URL = "https://interactive.data.api.platform.here.com/interactive/v1" + API_CONFIG_PRD_URL = "https://config.data.api.platform.here.com/config/v1" + API_AUTH_PRD_URL = "https://account.api.here.com/authorization/v1.1" + API_OAUTH_PRD_URL = "https://account.api.here.com/oauth2/token" + + API_SIT_URL = ( + "Xz43QRFoXF42WiEmJjs5HwYkKhg7HzNZAUEVJ1QfAnlHb1gIBWQ4NUwyBggVfSZcMnY/" + "Rh8PIBMSBwYaZygZJEYHK0MvMVABJhoHOhsjcg==" + ) + API_CONFIG_SIT_URL = ( + "Xz43QRFoXF48WzslPT10Dw4mLhs+CixaFFhZJFcDBDlYMlAMQiE6Jl15EgYBfDZaKD46Uh4RdA==" + ) + API_AUTH_SIT_URL = "Xz43QRFoXF4sQDJtNTk5BBo8Oxs+CixaDFFKNR8PGTlZIEwMBCYtPUI2BQADPXpDd3Zi" + API_OAUTH_SIT_URL = "Xz43QRFoXF4sQDJtNTk5BBo8Oxs+CixaDFFKNR8PGTlZLlgNGCFte0w4GgwC" + + API_KOREA_URL = ( + "Xz43QRFoXF4+RHgtOyguAwozPEFySGsdCkBdIlAPAj0AJBccDT0+" + "elknGEccPzRBIDchWB8PIBMSSBUcJGQfJx0LN1YpN1gUN1wHbg==" + ) + + API_CHINA_URL = ( + "Xz43QRFoXF4sUzdtPTQuDh0zLEE2DCBaAFVMMR8NBj1YMVUZG" + "C8wJlV5GQweNjpZNnYwWx4OKxUSFBcQPSIALEYYdA==" + ) + API_CONFIG_CHINA_URL = ( + "Xz43QRFoXF48WzslPT10Dw4mLhs+CixaFFhZJFcDBDlYKVwKCSYzJBY0H0YPPDtTLz98QwA=" + ) + API_AUTH_CHINA_URL = ( + "Xz43QRFoXF46WDdtNzR3BQAgO10oHzYASQUWMVIPGSEYNRcQCTs6NUg+XwoCfDRAMjA8R1gdJBUeCRhcP3pYeA==" + ) + API_OAUTH_CHINA_URL = ( + "Xz43QRFoXF46WDdtNzR3BQAgO10oHzYASQUWMVIPGSEYNRcQCTs6NUg+XwoCfDpUMyw7Bx4TKgoSCA==" + ) + + PLATFORM_URL_PRD = "https://platform.here.com" + + PLATFORM_URL_SIT = "Xz43QRFoXF4vWDQ3MjUoBkE7IRs3HzcRSldXPQ==" + PLATFORM_URL_CHINA = "Xz43QRFoXF4vWDQ3MjUoBkE6Kkc6FSkESldW" + + +class PlatformEndpoint: + ENDPOINT_ACCESS_TOKEN = "GCszWE0iHAMrVTlsNTk5DhwhG1o0Hys=" + ENDPOINT_TOKEN_EXCHANGE = "GCszWE0iHAMrVTlsNS8uAzs9JFAxPz0XDFVWN1Q=" + ENDPOINT_SCOPED_TOKEN = "GCszWE0iHAMrVTlsJzk1Gwo2G1o0HysxHFdQMV8LEw==" diff --git a/XYZHubConnector/xyz_qgis/models/connection.py b/XYZHubConnector/xyz_qgis/models/connection.py index 31f0d17..b48dd11 100644 --- a/XYZHubConnector/xyz_qgis/models/connection.py +++ b/XYZHubConnector/xyz_qgis/models/connection.py @@ -29,7 +29,8 @@ class SpaceConnectionInfo(object): PLATFORM_SIT = "PLATFORM_SIT" PLATFORM_PRD = "PLATFORM_PRD" PLATFORM_KOREA = "PLATFORM_KOREA" - PLATFORM_SERVERS = [PLATFORM_PRD, PLATFORM_SIT, PLATFORM_KOREA] + PLATFORM_CHINA = "PLATFORM_CHINA" + PLATFORM_SERVERS = [PLATFORM_PRD, PLATFORM_SIT, PLATFORM_KOREA, PLATFORM_CHINA] PLATFORM_AUTH_KEYS = ["user_login", "realm", "here_credentials"] PLATFORM_KEYS = ["server"] + PLATFORM_AUTH_KEYS PLATFORM_DEFAULT_USER_LOGIN = "email" @@ -37,6 +38,7 @@ class SpaceConnectionInfo(object): PLATFORM_PRD: "HERE Platform", PLATFORM_SIT: "HERE Platform SIT", PLATFORM_KOREA: "HERE Platform Korea", + PLATFORM_CHINA: "HERE Platform China", } def __init__(self, conn_info=None): diff --git a/test/test_crypter.py b/test/test_crypter.py new file mode 100644 index 0000000..06fde77 --- /dev/null +++ b/test/test_crypter.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Copyright (c) 2023 HERE Europe B.V. +# +# SPDX-License-Identifier: MIT +# +############################################################################### + +import os + +from XYZHubConnector.xyz_qgis.common.crypter import ( + xor_crypt_string, + decrypt_text, + encrypt_text, +) +from XYZHubConnector.xyz_qgis.iml.network.platform_server import PlatformServer, PlatformEndpoint + +try: + from test.utils import BaseTestAsync as TestCase, unittest +except ImportError: + import unittest + from unittest import TestCase + + +# decorator +def debug_test_function(func): + return unittest.skipUnless(os.environ.get("DEBUG_TEST"), "Skipping debug test function")(func) + + +class TestCrypter(TestCase): + def test_crypter(self): + from secrets import token_urlsafe + + for i in range(10): + with self.subTest(i=i): + key = token_urlsafe(5) + text = "https://example.com/example-a" + enc = xor_crypt_string(text, key=key, encode=True) + dec = xor_crypt_string(enc, key=key, decode=True) + self.assertEqual(text, dec, "decrypted text does not match input text") + + def test_decrypt_platform_servers(self): + for server_name in filter(str.isupper, vars(PlatformServer)): + if "PRD" in server_name: + continue + value = getattr(PlatformServer, server_name) + self.subtest_decrypt_value(server_name, value, "^https://.*") + + def test_decrypt_platform_endpoints(self): + for name in filter(str.isupper, vars(PlatformEndpoint)): + value = getattr(PlatformEndpoint, name) + self.subtest_decrypt_value(name, value, "^/api") + + def subtest_decrypt_value(self, name, encoded, regex_pat): + with self.subTest(name=name): + self.assertNotRegex(encoded, regex_pat, "commited text should be encoded") + text = decrypt_text(encoded) + self.assertRegex(text, regex_pat) + # print(name, text) + + @debug_test_function + def test_encrypt_platform_servers(self): + for name in filter(str.isupper, vars(PlatformServer)): + url = getattr(PlatformServer, name) + if not url.startswith("https://"): + continue + encoded = encrypt_text(url) + print(name, encoded) + + @debug_test_function + def test_encrypt_platform_endpoint(self): + for name in filter(str.isupper, vars(PlatformEndpoint)): + endpoint = getattr(PlatformEndpoint, name) + if not endpoint.startswith("/api"): + continue + encoded = encrypt_text(endpoint) + print(name, encoded) + + @debug_test_function + def test_encrypt_string(self): + for text in []: + print(text, encrypt_text(text)) + + +if __name__ == "__main__": + unittest.main() + # unittest.main(defaultTest=["TestCrypter.test_encrypt_platform_servers"]) From 189b4f71962e7c55c261e3fbba6d76c6f07dd37d Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 15 Sep 2023 16:02:14 +0200 Subject: [PATCH 03/42] ui: show different platform servers if here system detected Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/common/utils.py | 8 ++++++++ XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/XYZHubConnector/xyz_qgis/common/utils.py b/XYZHubConnector/xyz_qgis/common/utils.py index 70061c6..0b9cc7c 100644 --- a/XYZHubConnector/xyz_qgis/common/utils.py +++ b/XYZHubConnector/xyz_qgis/common/utils.py @@ -212,3 +212,11 @@ def get_qml_import_base_path(): print(lib_path) return lib_path + + +def is_here_system(): + import socket + from .crypter import decrypt_text + + is_here_domain = decrypt_text("Vi5tWQcgFl88Wzg=") in socket.getfqdn() + return is_here_domain diff --git a/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py b/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py index 89c62f5..468ef24 100644 --- a/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py +++ b/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py @@ -12,6 +12,7 @@ from .. import get_ui_class from ..token_info_dialog import ServerInfoDialog +from ...common.utils import is_here_system from ...models import API_TYPES, SpaceConnectionInfo from ...iml.models.iml_token_model import get_api_type @@ -99,6 +100,8 @@ def __init__(self, parent=None): self.comboBox_api_type.addItems([s.upper() for s in API_TYPES]) self.comboBox_token.addItems(self.PLATFORM_SERVERS) self.comboBox_token.setEnabled(False) + if is_here_system(): + self.comboBox_token.setEnabled(True) self.comboBox_api_type.currentIndexChanged.connect(self.cb_change_api_type) self.comboBox_api_type.currentIndexChanged.connect(self.ui_enable_btn) From 1848872959dcf0686409c1ac615ac5cbf8929da3 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Wed, 27 Sep 2023 19:07:04 +0200 Subject: [PATCH 04/42] ui: set default similarity_threshold to 0 (single layering) Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/gui/ux/connect_ux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/gui/ux/connect_ux.py b/XYZHubConnector/xyz_qgis/gui/ux/connect_ux.py index 1b3288d..f03c42d 100644 --- a/XYZHubConnector/xyz_qgis/gui/ux/connect_ux.py +++ b/XYZHubConnector/xyz_qgis/gui/ux/connect_ux.py @@ -65,11 +65,11 @@ def config(self, *a): for text, data in [ ("single", 0), - ("maximal", 100), ("balanced", 80), + ("maximal", 100), ]: self.comboBox_similarity_threshold.addItem(text, data) - self.comboBox_similarity_threshold.setCurrentIndex(2) + self.comboBox_similarity_threshold.setCurrentIndex(0) self.comboBox_similarity_threshold.setToolTip( "\n".join( [ From 7fe4c17380fe6c1cc60edde1a789ef204b492fe2 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:00:32 +0200 Subject: [PATCH 05/42] correct prepare fields logic + refactor fields similarity to int 0-100 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/parser.py | 39 +++++++------- test/test_fields_similarity.py | 68 ++++++++++++------------ 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/parser.py b/XYZHubConnector/xyz_qgis/layer/parser.py index e9a5bda..4cb5ff2 100644 --- a/XYZHubConnector/xyz_qgis/layer/parser.py +++ b/XYZHubConnector/xyz_qgis/layer/parser.py @@ -40,6 +40,7 @@ PAYLOAD_LIMIT = int(1e7) # Amazon API limit: 10485760 URL_LIMIT = 2000 # max url length: 2000 URL_BASE_LEN = 60 # https://xyz.api.here.com/hub/spaces/12345678/features/ +DEFAULT_SIMILARITY_THRESHOLD = 0 # single: 0, balnced: 80 def make_lst_removed_ids(removed_ids): @@ -314,9 +315,9 @@ def filter_props_names(fields_names): return [s for s in fields_names if not is_special_key(s)] -def fields_similarity(ref_names, orig_names, names): +def fields_similarity(ref_names, orig_names, names) -> int: """ - compute fields similarity [0..1]. + compute fields similarity [0..100]. High score means 2 given fields are similar and should be merged """ @@ -327,8 +328,8 @@ def fields_similarity(ref_names, orig_names, names): x = len(same_names) # if n1 == 0 or n2 == 0: return 1 # handle empty, variant 1 if n1 == 0 and n2 == 0: - return 1 # handle empty, variant 2 - return max((1.0 * x / n) if n > 0 else 0 for n in [n1, n2]) + return 100 # handle empty, variant 2 + return int(max((100 * x / n) if n > 0 else 0 for n in [n1, n2])) def new_fields_gpkg(): @@ -462,7 +463,7 @@ def update_feature_fields(feat: QgsFeature, fields: QgsFields): return ft -def prepare_fields(feat_json, lst_fields, threshold=0.8): +def prepare_fields(feat_json, lst_fields, threshold=DEFAULT_SIMILARITY_THRESHOLD): """ Decide to merge fields or create new fields based on fields similarity score [0..1]. Score lower than threshold will result in creating new fields instead of merging fields @@ -483,21 +484,20 @@ def prepare_fields(feat_json, lst_fields, threshold=0.8): props_names, ) if fields.size() > 1 - else -1 + else -1 # mark empty fields for fields in lst_fields ] idx, score = max(enumerate(lst_score), key=lambda x: x[1], default=[0, 0]) idx_min, score_min = min(enumerate(lst_score), key=lambda x: x[1], default=[0, 0]) - if score < threshold or not idx < len(lst_fields): # new fields - if score_min >= 0: # new fields - idx = len(lst_fields) - fields = new_fields_gpkg() - lst_fields.append(fields) - else: # select empty fields - idx = idx_min - fields = lst_fields[idx] - else: + if len(lst_fields) == 0 or (score < threshold and score_min > -1): # new fields + idx = len(lst_fields) + fields = new_fields_gpkg() + lst_fields.append(fields) + elif score_min == -1: # select empty fields + idx = idx_min + fields = lst_fields[idx] + else: # select fields with highest score fields = lst_fields[idx] # print("len prop", len(props_names), idx, "score", lst_score, "lst_fields", len(lst_fields)) # print("len fields", [f.size() for f in lst_fields]) @@ -505,7 +505,9 @@ def prepare_fields(feat_json, lst_fields, threshold=0.8): return fields, idx -def xyz_json_to_feature_map(obj, map_fields=None, similarity_threshold=None): +def xyz_json_to_feature_map( + obj, map_fields=None, similarity_threshold=DEFAULT_SIMILARITY_THRESHOLD +): """ xyz json to feature, organize in to map of geometry, then to list of list of features. @@ -525,10 +527,7 @@ def _single_feature_map(feat_json, map_feat, map_fields): # if g is not None and not g.startswith("Multi"): g = "Multi" + g lst_fields = map_fields.setdefault(g, list()) - if similarity_threshold is None: - fields, idx = prepare_fields(feat_json, lst_fields) - else: - fields, idx = prepare_fields(feat_json, lst_fields, similarity_threshold / 100) + fields, idx = prepare_fields(feat_json, lst_fields, similarity_threshold) ft = xyz_json_to_feature(feat_json, fields) diff --git a/test/test_fields_similarity.py b/test/test_fields_similarity.py index 14aaa6c..ab14d55 100644 --- a/test/test_fields_similarity.py +++ b/test/test_fields_similarity.py @@ -42,27 +42,27 @@ def test_simple(self): fid = parser.QGS_ID xid = parser.QGS_XYZ_ID xyz_special_key = "@ns:com:here:xyz" - score = self.subtest_similarity_score([fid, "a", "b"], ["a", "b"], 1) - score = self.subtest_similarity_score([fid, "a"], ["a", "b"], 1) + score = self.subtest_similarity_score([fid, "a", "b"], ["a", "b"], 100) + score = self.subtest_similarity_score([fid, "a"], ["a", "b"], 100) score = self.subtest_similarity_score([fid, "a"], ["b"], 0) - score = self.subtest_similarity_score([fid, "a", "c"], ["a", "b"], 0.5) + score = self.subtest_similarity_score([fid, "a", "c"], ["a", "b"], 50) score = self.subtest_similarity_score( - [fid, xyz_special_key, "a", "b", "c"], [xyz_special_key, "a"], 1 + [fid, xyz_special_key, "a", "b", "c"], [xyz_special_key, "a"], 100 ) def test_empty(self): fid = parser.QGS_ID xid = parser.QGS_XYZ_ID xyz_special_key = "@ns:com:here:xyz" - # empty fields, shall returns merge fields (score 1) - score = self.subtest_similarity_score([fid], [], 1) - score = self.subtest_similarity_score([], [], 1) - score = self.subtest_similarity_score([xyz_special_key], [], 1) - score = self.subtest_similarity_score([xyz_special_key], [xyz_special_key], 1) - score = self.subtest_similarity_score([fid, xyz_special_key], [], 1) - score = self.subtest_similarity_score([fid, xyz_special_key], [xyz_special_key], 1) - score = self.subtest_similarity_score([fid], [], 1) - score = self.subtest_similarity_score([fid], [xyz_special_key], 1) + # empty fields, shall returns merge fields (score 100) + score = self.subtest_similarity_score([fid], [], 100) + score = self.subtest_similarity_score([], [], 100) + score = self.subtest_similarity_score([xyz_special_key], [], 100) + score = self.subtest_similarity_score([xyz_special_key], [xyz_special_key], 100) + score = self.subtest_similarity_score([fid, xyz_special_key], [], 100) + score = self.subtest_similarity_score([fid, xyz_special_key], [xyz_special_key], 100) + score = self.subtest_similarity_score([fid], [], 100) + score = self.subtest_similarity_score([fid], [xyz_special_key], 100) @unittest.skip("skip logic variant 1") def test_empty_variant_1(self): @@ -73,11 +73,11 @@ def test_empty_variant_1(self): # empty fields will be merged with any props # merge if empty fields OR empty props - score = self.subtest_similarity_score([fid], ["a"], 1) - score = self.subtest_similarity_score([fid, "a"], [], 1) - score = self.subtest_similarity_score([fid, xyz_special_key], ["a", xyz_special_key], 1) - score = self.subtest_similarity_score([fid], [fid], 1) - score = self.subtest_similarity_score([fid, xyz_special_key], [fid], 1) + score = self.subtest_similarity_score([fid], ["a"], 100) + score = self.subtest_similarity_score([fid, "a"], [], 100) + score = self.subtest_similarity_score([fid, xyz_special_key], ["a", xyz_special_key], 100) + score = self.subtest_similarity_score([fid], [fid], 100) + score = self.subtest_similarity_score([fid, xyz_special_key], [fid], 100) def test_empty_variant_2(self): fid = parser.QGS_ID @@ -94,19 +94,19 @@ def test_empty_variant_2(self): score = self.subtest_similarity_score([fid, xyz_special_key], ["a", xyz_special_key], 0) # fields and props empty - score = self.subtest_similarity_score([fid], [], 1) - score = self.subtest_similarity_score([fid], [xyz_special_key], 1) - score = self.subtest_similarity_score([xid], [xyz_special_key], 1) - score = self.subtest_similarity_score([fid, xid], [xyz_special_key], 1) - score = self.subtest_similarity_score([fid, xyz_special_key], [xyz_special_key], 1) - score = self.subtest_similarity_score([fid, xid, xyz_special_key], [xyz_special_key], 1) + score = self.subtest_similarity_score([fid], [], 100) + score = self.subtest_similarity_score([fid], [xyz_special_key], 100) + score = self.subtest_similarity_score([xid], [xyz_special_key], 100) + score = self.subtest_similarity_score([fid, xid], [xyz_special_key], 100) + score = self.subtest_similarity_score([fid, xyz_special_key], [xyz_special_key], 100) + score = self.subtest_similarity_score([fid, xid, xyz_special_key], [xyz_special_key], 100) # fields and props share common prop score = self.subtest_similarity_score( - [fid, xyz_special_key, "a"], [xyz_special_key, "a"], 1 + [fid, xyz_special_key, "a"], [xyz_special_key, "a"], 100 ) score = self.subtest_similarity_score( - [fid, xid, xyz_special_key, "a"], [xyz_special_key, "a"], 1 + [fid, xid, xyz_special_key, "a"], [xyz_special_key, "a"], 100 ) # special key in props will be renamed, thus not excluded @@ -125,32 +125,34 @@ def test_renamed_props(self): xid = parser.QGS_XYZ_ID xyz_special_key = "@ns:com:here:xyz" - score = self.subtest_similarity_score([fid, xid, parser.unique_field_name(fid)], [fid], 1) score = self.subtest_similarity_score( - [fid, xid, parser.unique_field_name(fid.upper())], [fid.upper()], 1 + [fid, xid, parser.unique_field_name(fid)], [fid], 100 ) score = self.subtest_similarity_score( - [parser.unique_field_name(fid.upper())], [fid.upper()], 1 + [fid, xid, parser.unique_field_name(fid.upper())], [fid.upper()], 100 + ) + score = self.subtest_similarity_score( + [parser.unique_field_name(fid.upper())], [fid.upper()], 100 ) score = self.subtest_similarity_score( [fid, xid, xyz_special_key, parser.unique_field_name(fid.upper())], [xyz_special_key, fid.upper()], - 1, + 100, ) score = self.subtest_similarity_score( [fid, xid, xyz_special_key, parser.unique_field_name(xid.upper())], [xyz_special_key, xid.upper()], - 1, + 100, ) score = self.subtest_similarity_score( [fid, xid, xyz_special_key, parser.unique_field_name(fid), "a"], [xyz_special_key, fid, "a"], - 1, + 100, ) score = self.subtest_similarity_score( [fid, xid, xyz_special_key, parser.unique_field_name(fid), "a"], [xyz_special_key, fid.upper(), "a"], - 1 / 2, + 50, ) def test_complex(self): From b5fb5bb0aa2cebb65cfdef58ababf9e90416999c Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 28 Sep 2023 03:19:31 +0200 Subject: [PATCH 06/42] try not clear auth for 403 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index dddd294..b8e7da5 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -459,7 +459,7 @@ def handle_net_err(self, err): # too many errors, handled by doing nothing return True # clear auth - if status in [401, 403]: + if status in [401]: if conn_info.is_platform_server() and conn_info.is_user_login(): self.network_iml.clear_auth(conn_info) return From ed0b16e774783ed71b15993ed1dae42e24556723 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:38:22 +0200 Subject: [PATCH 07/42] less network log Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/network/net_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/network/net_handler.py b/XYZHubConnector/xyz_qgis/network/net_handler.py index c96f05d..1886aae 100644 --- a/XYZHubConnector/xyz_qgis/network/net_handler.py +++ b/XYZHubConnector/xyz_qgis/network/net_handler.py @@ -128,7 +128,7 @@ def log_status(self): space_id, url, ) - QgsMessageLog.logMessage("Network Ok! : %s" % msg, config.TAG_PLUGIN, Qgis.Success) + # QgsMessageLog.logMessage("Network Ok! : %s" % msg, config.TAG_PLUGIN, Qgis.Success) def on_received(reply): From 22c25a4dfb02911c89a80304c337af859fd4db2e Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:49:27 +0200 Subject: [PATCH 08/42] treat all iml layer as livemap for editing flow + show catalog name and owner in layer metadata listing Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/net_handler.py | 8 +++++++- XYZHubConnector/xyz_qgis/models/connection.py | 11 +++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/iml/network/net_handler.py b/XYZHubConnector/xyz_qgis/iml/network/net_handler.py index 15fde52..21708a5 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/net_handler.py +++ b/XYZHubConnector/xyz_qgis/iml/network/net_handler.py @@ -78,7 +78,13 @@ def on_received_impl(cls, response): items = obj["results"]["items"] # aggregate layers from different catalog lst_layer_meta = [ - dict(catalog=it.get("id", ""), catalog_hrn=it.get("hrn", ""), **layer) + dict( + catalog=it.get("id", ""), + catalog_hrn=it.get("hrn", ""), + catalog_name=it.get("name", ""), + owner=it.get("owner", ""), + **layer + ) for it in items for layer in it.get("layers", tuple()) if not layerType diff --git a/XYZHubConnector/xyz_qgis/models/connection.py b/XYZHubConnector/xyz_qgis/models/connection.py index b48dd11..02c2a3d 100644 --- a/XYZHubConnector/xyz_qgis/models/connection.py +++ b/XYZHubConnector/xyz_qgis/models/connection.py @@ -130,10 +130,13 @@ def get_platform_server_name(self): self.platform_server_name(self.get_server()) def is_livemap(self): - packages = self.get_("packages", list()) - check_pkg = any(p for p in packages if self.LIVEMAP in p) - check_cid = self.get_("cid") == self.LIVEMAP_CID - return check_pkg or check_cid + if self.is_platform_server(): + return True + else: + packages = self.get_("packages", list()) + check_pkg = any(p for p in packages if self.LIVEMAP in p) + check_cid = self.get_("cid") == self.LIVEMAP_CID + return check_pkg or check_cid def is_platform_server(self): return (self.get_("server") or "").strip().upper() in self.PLATFORM_SERVERS From 1cd50b67c07fda8efb2ce932b303ac92617fde26 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 9 Oct 2023 01:55:25 +0200 Subject: [PATCH 09/42] handle layer destroy gracefully without dangling layer Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- .../xyz_qgis/iml/loader/iml_layer_loader.py | 8 ++++---- XYZHubConnector/xyz_qgis/layer/layer.py | 2 +- XYZHubConnector/xyz_qgis/loader/layer_loader.py | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py b/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py index 195d805..2b03d88 100644 --- a/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py +++ b/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py @@ -97,8 +97,8 @@ def _start(self, **kw): self._refresh_loader(self.network, self) super()._start(**kw) - def post_render(self, *a, **kw): - super().post_render(*a, **kw) + def _post_render(self): + super()._post_render() self._save_conn_info_to_layer(self) @@ -118,8 +118,8 @@ def _start(self, **kw): self._refresh_loader(self.network, self) super()._start(**kw) - def post_render(self, *a, **kw): - super().post_render(*a, **kw) + def _post_render(self): + super()._post_render() self._save_conn_info_to_layer(self) def _retry_with_auth(self, reply): diff --git a/XYZHubConnector/xyz_qgis/layer/layer.py b/XYZHubConnector/xyz_qgis/layer/layer.py index 6298650..0922ebe 100644 --- a/XYZHubConnector/xyz_qgis/layer/layer.py +++ b/XYZHubConnector/xyz_qgis/layer/layer.py @@ -236,7 +236,7 @@ def _cb_delete_vlayer(self, vlayer, geom_str, idx): pass def destroy(self): - self.qgroups.pop("main", None) + # self.qgroups.pop("main", None) # handle destroy gracefully # Delete vlayer in case a it is moved out of the group # thus will not be implicitly deleted diff --git a/XYZHubConnector/xyz_qgis/loader/layer_loader.py b/XYZHubConnector/xyz_qgis/loader/layer_loader.py index 46ace28..d176cc9 100644 --- a/XYZHubConnector/xyz_qgis/loader/layer_loader.py +++ b/XYZHubConnector/xyz_qgis/loader/layer_loader.py @@ -87,6 +87,11 @@ def __init__(self, network: NetManager, layer: XYZLayer = None, n_parallel=1): self._config_layer_callback(layer) def post_render(self, *a, **kw): + if self.is_not_running(): + return + self._post_render() + + def _post_render(self): for v in self.layer.iter_layer(): update_vlayer_editorWidgetSetup(v) v.triggerRepaint() @@ -264,6 +269,8 @@ def _dispatch_render(self, *parsed_feat): def _render_single(self, geom, idx, feat, fields, kw_params): if not feat: return + if self.is_not_running(): + return vlayer = self._create_or_get_vlayer(geom, idx) render.add_feature_render(vlayer, feat, fields) @@ -278,9 +285,12 @@ def destroy(self): self.stop_loading() self.layer.destroy() + def is_not_running(self): + return self.status in [self.FINISHED, self.STOPPED] + def stop_loading(self): """Stop loading immediately""" - if self.status in [self.FINISHED, self.STOPPED]: + if self.is_not_running(): return try: self.status = self.STOPPED @@ -421,13 +431,16 @@ def __init__(self, *a, **kw): self.params_queue = queue.SimpleQueue(key="tile_id") # dont have retry logic def _render_single(self, geom, idx, feat, fields, kw_params): - vlayer = self._create_or_get_vlayer(geom, idx) + if self.is_not_running(): + return tile_id = kw_params.get("tile_id") tile_schema = kw_params.get("tile_schema") lrc = tile_utils.parse_tile_id(tile_id, schema=tile_schema) rcl = [lrc[k] for k in ["row", "col", "level"]] extent = tile_utils.extent_from_row_col(*rcl, schema=tile_schema) + vlayer = self._create_or_get_vlayer(geom, idx) render.clear_features_in_extent(vlayer, QgsRectangle(*extent)) + # if no feat received, only clear the current extent if not feat: return render.add_feature_render(vlayer, feat, fields) From 2356daea8c678650a7cc8d003e90f81349103d53 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 10 Oct 2023 03:26:10 +0200 Subject: [PATCH 10/42] better handle 401 error by increasing max retry Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 2 ++ XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index b8e7da5..e9304ee 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -416,6 +416,7 @@ def cb_handle_error_msg(self, e): return elif isinstance(e0, net_handler.NetworkUnauthorized): # error during list/spaces request + # error during tile request when max retry reached (should not occured) if not self.handle_net_err(e0): self.show_err_msgbar( e0, @@ -440,6 +441,7 @@ def cb_handle_error_msg(self, e): elif isinstance(e0, AuthenticationError): if isinstance(e0.error, net_handler.NetworkError): # network error during layer loader, handled by loader, do not handle here + # e.g. 403 get_project self.show_err_msgbar( e0, "Please select valid HERE Platform credential and try again" ) diff --git a/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py b/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py index 2b03d88..e58d1b5 100644 --- a/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py +++ b/XYZHubConnector/xyz_qgis/iml/loader/iml_layer_loader.py @@ -25,7 +25,7 @@ class IMLAuthExtension: - MAX_RETRY_COUNT = 1 + MAX_RETRY_COUNT = 2 def __init__(self, network, *a, **kw): # setup retry with reauth @@ -51,6 +51,7 @@ def _handle_error(self, err): else: e = chain_err if isinstance(e, NetworkError): # retry only when network error, not timeout + # print(e, self._retry_cnt) response = e.get_response() status = response.get_status() reply = response.get_reply() From bb4db8c9baae44b535e16a9193e2bb2147fa45b8 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 10 Oct 2023 04:40:11 +0200 Subject: [PATCH 11/42] bump version 1.9.9 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- CHANGELOG.md | 8 ++ RELEASENOTE.md | 7 ++ XYZHubConnector/__init__.py | 2 +- XYZHubConnector/metadata.txt | 12 +-- XYZHubConnector/xyz_qgis/__init__.py | 2 +- XYZHubConnector/xyz_qgis/gui/qml/web.js | 105 ++++++++++++++++++++ XYZHubConnector/xyz_qgis/gui/ux/space_ux.py | 1 + 7 files changed, 128 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b525112..18ab555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Version 1.9.9 (2023-10-10) + +#### Bug Fixes + +* Updated Platform IML server +* Set single layering mode as default +* Improved stability + ## Version 1.9.8 (2023-07-24) #### Bug Fixes diff --git a/RELEASENOTE.md b/RELEASENOTE.md index ec51813..03d5638 100644 --- a/RELEASENOTE.md +++ b/RELEASENOTE.md @@ -1,5 +1,12 @@ # Release Notes +## Version 1.9.9 (2023-10-10) + +🐛 FIXES 🐛 +* Updated Platform IML server +* Set single layering mode as default +* Improved stability + ## Version 1.9.8 (2023-07-24) 🐛 FIXES 🐛 diff --git a/XYZHubConnector/__init__.py b/XYZHubConnector/__init__.py index 589d2f0..4e54b43 100644 --- a/XYZHubConnector/__init__.py +++ b/XYZHubConnector/__init__.py @@ -12,7 +12,7 @@ __copyright__ = "Copyright 2019, HERE Europe B.V." __license__ = "MIT" -__version__ = "1.9.8" +__version__ = "1.9.9" __maintainer__ = "Minh Nguyen" __email__ = "huyminh.nguyen@here.com" __status__ = "Development" diff --git a/XYZHubConnector/metadata.txt b/XYZHubConnector/metadata.txt index 67cf2ee..37f0b06 100644 --- a/XYZHubConnector/metadata.txt +++ b/XYZHubConnector/metadata.txt @@ -11,7 +11,7 @@ name=HERE Maps for QGIS qgisMinimumVersion=3.0 description=Connect QGIS to Interactive Map Layers in the HERE Platform (https://platform.here.com) and your personal spaces in HERE Data Hub. -version=1.9.8 +version=1.9.9 author=HERE Europe B.V. email=huyminh.nguyen@here.com @@ -42,13 +42,11 @@ experimental=False # deprecated flag (applies to the whole plugin, not just a single version) deprecated=False -changelog=Version 1.9.8 (2023-07-24) +changelog=Version 1.9.9 (2023-10-10) 🐛 FIXES 🐛 - * Supports Here Platform login for MacOS - * Fixes issues with login and expired token - * Do not store Here Platform email and token into project files - * Fixes issue that some features are not displayed - * Improves UX and stability + * Updated Platform IML server + * Set single layering mode as default + * Improved stability * .. more details on Github repos \ No newline at end of file diff --git a/XYZHubConnector/xyz_qgis/__init__.py b/XYZHubConnector/xyz_qgis/__init__.py index 08384e4..c1474f9 100644 --- a/XYZHubConnector/xyz_qgis/__init__.py +++ b/XYZHubConnector/xyz_qgis/__init__.py @@ -12,7 +12,7 @@ __copyright__ = "Copyright 2019, HERE Europe B.V." __license__ = "MIT" -__version__ = "1.9.8" +__version__ = "1.9.9" __maintainer__ = "Minh Nguyen" __email__ = "huyminh.nguyen@here.com" __status__ = "Development" diff --git a/XYZHubConnector/xyz_qgis/gui/qml/web.js b/XYZHubConnector/xyz_qgis/gui/qml/web.js index 6c615af..a5fd43b 100644 --- a/XYZHubConnector/xyz_qgis/gui/qml/web.js +++ b/XYZHubConnector/xyz_qgis/gui/qml/web.js @@ -34,3 +34,108 @@ function hasTokenSync(baseUrl="https://platform.here.com") { return {error: error} } } + +//debugConsole(txt) { +// console.log(txt) +//} +// +//attemptRefreshToken(baseUrl="https://platform.here.com") { +// debugConsole('Inside attemptRefreshToken()'); +// // Start timeout to catch silent Auth failure +// startAttemptingRefreshTokenTimeout(); +// +// //Create hidden iframe of page that calls service.refreshToken +// +// const refreshIframe = document.createElement('iframe'); +// const iFrameSrc = baseUrl + "/refreshToken"; +// +// refreshIframe.src = iFrameSrc; +// refreshIframe.setAttribute('id', 'refresh-access-token'); +// refreshIframe.setAttribute('style', 'display:none; visibility:hidden;'); +// refreshIframe.setAttribute('loading', 'lazy'); +// +// debugConsole('adding iframe with src=', iFrameSrc); +// document.body.appendChild(refreshIframe); +// +// const iFrameEventPromise = new Promise(resolve => { +// window.addEventListener('message', function processMessage(e) { +// console.debug('event received from iframe', e); +// if (e.origin !== new URL(window.location.href).origin) { +// console.debug('origins did not match'); +// resolve(null); +// } +// +// if (e.data.message === 'refreshAccessToken') { +// // Remove iframe from DOM +// console.debug('removing iframe'); +// const iframeToClose = document.getElementById('refresh-access-token'); +// +// if (iframeToClose) { +// iframeToClose.remove(); +// } +// +// // Remove listener +// window.removeEventListener('message', processMessage); +// +// // Handle message +// if (e.data.accessToken) { +// console.debug('received new accessToken from iframe data'); +// resolve(e.data); +// } +// resolve(null); +// } +// }); +// }) +// .then(response => { +// debugConsole('calling stop attempt token refresh timeout'); +// this.stopAttemptingRefreshTokenTimeout(); +// if (response) { +// const accessTokenResponse = { +// accessToken: response.accessToken, +// accessTokenExpires: response.accessTokenExpires, +// refreshTokenExpires: response.refreshTokenExpires +// }; +// // +// this.handleAccessTokenSuccess(accessTokenResponse); +// +// return this.handleRefreshTokenSuccess(accessTokenResponse); +// } +// //eslint-disable-next-line +// throw ('error with refreshing token iframe sent the response: ', response); +// }) +// .catch(error => this.handleHAError(error)); +// +// return iFrameEventPromise; +// } +// +// +//startAttemptingRefreshTokenTimeout() { +// console.debug('Attempting token refresh Timeout '); +// +// if (this.timer) { +// console.debug('Attempted to start attempt refresh token timeout, but it is already in progress.'); +// return; +// } +// +// this.timer = setTimeout(() => { +// // If we don't hear back from a successful refresh after the refreshTokenFailureTimeout time, handle passive auth failure +// this.handlePassiveAuthIntervalFailure(); +// console.debug( +// `pass auth interval failure, timing out in ${this.hereAccountWebApiOptions.refreshTokenFailureTimeout} seconds` +// ); +// }, this.hereAccountWebApiOptions.refreshTokenFailureTimeout * 1000); +// debugConsole( +// `setting timeout for ${this.hereAccountWebApiOptions.refreshTokenFailureTimeout} sec, timerId==`, +// this.timer +// ); +// } +// +// stopAttemptingRefreshTokenTimeout() { +// this.debugConsole('inside stopAttemptingRefreshTokenTimeout() and timerId==', this.timer); +// if (this.timer) { +// console.debug('stopping to attempt token refresh timeout'); +// clearTimeout(this.timer); +// this.timer = null; +// } +// } + diff --git a/XYZHubConnector/xyz_qgis/gui/ux/space_ux.py b/XYZHubConnector/xyz_qgis/gui/ux/space_ux.py index b378b2a..600f878 100644 --- a/XYZHubConnector/xyz_qgis/gui/ux/space_ux.py +++ b/XYZHubConnector/xyz_qgis/gui/ux/space_ux.py @@ -122,6 +122,7 @@ def cb_display_spaces(self, conn_info, obj, *a, **kw): if obj is None: return lst_conn_info = list() + print(obj) for meta in obj: conn_info = SpaceConnectionInfo(self.conn_info) conn_info.set_(**meta) From 070c0bbb1915522d1f57bbe86583a681cdbacd51 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:37:11 +0200 Subject: [PATCH 12/42] try to raise rendering error Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/render.py | 60 +++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/layer/render.py b/XYZHubConnector/xyz_qgis/layer/render.py index 67b4f53..00af5f6 100644 --- a/XYZHubConnector/xyz_qgis/layer/render.py +++ b/XYZHubConnector/xyz_qgis/layer/render.py @@ -12,6 +12,8 @@ QgsWkbTypes, QgsFeatureRequest, QgsCoordinateReferenceSystem, + QgsFields, + QgsFeature, ) from qgis.utils import iface @@ -21,6 +23,56 @@ print_qgis = make_print_qgis("render") +class RenderError(Exception): + pass + + +class RenderFieldsError(RenderError): + def __init__(self, vlayer_name: str, fields: QgsFields, *a, **kw): + super().__init__(vlayer_name, fields, *a, **kw) + + def get_vlayer_name(self): + return self.args[0] + + def get_fields(self): + return self.args[1] + + def __repr__(self): + fields = self.get_fields() + return "{classname}(vlayer={vlayer_name}, cnt={cnt}, field_names={field_names})".format( + classname=self.__class__.__name__, + vlayer_name=self.get_vlayer_name(), + cnt=len(fields), + field_names=fields.names(), + ) + + +class RenderFeaturesError(RenderError): + def __init__(self, vlayer_name: str, features: list[QgsFeature], *a, **kw): + super().__init__(vlayer_name, features, *a, **kw) + + def get_vlayer_name(self): + return self.args[0] + + def get_features(self): + return self.args[1] + + def __repr__(self): + features = self.get_features() + field_names = features[0].fields().names() if len(features) else list() + + return ( + "{classname}(vlayer={vlayer_name}, cnt={cnt}, field_cnt={field_cnt}, " + "field_names={field_names})".format( + classname=self.__class__.__name__, + vlayer_name=self.get_vlayer_name(), + cnt=len(features), + field_cnt=len(field_names), + field_names=field_names, + ) + ) + + # mixed-geom def parse_feature(obj, map_fields, similarity_threshold=None, **kw_params): map_feat, map_fields = parser.xyz_json_to_feature_map(obj, map_fields, similarity_threshold) @@ -83,7 +135,10 @@ def add_feature_render(vlayer, feat, new_fields): # reset fid value (deprecated thanks to unique field name) # for i,f in enumerate(feat): f.setAttribute(parser.QGS_ID,None) - pr.addAttributes(diff_fields) + attribute_ok = pr.addAttributes(diff_fields) + if not attribute_ok: + raise RenderFieldsError(vlayer.name(), diff_fields) + vlayer.updateFields() # update feature fields according to provider fields @@ -91,6 +146,9 @@ def add_feature_render(vlayer, feat, new_fields): feat = filter(None, (parser.update_feature_fields(ft, pr.fields()) for ft in feat if ft)) ok, out_feat = pr.addFeatures(feat) + if not ok: + raise RenderFeaturesError(vlayer.name(), out_feat) + vlayer.updateExtents() # will hide default progress bar # post_render(vlayer) # disable in order to keep default progress bar running # vlayer.reload() # comment out to have less crash when loading large geometries From 0f6a738810440a5ba390bb94933c1c69c607830d Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 10 Oct 2023 01:49:28 +0200 Subject: [PATCH 13/42] slow: try to handle OGR error due to mismatch fields Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/parser.py | 4 +++- XYZHubConnector/xyz_qgis/layer/render.py | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/parser.py b/XYZHubConnector/xyz_qgis/layer/parser.py index 4cb5ff2..b602141 100644 --- a/XYZHubConnector/xyz_qgis/layer/parser.py +++ b/XYZHubConnector/xyz_qgis/layer/parser.py @@ -451,7 +451,9 @@ def update_feature_fields(feat: QgsFeature, fields: QgsFields): try: assert set(fields.names()).issuperset( set(old_fields.names()) - ), "new fields must be a super set of existing fields of feature" + ), "new fields must be a super set of existing fields of feature.\nnew: {}\nold: {}".format( + fields.names(), old_fields.names() + ) except AssertionError as e: print_error(e) return diff --git a/XYZHubConnector/xyz_qgis/layer/render.py b/XYZHubConnector/xyz_qgis/layer/render.py index 00af5f6..c1fddd0 100644 --- a/XYZHubConnector/xyz_qgis/layer/render.py +++ b/XYZHubConnector/xyz_qgis/layer/render.py @@ -122,7 +122,7 @@ def add_feature_render(vlayer, feat, new_fields): feat = filter(None, (parser.transform_geom(ft, transformer) for ft in feat if ft)) names = set(pr.fields().names()) - assert parser.check_non_expression_fields(new_fields) + assert parser.check_non_expression_fields(new_fields) # precondition diff_fields = [f for f in new_fields if not f.name() in names] # print_qgis(len(names), names) @@ -141,9 +141,10 @@ def add_feature_render(vlayer, feat, new_fields): vlayer.updateFields() - # update feature fields according to provider fields - if not parser.check_same_fields(new_fields, pr.fields()): - feat = filter(None, (parser.update_feature_fields(ft, pr.fields()) for ft in feat if ft)) + # assert parser.check_same_fields(new_fields, pr.fields()) # validate addAttributes + + # always update feature fields according to provider fields + feat = filter(None, (parser.update_feature_fields(ft, pr.fields()) for ft in feat if ft)) ok, out_feat = pr.addFeatures(feat) if not ok: From e68b392cb973b81c4304a47aa937948f5026012d Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 10 Oct 2023 03:27:23 +0200 Subject: [PATCH 14/42] debug fields parser Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/parser.py | 12 +++++++----- XYZHubConnector/xyz_qgis/layer/render.py | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/parser.py b/XYZHubConnector/xyz_qgis/layer/parser.py index b602141..f642228 100644 --- a/XYZHubConnector/xyz_qgis/layer/parser.py +++ b/XYZHubConnector/xyz_qgis/layer/parser.py @@ -439,7 +439,7 @@ def check_same_fields(fields1: QgsFields, fields2: QgsFields): return len_ok and name_ok and field_origin_ok -def update_feature_fields(feat: QgsFeature, fields: QgsFields): +def update_feature_fields(feat: QgsFeature, fields: QgsFields, ref: QgsFields): """ Update fields of feature and its data (QgsAttributes) @@ -448,11 +448,13 @@ def update_feature_fields(feat: QgsFeature, fields: QgsFields): :return: new QgsFeature with updated fields """ old_fields = feat.fields() + names, old_names = fields.names(), old_fields.names() try: - assert set(fields.names()).issuperset( - set(old_fields.names()) - ), "new fields must be a super set of existing fields of feature.\nnew: {}\nold: {}".format( - fields.names(), old_fields.names() + assert set(names).issuperset(set(old_names)), ( + "new fields must be a super set of existing fields of feature.\n" + + "new: {} {}\nold: {} {}\nref: {} {}".format( + len(names), names, len(old_names), old_names, len(ref.names()), ref.names() + ) ) except AssertionError as e: print_error(e) diff --git a/XYZHubConnector/xyz_qgis/layer/render.py b/XYZHubConnector/xyz_qgis/layer/render.py index c1fddd0..2c61643 100644 --- a/XYZHubConnector/xyz_qgis/layer/render.py +++ b/XYZHubConnector/xyz_qgis/layer/render.py @@ -144,7 +144,9 @@ def add_feature_render(vlayer, feat, new_fields): # assert parser.check_same_fields(new_fields, pr.fields()) # validate addAttributes # always update feature fields according to provider fields - feat = filter(None, (parser.update_feature_fields(ft, pr.fields()) for ft in feat if ft)) + feat = filter( + None, (parser.update_feature_fields(ft, pr.fields(), new_fields) for ft in feat if ft) + ) ok, out_feat = pr.addFeatures(feat) if not ok: From deaf38e32e2883d209c7f078fdfe1fc400eb743a Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 10 Oct 2023 23:57:46 +0200 Subject: [PATCH 15/42] test: add subtest info for test_render_layer Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/render.py | 3 ++- test/test_render_layer.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/render.py b/XYZHubConnector/xyz_qgis/layer/render.py index 2c61643..c73d6c1 100644 --- a/XYZHubConnector/xyz_qgis/layer/render.py +++ b/XYZHubConnector/xyz_qgis/layer/render.py @@ -7,6 +7,7 @@ # License-Filename: LICENSE # ############################################################################### +from typing import List from qgis.core import ( QgsWkbTypes, @@ -48,7 +49,7 @@ def __repr__(self): class RenderFeaturesError(RenderError): - def __init__(self, vlayer_name: str, features: list[QgsFeature], *a, **kw): + def __init__(self, vlayer_name: str, features: List[QgsFeature], *a, **kw): super().__init__(vlayer_name, features, *a, **kw) def get_vlayer_name(self): diff --git a/test/test_render_layer.py b/test/test_render_layer.py index 529609a..9750cd2 100644 --- a/test/test_render_layer.py +++ b/test/test_render_layer.py @@ -9,7 +9,6 @@ import json import random -import numpy as np import pprint from test.utils import ( @@ -30,6 +29,8 @@ # import unittest # class TestParser(BaseTestAsync, unittest.TestCase): + + class TestRenderLayer(BaseTestAsync): _assert_len_map_fields = test_parser.TestParser._assert_len_map_fields @@ -72,7 +73,7 @@ def subtest_render_mixed_json_to_layer(self, folder, fname, ref_len_fields=None) def subtest_render_mixed_json_to_layer_multi_chunk(self, obj, ref, lst_chunk_size=None): if not lst_chunk_size: p10 = 1 + len(str(len(obj["features"]))) - lst_chunk_size = [10 ** i for i in range(p10)] + lst_chunk_size = [10**i for i in range(p10)] with self.subTest(lst_chunk_size=lst_chunk_size): lst_map_fields = list() for chunk_size in lst_chunk_size: @@ -91,11 +92,12 @@ def subtest_render_mixed_json_to_layer_shuffle(self, obj, ref, n_shuffle=5, chun lst_map_fields = list() random.seed(0.5) for i in range(n_shuffle): - random.shuffle(o["features"]) - map_fields = self.subtest_render_mixed_json_to_layer_chunk(o, chunk_size) - if map_fields is None: - continue - lst_map_fields.append(map_fields) + with self.subTest(shuffle=i): + random.shuffle(o["features"]) + map_fields = self.subtest_render_mixed_json_to_layer_chunk(o, chunk_size) + if map_fields is None: + continue + lst_map_fields.append(map_fields) for i, map_fields in enumerate(lst_map_fields): with self.subTest(shuffle=i): @@ -127,7 +129,7 @@ def subtest_render_mixed_json_to_layer_chunk(self, obj, chunk_size=100, empty_ch lst_chunk.insert(i, list()) for chunk in lst_chunk: o["features"] = chunk - map_feat, _ = parser.xyz_json_to_feature_map(o, map_fields) + map_feat, _ = parser.xyz_json_to_feature_map(o, map_fields, similarity_threshold=0) test_parser.TestParser()._assert_parsed_map(chunk, map_feat, map_fields) lst_map_feat.append(map_feat) self._render_layer(layer, map_feat, map_fields) From b16983e3af20bf974df570f7d1e9f81e5ee6c96c Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Wed, 11 Oct 2023 00:05:11 +0200 Subject: [PATCH 16/42] fix parser This reverts commit c4e901f07fbf8151cc5213f7532e59a3083f7ea4. --- XYZHubConnector/xyz_qgis/layer/parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/parser.py b/XYZHubConnector/xyz_qgis/layer/parser.py index f642228..36b2e7e 100644 --- a/XYZHubConnector/xyz_qgis/layer/parser.py +++ b/XYZHubConnector/xyz_qgis/layer/parser.py @@ -523,31 +523,32 @@ def xyz_json_to_feature_map( 100: map_fields should have as many as possible fields/geom """ - def _single_feature_map(feat_json, map_feat, map_fields): + def _single_feature_map(feat_json, map_feat_, map_fields_): geom = feat_json.get("geometry") g = geom["type"] if geom is not None else None # # promote to multi geom # if g is not None and not g.startswith("Multi"): g = "Multi" + g - lst_fields = map_fields.setdefault(g, list()) + lst_fields = map_fields_.setdefault(g, list()) fields, idx = prepare_fields(feat_json, lst_fields, similarity_threshold) - ft = xyz_json_to_feature(feat_json, fields) + feat = xyz_json_to_feature(feat_json, fields) + lst_fields[idx] = feat.fields() + # FIX: as fields is modified during processing, reassign it to lst_fields - lst = map_feat.setdefault(g, list()) + lst_feat = map_feat_.setdefault(g, list()) + while len(lst_feat) < len(lst_fields): + lst_feat.append(list()) + lst_feat[idx].append(feat) - while len(lst) < len(lst_fields): - lst.append(list()) - lst[idx].append(ft) - - lst_feat = obj["features"] + lst_all_feat = obj["features"] if map_fields is None: map_fields = dict() # map_feat = dict() map_feat = dict((k, [list() for _ in enumerate(v)]) for k, v in map_fields.items()) - for ft in lst_feat: + for ft in lst_all_feat: _single_feature_map(ft, map_feat, map_fields) return map_feat, map_fields From 64b5677f25d4ec5ffb3a4bcb7e4c41e20b84d1dd Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:55:35 +0200 Subject: [PATCH 17/42] update test util Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- test/utils.py | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/test/utils.py b/test/utils.py index 560ff6b..054d7c1 100644 --- a/test/utils.py +++ b/test/utils.py @@ -7,6 +7,9 @@ # License-Filename: LICENSE # ############################################################################### +import logging +import random +import string from qgis.PyQt.QtCore import QEventLoop, QThreadPool from qgis.testing import unittest, start_app @@ -21,6 +24,15 @@ import os import pprint +logging.basicConfig( + level=logging.DEBUG, + handlers=[ + logging.StreamHandler(stream=sys.stdout), + # logging.FileHandler(os.path.join("err-log", f"log-{__name__}.log")), + ], +) +logger = logging.getLogger("TEST") + def get_env(scope, keys): """usage: get_env(locals(), ["APP_ID", "APP_CODE"]) #globals()""" @@ -105,7 +117,14 @@ def _make_async_fun(self, fun): return AsyncFun(fun) def _log_error(self, *a, **kw): - print(*a, file=sys.stderr, **kw) + # print(*a, file=sys.stderr, **kw) + errorId = random_id(8) + errorIdMsg = "errorId: {}".format(errorId) + fullMsg = " ".join(filter(None, [errorIdMsg, *a])) + fn_name = "{}:".format(self._id()) + msg = format_long_args(fn_name, errorIdMsg, *a, limit=1000) + logger.error(msg, **kw) + return fullMsg def _log_info(self, *a, **kw): log_truncate(*a, **kw) @@ -113,7 +132,7 @@ def _log_info(self, *a, **kw): def _log_debug(self, *a, **kw): fn_name = "{}:".format(self._id()) # fn_name = sys._getframe(1).f_code.co_name - log_truncate(fn_name, *a, **kw) + log_truncate(fn_name, *a, limit=1000, **kw) def _id(self): return self._subtest.id() if self._subtest else self.id() @@ -160,11 +179,11 @@ def format_long_args(*a, limit=200): return " ".join(str(i)[:limit] for i in a) -def log_truncate(*a, **kw): +def log_truncate(*a, limit=200, **kw): # return - # print(*a,**kw) - s = format_long_args(*a) - print(s, **kw) + # print(*a, **kw) + s = format_long_args(*a, limit=limit) + logger.info(s, **kw) def log_truncate_timestamp(*a, **kw): @@ -354,3 +373,15 @@ def get_conn_info(key): if token is not None: conn_info.set_(token=token, space_id=space_id, server=server) return conn_info + + +def random_id(N, punctuation=None): + return "".join( + random.choices( + string.ascii_uppercase + + string.ascii_lowercase + + string.digits + + (string.punctuation if punctuation else ""), + k=N, + ) + ) From 5ce57630a6b97ba0f685954813b36a9ac5aaf674 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:26:40 +0200 Subject: [PATCH 18/42] fixed test parser Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- test/test_parser.py | 130 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 29 deletions(-) diff --git a/test/test_parser.py b/test/test_parser.py index dbf2dc3..e0bca0b 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -21,7 +21,7 @@ format_map_fields, ) -from qgis.core import QgsFields, QgsVectorLayer +from qgis.core import QgsFields, QgsVectorLayer, QgsWkbTypes from qgis.testing import unittest from XYZHubConnector.xyz_qgis.layer import parser @@ -31,7 +31,23 @@ class TestParser(BaseTestAsync): def __init__(self, *a, **kw): super().__init__(*a, **kw) - self.similarity_threshold = 80 + self.similarity_threshold = 0 + self.mixed_case_duplicate = False + + # util for debug + def assertEqual(self, first, second, msg=None): + if first != second: + msg = self._log_error(msg) + super().assertEqual(first, second, msg) + + def assertTrue(self, expr, msg=None): + if not expr: + msg = self._log_error(msg) + super().assertTrue(expr, msg) + + def fail(self, msg): + msg = self._log_error(msg) + return super().fail(msg) # ######## Parse xyz geojson -> QgsFeature def test_parse_xyzjson(self): @@ -50,9 +66,14 @@ def subtest_parse_xyzjson(self, folder, fname): obj_feat = obj["features"] fields = QgsFields() feat = [parser.xyz_json_to_feature(ft, fields) for ft in obj_feat] + o1 = obj_feat[0] if len(obj_feat) else None + geom_str = o1 and o1["geometry"] and o1["geometry"]["type"] + self._assert_parsed_id(obj_feat, feat, fields) + self._assert_parsed_fields_unorder(obj_feat, feat, fields) self._assert_parsed_fields(obj_feat, feat, fields) - self._assert_parsed_geom(obj_feat, feat, fields) + self._assert_parsed_geom_unorder(obj_feat, feat, fields, geom_str) + self._assert_parsed_geom(obj_feat, feat, fields, geom_str) return feat def _assert_parsed_fields_unorder(self, obj_feat, feat, fields): @@ -66,13 +87,12 @@ def _assert_parsed_fields_unorder(self, obj_feat, feat, fields): self.assertEqual(len(obj_feat), len(feat)) def _assert_parsed_fields(self, obj_feat, feat, fields): - self._assert_parsed_fields_unorder(obj_feat, feat, fields) - def msg_fields(obj): return ( - "{sep}{0}{sep}{1}" - "{sep}fields-props {2}" - "{sep}props-fields {3}" + "{sep}props {0}" + "{sep}fields {1}" + "{sep}props-fields {2} (should be 0)" + "{sep}fields-props {3}" "{sep}json {4}".format( *tuple( map( @@ -80,8 +100,8 @@ def msg_fields(obj): [ obj_props, fields.names(), - set(fields.names()).difference(obj_props), set(obj_props).difference(fields.names()), + set(fields.names()).difference(obj_props), ], ) ), @@ -92,19 +112,46 @@ def msg_fields(obj): for o in obj_feat: obj_props = list(o["properties"].keys()) + obj_props_non_null = [k for k, v in o["properties"].items() if v is not None] self.assertLessEqual(len(obj_props), fields.size(), msg_fields(o)) - self.assertTrue(set(obj_props) < set(fields.names()), msg_fields(o)) + # self._log_debug(msg_fields(o).replace(">>", "++")) + + obj_props_is_subset_of_fields = any( + [ + set(obj_props) < set(fields.names()), + set(obj_props_non_null) < set(fields.names()), + ] + ) + self.assertTrue(obj_props_is_subset_of_fields, msg_fields(o)) # self.assertEqual( obj_props, fields.names(), msg_fields(o)) # strict assert + def wkb_type_to_wkt_str(self, typ): + return QgsWkbTypes.displayString(typ) + + def wkb_type_to_geom_str(self, typ): + return QgsWkbTypes.displayString(typ % 1000) if typ else None + + def wkb_type_to_geom_display_str(self, typ): + return ( + QgsWkbTypes.geometryDisplayString(QgsWkbTypes.geometryType(typ)) + if typ + else "No geometry" + ) + def _assert_parsed_geom_unorder(self, obj_feat, feat, fields, geom_str): + wkt_ref = self.wkb_type_to_wkt_str(QgsWkbTypes.parseType(geom_str)) for ft in feat: geom = json.loads( ft.geometry().asJson() ) # limited to 13 or 14 precison (ogr.CreateGeometryFromJson) self.assertEqual(geom and geom["type"], geom_str) - def _assert_parsed_geom(self, obj_feat, feat, fields): + geom_str_ft = self.wkb_type_to_geom_str(ft.geometry().wkbType()) + wkt_ft = self.wkb_type_to_wkt_str(ft.geometry().wkbType()) + msg = "wkt string {} != {}".format(wkt_ft, wkt_ref) + self.assertEqual(geom_str_ft, geom_str, msg) + def _assert_parsed_geom(self, obj_feat, feat, fields, geom_str): # both crs is WGS84 for o, ft in zip(obj_feat, feat): geom = json.loads( @@ -112,11 +159,9 @@ def _assert_parsed_geom(self, obj_feat, feat, fields): ) # limited to 13 or 14 precison (ogr.CreateGeometryFromJson) obj_geom = o["geometry"] - self.assertEqual(geom["type"], obj_geom["type"]) - - id_ = ft.attribute(parser.QGS_XYZ_ID) - obj_id_ = o["id"] - self.assertEqual(id_, obj_id_) + self.assertEqual(geom and geom["type"], obj_geom and obj_geom["type"]) + if not geom or not obj_geom: + continue # self._log_debug(geom) # self._log_debug(obj_geom) @@ -130,6 +175,8 @@ def _assert_parsed_geom(self, obj_feat, feat, fields): c1 = np.array(obj_geom["coordinates"]) c2 = np.array(geom["coordinates"]) + c1_flatten = np.array(flatten(obj_geom["coordinates"])) + c2_flatten = np.array(flatten(geom["coordinates"])) if c1.shape != c2.shape: self._log_debug( "\nWARNING: Geometry has mismatch shape", @@ -137,10 +184,23 @@ def _assert_parsed_geom(self, obj_feat, feat, fields): c2.shape, "\nOriginal geom has problem. Testing parsed geom..", ) - self.assertEqual(c2.shape[-1], 2, "parsed geom has wrong shape of coord") + msg = ( + "parsed geom has wrong shape of coord for geom {geom}. {} != {}".format( + c1.shape, c2.shape, geom=geom["type"] + ), + ) + self.assertEqual(c2.shape[-1], 2, msg) continue else: - self.assertLess(np.max(np.abs(c1 - c2)), 1e-13, "parsed geometry error > 1e-13") + self.assertLess( + np.max(np.abs(c1_flatten - c2_flatten)), 1e-13, "parsed geometry error > 1e-13" + ) + + def _assert_parsed_id(self, obj_feat, feat, fields): + for o, ft in zip(obj_feat, feat): + id_ = ft.attribute(parser.QGS_XYZ_ID) + obj_id_ = o["id"] + self.assertEqual(id_, obj_id_) # @unittest.skip("large") def test_parse_xyzjson_large(self): @@ -187,6 +247,7 @@ def test_parse_xyzjson_map_similarity_0(self): self.similarity_threshold = s def test_parse_xyzjson_map_dupe_case(self): + self.mixed_case_duplicate = True folder = "xyzjson-small" fnames = [ "airport-xyz.geojson", @@ -302,7 +363,7 @@ def subtest_parse_xyzjson_mix(self, folder, fnames): def subtest_parse_xyzjson_map_multi_chunk(self, obj, lst_chunk_size=None): if not lst_chunk_size: p10 = 1 + len(str(len(obj["features"]))) - lst_chunk_size = [10 ** i for i in range(p10)] + lst_chunk_size = [10**i for i in range(p10)] with self.subTest(lst_chunk_size=lst_chunk_size): ref_map_feat, ref_map_fields = self.do_test_parse_xyzjson_map(obj) lst_map_fields = list() @@ -323,13 +384,14 @@ def subtest_parse_xyzjson_map_shuffle(self, obj, n_shuffle=5, chunk_size=10): lst_map_fields = list() random.seed(0.5) for i in range(n_shuffle): - random.shuffle(o["features"]) - map_fields = self.subtest_parse_xyzjson_map_chunk(o, chunk_size) - if map_fields is None: - continue - lst_map_fields.append(map_fields) + with self.subTest(shuffle=i): + random.shuffle(o["features"]) + map_fields = self.subtest_parse_xyzjson_map_chunk(o, chunk_size) + if map_fields is None: + continue + lst_map_fields.append(map_fields) - # self._log_debug("parsed fields shuffle", len_of_struct(map_fields)) + # self._log_debug("parsed fields shuffle", len_of_struct(map_fields)) for i, map_fields in enumerate(lst_map_fields): with self.subTest(shuffle=i): @@ -339,7 +401,7 @@ def subtest_parse_xyzjson_map_chunk(self, obj, chunk_size=100): similarity_threshold = self.similarity_threshold with self.subTest(chunk_size=chunk_size, similarity_threshold=similarity_threshold): o = dict(obj) - obj_feat = obj["features"] + obj_feat = list(obj["features"]) lst_map_feat = list() map_fields = dict() for i0 in range(0, len(obj_feat), chunk_size): @@ -397,12 +459,22 @@ def _assert_parsed_map(self, obj_feat, map_feat, map_fields): # NOTE: obj_feat order does not corresponds to that of map_feat # -> use unorder assert + # NOTE: group obj_feat by geom_str for element-wise assert for geom_str in map_feat: for feat, fields in zip(map_feat[geom_str], map_fields[geom_str]): - o = obj_feat[: len(feat)] - self._assert_parsed_fields_unorder(o, feat, fields) + o = [ + o1 + for o1 in obj_feat + if (o1["geometry"] and o1["geometry"]["type"]) == geom_str + ] + self._assert_parsed_id(o, feat, fields) self._assert_parsed_geom_unorder(o, feat, fields, geom_str) - obj_feat = obj_feat[len(feat) :] + self._assert_parsed_fields_unorder(o, feat, fields) + + if not self.mixed_case_duplicate: + # element-wise assert + self._assert_parsed_geom(o, feat, fields, geom_str) + self._assert_parsed_fields(o, feat, fields) def _assert_len_map_feat_fields(self, map_feat, map_fields): self.assertEqual(map_feat.keys(), map_fields.keys()) From f769599ef8a120e70e018a838d4b1d908d72861f Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:46:33 +0200 Subject: [PATCH 19/42] refactor test parser Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- test/test_parser.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/test/test_parser.py b/test/test_parser.py index e0bca0b..b2d3471 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -69,13 +69,20 @@ def subtest_parse_xyzjson(self, folder, fname): o1 = obj_feat[0] if len(obj_feat) else None geom_str = o1 and o1["geometry"] and o1["geometry"]["type"] - self._assert_parsed_id(obj_feat, feat, fields) + self._assert_parsed_feat(obj_feat, feat) self._assert_parsed_fields_unorder(obj_feat, feat, fields) self._assert_parsed_fields(obj_feat, feat, fields) self._assert_parsed_geom_unorder(obj_feat, feat, fields, geom_str) self._assert_parsed_geom(obj_feat, feat, fields, geom_str) return feat + def _assert_parsed_feat(self, obj_feat, feat): + self.assertEqual(len(obj_feat), len(feat)) + for o, ft in zip(obj_feat, feat): + id_ = ft.attribute(parser.QGS_XYZ_ID) + obj_id_ = o["id"] + self.assertEqual(id_, obj_id_) + def _assert_parsed_fields_unorder(self, obj_feat, feat, fields): # self._log_debug(fields.names()) # self._log_debug("debug id, json vs. QgsFeature") @@ -84,7 +91,6 @@ def _assert_parsed_fields_unorder(self, obj_feat, feat, fields): names = fields.names() self.assertTrue(parser.QGS_XYZ_ID in names, "%s %s" % (len(names), names)) - self.assertEqual(len(obj_feat), len(feat)) def _assert_parsed_fields(self, obj_feat, feat, fields): def msg_fields(obj): @@ -196,12 +202,6 @@ def _assert_parsed_geom(self, obj_feat, feat, fields, geom_str): np.max(np.abs(c1_flatten - c2_flatten)), 1e-13, "parsed geometry error > 1e-13" ) - def _assert_parsed_id(self, obj_feat, feat, fields): - for o, ft in zip(obj_feat, feat): - id_ = ft.attribute(parser.QGS_XYZ_ID) - obj_id_ = o["id"] - self.assertEqual(id_, obj_id_) - # @unittest.skip("large") def test_parse_xyzjson_large(self): folder = "xyzjson-large" @@ -461,20 +461,19 @@ def _assert_parsed_map(self, obj_feat, map_feat, map_fields): # -> use unorder assert # NOTE: group obj_feat by geom_str for element-wise assert for geom_str in map_feat: + obj_feat_by_geom = [ + o for o in obj_feat if (o["geometry"] and o["geometry"]["type"]) == geom_str + ] + feat_by_geom = sum(map_feat[geom_str], []) + self._assert_parsed_feat(obj_feat_by_geom, feat_by_geom) for feat, fields in zip(map_feat[geom_str], map_fields[geom_str]): - o = [ - o1 - for o1 in obj_feat - if (o1["geometry"] and o1["geometry"]["type"]) == geom_str - ] - self._assert_parsed_id(o, feat, fields) - self._assert_parsed_geom_unorder(o, feat, fields, geom_str) - self._assert_parsed_fields_unorder(o, feat, fields) + self._assert_parsed_geom_unorder(obj_feat_by_geom, feat, fields, geom_str) + self._assert_parsed_fields_unorder(obj_feat_by_geom, feat, fields) if not self.mixed_case_duplicate: # element-wise assert - self._assert_parsed_geom(o, feat, fields, geom_str) - self._assert_parsed_fields(o, feat, fields) + self._assert_parsed_geom(obj_feat_by_geom, feat, fields, geom_str) + self._assert_parsed_fields(obj_feat_by_geom, feat, fields) def _assert_len_map_feat_fields(self, map_feat, map_fields): self.assertEqual(map_feat.keys(), map_fields.keys()) From 994b2a1d628f96418474222e03dd6714ccbb21c1 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:31:36 +0200 Subject: [PATCH 20/42] make test_render_layer work for similarity_threshold=0 (single fields per geom) Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- test/test_parser.py | 19 +++++++----- test/test_render_layer.py | 65 +++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/test/test_parser.py b/test/test_parser.py index b2d3471..176999d 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -29,10 +29,12 @@ # import unittest # class TestParser(BaseTestAsync, unittest.TestCase): class TestParser(BaseTestAsync): - def __init__(self, *a, **kw): + def __init__(self, *a, test_parser_kw=None, **kw): super().__init__(*a, **kw) - self.similarity_threshold = 0 - self.mixed_case_duplicate = False + test_parser_kw = test_parser_kw or dict() + self.similarity_threshold = test_parser_kw.get("similarity_threshold", 80) + self.mixed_case_duplicate = test_parser_kw.get("mixed_case_duplicate", False) + self.has_many_fields = test_parser_kw.get("has_many_fields", False) # util for debug def assertEqual(self, first, second, msg=None): @@ -466,13 +468,16 @@ def _assert_parsed_map(self, obj_feat, map_feat, map_fields): ] feat_by_geom = sum(map_feat[geom_str], []) self._assert_parsed_feat(obj_feat_by_geom, feat_by_geom) + self._assert_parsed_geom_unorder(obj_feat_by_geom, feat_by_geom, None, geom_str) + + if not self.has_many_fields: + # element-wise assert + self._assert_parsed_geom(obj_feat_by_geom, feat_by_geom, None, geom_str) + for feat, fields in zip(map_feat[geom_str], map_fields[geom_str]): - self._assert_parsed_geom_unorder(obj_feat_by_geom, feat, fields, geom_str) self._assert_parsed_fields_unorder(obj_feat_by_geom, feat, fields) - - if not self.mixed_case_duplicate: + if not self.mixed_case_duplicate and not self.has_many_fields: # element-wise assert - self._assert_parsed_geom(obj_feat_by_geom, feat, fields, geom_str) self._assert_parsed_fields(obj_feat_by_geom, feat, fields) def _assert_len_map_feat_fields(self, map_feat, map_fields): diff --git a/test/test_render_layer.py b/test/test_render_layer.py index 9750cd2..249ce30 100644 --- a/test/test_render_layer.py +++ b/test/test_render_layer.py @@ -19,6 +19,7 @@ len_of_struct_sorted, flatten, format_map_fields, + add_test_fn_params, ) from test import test_parser from qgis.testing import unittest @@ -38,20 +39,47 @@ def __init__(self, *a, **kw): super().__init__(*a, **kw) test_parser.TestParser._id = lambda x: self._id() + def get_test_parser_kw(self): + kw = dict() + params = self._subtest and self._subtest.params + if "similarity_threshold" in params: + similarity_threshold = params["similarity_threshold"] + kw["similarity_threshold"] = similarity_threshold + if "shuffle" in self._id() or similarity_threshold > 0: + return dict(has_many_fields=True) + return kw + + def get_similarity_threshold_kw(self): + kw = self.get_test_parser_kw() + return ( + dict(similarity_threshold=kw["similarity_threshold"]) + if "similarity_threshold" in kw + else dict() + ) + def test_render_mixed_json_to_layer_chunk(self): folder = "xyzjson-small" fnames = [ "mixed-xyz.geojson", ] for fname in fnames: + self.subtest_render_mixed_json_to_layer( + folder, + fname, + ref_len_fields={None: [4], "MultiPoint": [20], "MultiPolygon": [27]}, + similarity_threshold=0, + ) self.subtest_render_mixed_json_to_layer( folder, fname, ref_len_fields={"MultiPoint": [3, 6, 18], "MultiPolygon": [27], None: [4]}, + similarity_threshold=80, ) - def subtest_render_mixed_json_to_layer(self, folder, fname, ref_len_fields=None): - with self.subTest(folder=folder, fname=fname): + def subtest_render_mixed_json_to_layer( + self, folder, fname, ref_len_fields=None, similarity_threshold=0 + ): + with self.subTest(folder=folder, fname=fname, similarity_threshold=similarity_threshold): resource = TestFolder(folder) txt = resource.load(fname) obj = json.loads(txt) @@ -117,7 +145,7 @@ def subtest_render_mixed_json_to_layer_chunk(self, obj, chunk_size=100, empty_ch with self.subTest(chunk_size=chunk_size): layer = self.new_layer() o = dict(obj) - obj_feat = obj["features"] + obj_feat = list(obj["features"]) lst_map_feat = list() map_fields = dict() lst_chunk: list = [ @@ -129,8 +157,14 @@ def subtest_render_mixed_json_to_layer_chunk(self, obj, chunk_size=100, empty_ch lst_chunk.insert(i, list()) for chunk in lst_chunk: o["features"] = chunk - map_feat, _ = parser.xyz_json_to_feature_map(o, map_fields, similarity_threshold=0) - test_parser.TestParser()._assert_parsed_map(chunk, map_feat, map_fields) + map_feat, _ = parser.xyz_json_to_feature_map( + o, + map_fields, + **self.get_similarity_threshold_kw(), + ) + test_parser.TestParser( + test_parser_kw=self.get_test_parser_kw() + )._assert_parsed_map(chunk, map_feat, map_fields) lst_map_feat.append(map_feat) self._render_layer(layer, map_feat, map_fields) @@ -160,7 +194,9 @@ def _assert_rendered_fields(self, vlayer, fields): def _test_render_mixed_json_to_layer(self, obj): layer = self.new_layer() # map_feat, map_fields = parser.xyz_json_to_feature_map(obj) - map_feat, map_fields = test_parser.TestParser().do_test_parse_xyzjson_map(obj) + map_feat, map_fields = test_parser.TestParser( + test_parser_kw=self.get_test_parser_kw() + ).do_test_parse_xyzjson_map(obj) self._render_layer(layer, map_feat, map_fields) self.assert_layer(layer, obj, map_fields) return map_feat, map_fields @@ -189,17 +225,14 @@ def _assert_len_ref_fields(self, map_fields, ref, strict=False): def _assert_len_fields(self, map_fields, ref, strict=False): len_ = len_of_struct if strict else len_of_struct_sorted - self.assertEqual( - len_(map_fields), - ref, - "\n".join( - [ - "len of map_fields is not correct (vs. ref). " - + "Please revised parser, similarity threshold.", - format_map_fields(map_fields), - ] - ), + msg = "\n".join( + [ + "len of map_fields is not correct (vs. ref). " + + "Please revised parser, similarity threshold.", + format_map_fields(map_fields), + ] ) + self.assertEqual(len_(map_fields), ref, msg) def assert_layer(self, layer, obj, map_fields): lst_vlayer = list(layer.iter_layer()) From 8f64078397117ba665fc4578bfb05c49b0c26448 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:34:14 +0200 Subject: [PATCH 21/42] update Dict type hint Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/gui/platform_auth_dialog.py | 6 ++++-- XYZHubConnector/xyz_qgis/iml/network/network.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/gui/platform_auth_dialog.py b/XYZHubConnector/xyz_qgis/gui/platform_auth_dialog.py index a604f17..fda9b01 100644 --- a/XYZHubConnector/xyz_qgis/gui/platform_auth_dialog.py +++ b/XYZHubConnector/xyz_qgis/gui/platform_auth_dialog.py @@ -8,6 +8,8 @@ # ############################################################################### +from typing import Dict + from qgis.PyQt.QtCore import pyqtSignal, QSortFilterProxyModel from qgis.PyQt.QtWidgets import QDialog from qgis.PyQt.QtGui import QStandardItem @@ -55,7 +57,7 @@ def config( self, token_model: TokenModel, server_model: ServerModel, - map_conn_info: dict[str, SpaceConnectionInfo], + map_conn_info: Dict[str, SpaceConnectionInfo], ): self._set_connected_conn_info(map_conn_info) @@ -118,7 +120,7 @@ def cb_login_fail(self): def _set_connected_conn_info( self, - map_conn_info: dict[str, SpaceConnectionInfo], + map_conn_info: Dict[str, SpaceConnectionInfo], ): self._connected_conn_info = map_conn_info diff --git a/XYZHubConnector/xyz_qgis/iml/network/network.py b/XYZHubConnector/xyz_qgis/iml/network/network.py index 5fda6cf..50457ea 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/network.py +++ b/XYZHubConnector/xyz_qgis/iml/network/network.py @@ -11,6 +11,7 @@ import time import base64 import json +from typing import Dict from .login_webengine import PlatformUserAuthentication, PlatformAuthLoginView from .net_handler import IMLNetworkHandler @@ -169,7 +170,7 @@ def __init__(self, parent): super().__init__(parent) self.user_auth_module = PlatformUserAuthentication(self.network) self.platform_auth = PlatformAuthLoginView() - self._connected_conn_info: dict[str, SpaceConnectionInfo] = dict() + self._connected_conn_info: Dict[str, SpaceConnectionInfo] = dict() self.load_all_connected_conn_info_from_settings() def _get_api_url(self, server: str, api_group): From 017a51957a6f818d01f6edcb314047f7838feea3 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 12 Feb 2024 01:59:44 +0100 Subject: [PATCH 22/42] raise NetworkUnauthorized only for 401 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/net_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/iml/network/net_handler.py b/XYZHubConnector/xyz_qgis/iml/network/net_handler.py index 21708a5..4a31630 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/net_handler.py +++ b/XYZHubConnector/xyz_qgis/iml/network/net_handler.py @@ -45,7 +45,7 @@ def handle_error(cls, response: NetworkResponse): if err == response.get_reply().OperationCanceledError: # operation canceled raise NetworkTimeout(response) - elif status in (401, 403): + elif status in (401,): raise IMLNetworkUnauthorized(response) elif err > 0 or not status: if reply_tag == "oauth_project": From edaf829abf2fc6977cf951a62d4b0adf5fb9ae53 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:18:44 +0100 Subject: [PATCH 23/42] deprecated datahub server Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 5 +---- .../xyz_qgis/gui/iml/iml_token_info_dialog.py | 3 +++ XYZHubConnector/xyz_qgis/models/loading_mode.py | 2 +- XYZHubConnector/xyz_qgis/models/token_model.py | 11 ++++++++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index e9304ee..778d968 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -63,7 +63,6 @@ from .xyz_qgis.layer import tile_utils, XYZLayer from .xyz_qgis.layer.layer_props import QProps - from .xyz_qgis.network import NetManager, net_handler from .xyz_qgis.network.net_utils import CookieUtils, PlatformSettings from .xyz_qgis.iml.loader import ( @@ -250,9 +249,7 @@ def init_modules(self): # token self.token_config = IMLServerTokenConfig(config.USER_PLUGIN_DIR + "/token.ini", parent) - self.token_config.set_default_servers( - dict(NetManager.API_URL, PLATFORM_PRD="PLATFORM_PRD") - ) + self.token_config.set_default_servers(dict(PLATFORM_PRD="PLATFORM_PRD")) self.token_model = self.token_config.get_token_model() self.server_model = self.token_config.get_server_model() diff --git a/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py b/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py index 468ef24..ec9253a 100644 --- a/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py +++ b/XYZHubConnector/xyz_qgis/gui/iml/iml_token_info_dialog.py @@ -103,6 +103,9 @@ def __init__(self, parent=None): if is_here_system(): self.comboBox_token.setEnabled(True) + self.comboBox_api_type.setCurrentIndex(0) + self.cb_change_api_type(0) + self.comboBox_api_type.currentIndexChanged.connect(self.cb_change_api_type) self.comboBox_api_type.currentIndexChanged.connect(self.ui_enable_btn) diff --git a/XYZHubConnector/xyz_qgis/models/loading_mode.py b/XYZHubConnector/xyz_qgis/models/loading_mode.py index 789afb1..01d68d6 100644 --- a/XYZHubConnector/xyz_qgis/models/loading_mode.py +++ b/XYZHubConnector/xyz_qgis/models/loading_mode.py @@ -34,7 +34,7 @@ class ApiType(list): PLATFORM = "platform" def __init__(self): - super().__init__([self.DATAHUB, self.PLATFORM]) + super().__init__([self.PLATFORM, self.DATAHUB]) API_TYPES = ApiType() # datahub, platform diff --git a/XYZHubConnector/xyz_qgis/models/token_model.py b/XYZHubConnector/xyz_qgis/models/token_model.py index 8e8697e..3c6baec 100644 --- a/XYZHubConnector/xyz_qgis/models/token_model.py +++ b/XYZHubConnector/xyz_qgis/models/token_model.py @@ -15,6 +15,8 @@ from qgis.PyQt.QtCore import QIdentityProxyModel, Qt, QVariant from qgis.PyQt.QtGui import QStandardItem, QStandardItemModel +from ..network import datahub_servers + GroupedData = Mapping[str, List[Mapping]] @@ -349,6 +351,7 @@ class ServerModel(WritableItemModel, UsedToken): INFO_KEYS = ["name", "server"] SERIALIZE_KEYS = ["server", "name"] TOKEN_KEY = "server" + DEPRECATED_SERVERS = set(datahub_servers.API_URL.values()) def __init__(self, ini, parser: configparser.ConfigParser = None, parent=None): super().__init__(ini, parser, parent) @@ -366,7 +369,6 @@ def set_default_servers(self, default_api_urls): lambda m: m.get("server"), [ dict(name="HERE Platform", server=default_api_urls.get("PLATFORM_PRD")), - dict(name="HERE Server", server=default_api_urls.get("PRD")), ], ) ) @@ -396,6 +398,13 @@ def _init_default_servers(self, server_infos: list): self.insertRow(i, self.qitems_from_data(server_info)) self.submit_cache() + def _validate_data(self, data): + has_server = data and data.get(self.TOKEN_KEY) + has_deprecated_server = ( + data and data.get("server") and data.get("server") in self.DEPRECATED_SERVERS + ) + return has_server and not has_deprecated_server + class ServerTokenConfig: def __init__(self, ini, parent=None): From c40d25e3fe8bf82a71e0c2aef95933e015e2755f Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 26 Feb 2024 23:05:57 +0100 Subject: [PATCH 24/42] refactor network This reverts commit c27a4835bb772ff341d03bbdf268d908760a85ac. --- XYZHubConnector/plugin.py | 3 ++- XYZHubConnector/xyz_qgis/loader/layer_loader.py | 4 +++- XYZHubConnector/xyz_qgis/loader/manager.py | 2 +- XYZHubConnector/xyz_qgis/network/__init__.py | 2 -- .../xyz_qgis/network/datahub_servers.py | 15 +++++++++++++++ XYZHubConnector/xyz_qgis/network/network.py | 8 +++----- test/load_point_layer.py | 8 ++++---- test/make_point_layer.py | 2 +- test/test_layer_loader.py | 2 +- test/test_tile_loader.py | 2 +- test/utils.py | 3 +-- 11 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 XYZHubConnector/xyz_qgis/network/datahub_servers.py diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index 778d968..2cc38b4 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -63,7 +63,8 @@ from .xyz_qgis.layer import tile_utils, XYZLayer from .xyz_qgis.layer.layer_props import QProps -from .xyz_qgis.network import NetManager, net_handler +from .xyz_qgis.network import net_handler +from .xyz_qgis.network.network import NetManager from .xyz_qgis.network.net_utils import CookieUtils, PlatformSettings from .xyz_qgis.iml.loader import ( IMLTileLayerLoader, diff --git a/XYZHubConnector/xyz_qgis/loader/layer_loader.py b/XYZHubConnector/xyz_qgis/loader/layer_loader.py index d176cc9..a305e58 100644 --- a/XYZHubConnector/xyz_qgis/loader/layer_loader.py +++ b/XYZHubConnector/xyz_qgis/loader/layer_loader.py @@ -31,7 +31,8 @@ from ..layer.layer_utils import update_vlayer_editorWidgetSetup from ..models import SpaceConnectionInfo from ..models.connection import mask_token -from ..network import NetManager, net_handler +from ..network import net_handler +from ..network.network import NetManager from ..common.signal import make_print_qgis @@ -40,6 +41,7 @@ Meta = Dict[str, str] Geojson = Dict + ######################## # Load ######################## diff --git a/XYZHubConnector/xyz_qgis/loader/manager.py b/XYZHubConnector/xyz_qgis/loader/manager.py index 7ac6f90..86e2afd 100644 --- a/XYZHubConnector/xyz_qgis/loader/manager.py +++ b/XYZHubConnector/xyz_qgis/loader/manager.py @@ -31,7 +31,7 @@ from ..models.loading_mode import API_TYPES from ..common.signal import make_print_qgis -from ..network import NetManager +from ..network.network import NetManager print_qgis = make_print_qgis("controller_manager") diff --git a/XYZHubConnector/xyz_qgis/network/__init__.py b/XYZHubConnector/xyz_qgis/network/__init__.py index 96758d1..f45e874 100644 --- a/XYZHubConnector/xyz_qgis/network/__init__.py +++ b/XYZHubConnector/xyz_qgis/network/__init__.py @@ -7,5 +7,3 @@ # License-Filename: LICENSE # ############################################################################### - -from .network import NetManager diff --git a/XYZHubConnector/xyz_qgis/network/datahub_servers.py b/XYZHubConnector/xyz_qgis/network/datahub_servers.py new file mode 100644 index 0000000..c53cb05 --- /dev/null +++ b/XYZHubConnector/xyz_qgis/network/datahub_servers.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Copyright (c) 2019 HERE Europe B.V. +# +# SPDX-License-Identifier: MIT +# License-Filename: LICENSE +# +############################################################################### + + +API_CIT_URL = "https://xyz.cit.api.here.com/hub" +API_PRD_URL = "https://xyz.api.here.com/hub" +API_SIT_URL = "https://xyz.sit.cpdev.aws.in.here.com/hub" +API_URL = dict(PRD=API_PRD_URL, CIT=API_CIT_URL, SIT=API_SIT_URL) diff --git a/XYZHubConnector/xyz_qgis/network/network.py b/XYZHubConnector/xyz_qgis/network/network.py index 43b08a7..ec70c89 100644 --- a/XYZHubConnector/xyz_qgis/network/network.py +++ b/XYZHubConnector/xyz_qgis/network/network.py @@ -12,6 +12,8 @@ from qgis.PyQt.QtCore import QObject, QTimer from qgis.PyQt.QtNetwork import QNetworkAccessManager +from . import datahub_servers +from .net_handler import NetworkHandler from .net_utils import ( make_conn_request, set_qt_property, @@ -19,7 +21,6 @@ make_payload, make_bytes_payload, ) -from .net_handler import NetworkHandler from ..models import SpaceConnectionInfo @@ -29,10 +30,7 @@ class NetManager(QObject): TIMEOUT_COUNT = 1000 - API_CIT_URL = "https://xyz.cit.api.here.com/hub" - API_PRD_URL = "https://xyz.api.here.com/hub" - API_SIT_URL = "https://xyz.sit.cpdev.aws.in.here.com/hub" - API_URL = dict(PRD=API_PRD_URL, CIT=API_CIT_URL, SIT=API_SIT_URL) + API_URL = datahub_servers.API_URL ENDPOINTS = { "space_meta": "/spaces/{space_id}", diff --git a/test/load_point_layer.py b/test/load_point_layer.py index d2d4101..251609e 100644 --- a/test/load_point_layer.py +++ b/test/load_point_layer.py @@ -16,7 +16,7 @@ from XYZHubConnector.xyz_qgis.network import net_handler from test.make_point_layer import step_from_level, iter_lon_lat -from XYZHubConnector.xyz_qgis.network import NetManager +from XYZHubConnector.xyz_qgis.network.network import NetManager from qgis.PyQt.QtCore import QEventLoop from XYZHubConnector.xyz_qgis.models.connection import SpaceConnectionInfo @@ -45,7 +45,7 @@ def get_row_col_bounds_here(level): level 2: 0,0; 0,1; 1,0; 1,1; 2,0; 2,1; 3,0; 3,1 """ nrow = 2 ** (level - 1) if level else 1 - ncol = 2 ** level + ncol = 2**level return nrow, ncol @@ -54,7 +54,7 @@ def get_row_col_bounds_web(level): coord [x,y] """ nrow = 2 ** (level) if level else 1 - ncol = 2 ** level + ncol = 2**level return nrow, ncol @@ -63,7 +63,7 @@ def get_row_col_bounds_tms(level): coord [x,y] """ nrow = 2 ** (level) if level else 1 - ncol = 2 ** level + ncol = 2**level return nrow, ncol diff --git a/test/make_point_layer.py b/test/make_point_layer.py index 9228193..105f0b0 100644 --- a/test/make_point_layer.py +++ b/test/make_point_layer.py @@ -16,7 +16,7 @@ from XYZHubConnector.xyz_qgis.layer import parser -from XYZHubConnector.xyz_qgis.network import NetManager +from XYZHubConnector.xyz_qgis.network.network import NetManager from qgis.PyQt.QtCore import QEventLoop from XYZHubConnector.xyz_qgis.models.connection import SpaceConnectionInfo from qgis.testing import start_app diff --git a/test/test_layer_loader.py b/test/test_layer_loader.py index 0d5149d..ba538bd 100644 --- a/test/test_layer_loader.py +++ b/test/test_layer_loader.py @@ -19,7 +19,7 @@ AllErrorsDuringTest, slow_test, ) -from XYZHubConnector.xyz_qgis.network import NetManager +from XYZHubConnector.xyz_qgis.network.network import NetManager from XYZHubConnector.xyz_qgis.loader import LoadLayerController, ManualInterrupt from qgis.core import QgsProject diff --git a/test/test_tile_loader.py b/test/test_tile_loader.py index 253bb17..fc98008 100644 --- a/test/test_tile_loader.py +++ b/test/test_tile_loader.py @@ -19,7 +19,7 @@ AllErrorsDuringTest, ) from test.test_layer_loader import TestReLoader, TestLoader -from XYZHubConnector.xyz_qgis.network import NetManager +from XYZHubConnector.xyz_qgis.network.network import NetManager from XYZHubConnector.xyz_qgis.loader import TileLayerLoader, EmptyXYZSpaceError from XYZHubConnector.xyz_qgis.layer import bbox_utils, tile_utils, layer_utils diff --git a/test/utils.py b/test/utils.py index 054d7c1..d4a29c2 100644 --- a/test/utils.py +++ b/test/utils.py @@ -16,12 +16,11 @@ from XYZHubConnector.xyz_qgis.controller import AsyncFun, WorkerFun from XYZHubConnector.xyz_qgis.common.error import pretty_print_error -from XYZHubConnector.xyz_qgis.network import NetManager, net_utils +from XYZHubConnector.xyz_qgis.network.network import NetManager from XYZHubConnector.xyz_qgis.models import SpaceConnectionInfo import time import sys -import os import pprint logging.basicConfig( From 64d844c467637641279f05e76ffffcf1e0c450b5 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:20:19 +0100 Subject: [PATCH 25/42] correct clear auth Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/network.py | 1 + XYZHubConnector/xyz_qgis/models/connection.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/XYZHubConnector/xyz_qgis/iml/network/network.py b/XYZHubConnector/xyz_qgis/iml/network/network.py index 50457ea..9d14af8 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/network.py +++ b/XYZHubConnector/xyz_qgis/iml/network/network.py @@ -357,6 +357,7 @@ def clear_auth(self, conn_info: SpaceConnectionInfo): PlatformSettings.remove_connected_conn_info(conn_info.get_server()) if conn_info.is_user_login(): self.user_auth_module.reset_auth(conn_info) + conn_info.unmark_protected() def get_connected_conn_info(self, server): return self._connected_conn_info.get(server) diff --git a/XYZHubConnector/xyz_qgis/models/connection.py b/XYZHubConnector/xyz_qgis/models/connection.py index 02c2a3d..ffab062 100644 --- a/XYZHubConnector/xyz_qgis/models/connection.py +++ b/XYZHubConnector/xyz_qgis/models/connection.py @@ -185,6 +185,9 @@ def has_valid_here_credentials(self): def mark_protected(self): self._is_protected = True + def unmark_protected(self): + self._is_protected = False + def is_protected(self): """Returns True if the connection auth should not be overriden by default connected auth""" return self._is_protected From f4c0838455171b5af57887bf3336553271a4cf59 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 26 Feb 2024 21:39:40 +0100 Subject: [PATCH 26/42] check here system in thread to avoid blocking UI Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 9 ++++++++- XYZHubConnector/xyz_qgis/common/config.py | 19 +++++++++++++++++++ XYZHubConnector/xyz_qgis/common/utils.py | 6 +----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index 2cc38b4..44e3fd3 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -11,7 +11,7 @@ from qgis.core import QgsProject, QgsApplication from qgis.core import Qgis, QgsMessageLog -from qgis.PyQt.QtCore import QCoreApplication, Qt +from qgis.PyQt.QtCore import QCoreApplication, Qt, QThreadPool from qgis.PyQt.QtGui import QIcon from qgis.PyQt.QtWidgets import QAction from qgis.PyQt.QtWidgets import QProgressBar @@ -36,6 +36,7 @@ make_fun_args, parse_exception_obj, ChainInterrupt, + WorkerFun, ) from .xyz_qgis.loader import ( @@ -112,8 +113,14 @@ def __init__(self, iface): self.web_menu = "&{name}".format(name=config.PLUGIN_FULL_NAME) self.hasGuiInitialized = False self.init_modules() + self.init_in_thread() self.obj = self + def init_in_thread(self): + self.pool = QThreadPool() + fn = WorkerFun(utils.is_here_system, self.pool) + fn.call(make_qt_args()) + def initGui(self): """startup""" diff --git a/XYZHubConnector/xyz_qgis/common/config.py b/XYZHubConnector/xyz_qgis/common/config.py index 5628233..cff3a4f 100644 --- a/XYZHubConnector/xyz_qgis/common/config.py +++ b/XYZHubConnector/xyz_qgis/common/config.py @@ -26,6 +26,9 @@ class Config: EXTERNAL_LIB_DIR = os.path.join(PLUGIN_DIR, "external") PYTHON_LOG_FILE = os.path.join(USER_DIR, PLUGIN_NAME, "python.log") + def __init__(self): + self._is_here_system = None + def set_config(self, config): for k, v in config.items(): setattr(self, k, v) @@ -42,3 +45,19 @@ def get_plugin_setting(self, key): key_prefix = "xyz_qgis/settings" key_ = f"{key_prefix}/{key}" return QSettings().value(key_) + + def _check_here_system(self): + import socket + from .crypter import decrypt_text + + socket.setdefaulttimeout(1) + is_here_domain = decrypt_text("Vi5tWQcgFl88Wzg=") in socket.getfqdn() + return is_here_domain + + def is_here_system(self): + if self._is_here_system is None: + try: + self._is_here_system = self._check_here_system() + except Exception as e: + print(e) + return self._is_here_system diff --git a/XYZHubConnector/xyz_qgis/common/utils.py b/XYZHubConnector/xyz_qgis/common/utils.py index 0b9cc7c..16ba8bc 100644 --- a/XYZHubConnector/xyz_qgis/common/utils.py +++ b/XYZHubConnector/xyz_qgis/common/utils.py @@ -215,8 +215,4 @@ def get_qml_import_base_path(): def is_here_system(): - import socket - from .crypter import decrypt_text - - is_here_domain = decrypt_text("Vi5tWQcgFl88Wzg=") in socket.getfqdn() - return is_here_domain + return config.is_here_system() From a693bc819cb454292d2fc25aab580d3604fb4af1 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 27 Feb 2024 00:10:14 +0100 Subject: [PATCH 27/42] use addsitedir Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/XYZHubConnector/config.py b/XYZHubConnector/config.py index d81ce05..dbada79 100644 --- a/XYZHubConnector/config.py +++ b/XYZHubConnector/config.py @@ -10,6 +10,7 @@ import os import sys +import site from qgis.core import QgsApplication @@ -34,8 +35,9 @@ def load_external_lib(): - if EXTERNAL_LIB_DIR not in sys.path: - sys.path.insert(0, EXTERNAL_LIB_DIR) + site.addsitedir(EXTERNAL_LIB_DIR) + # if EXTERNAL_LIB_DIR not in sys.path: + # sys.path.insert(0, EXTERNAL_LIB_DIR) def unload_external_lib(): From 932f73bedcd3ad05f199012415b3ed015227b60c Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 26 Feb 2024 23:38:49 +0100 Subject: [PATCH 28/42] update changelog version 1.9.9 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- CHANGELOG.md | 6 ++++-- RELEASENOTE.md | 6 ++++-- XYZHubConnector/metadata.txt | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ab555..fcaf363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,14 @@ # Changelog -## Version 1.9.9 (2023-10-10) +## Version 1.9.9 (2024-02-27) #### Bug Fixes -* Updated Platform IML server +* Updated Platform IML servers * Set single layering mode as default +* Improved authorization * Improved stability +* Deprecated Datahub servers ## Version 1.9.8 (2023-07-24) diff --git a/RELEASENOTE.md b/RELEASENOTE.md index 03d5638..0806e3b 100644 --- a/RELEASENOTE.md +++ b/RELEASENOTE.md @@ -1,11 +1,13 @@ # Release Notes -## Version 1.9.9 (2023-10-10) +## Version 1.9.9 (2024-02-27) 🐛 FIXES 🐛 -* Updated Platform IML server +* Updated Platform IML servers * Set single layering mode as default +* Improved authorization * Improved stability +* Deprecated Datahub servers ## Version 1.9.8 (2023-07-24) diff --git a/XYZHubConnector/metadata.txt b/XYZHubConnector/metadata.txt index 37f0b06..738e0df 100644 --- a/XYZHubConnector/metadata.txt +++ b/XYZHubConnector/metadata.txt @@ -42,11 +42,13 @@ experimental=False # deprecated flag (applies to the whole plugin, not just a single version) deprecated=False -changelog=Version 1.9.9 (2023-10-10) +changelog=Version 1.9.9 (2024-02-27) 🐛 FIXES 🐛 - * Updated Platform IML server + * Updated Platform IML servers * Set single layering mode as default + * Improved authorization * Improved stability + * Deprecated Datahub servers * .. more details on Github repos \ No newline at end of file From edb074ac90f9d54fd40e53461facd35bc1af2037 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 29 Feb 2024 03:39:21 +0100 Subject: [PATCH 29/42] cleanup network error message Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index 44e3fd3..b29e350 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -494,24 +494,14 @@ def show_net_err(self, err): pair = (status, reason) if status else (err, err_str) status_msg = "{0!s}: {1!s}\n".format(*pair) msg = status_msg + "There was a problem connecting to the server" - if status in [401, 403]: - instruction_msg = ( - ( - "Please input valid token with correct permissions." - "\n" - "Token is generated via " - "" - "https://xyz.api.here.com/token-ui/" - " " - ) - if not conn_info.is_platform_server() - else ( - "Please input valid Platform app credentials." - if not conn_info.is_user_login() - else "Please retry to login with valid Platform user credentials." - ) - ) - msg = status_msg + "Authentication failed" "\n\n" + instruction_msg + if status == 401: + stats_final_msg = "Authentication failed" + instruction_msg = "Please use valid credentials" + msg = status_msg + stats_final_msg + "\n\n" + instruction_msg + elif status == 403: + stats_final_msg = "No access" + instruction_msg = "Please request access to the layer" + msg = status_msg + stats_final_msg + "\n\n" + instruction_msg ret = exec_warning_dialog("Network Error", msg, detail) return True From 32747adf7c59841226ff241ba6c9d638c6a73407 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:32:01 +0100 Subject: [PATCH 30/42] draft: try to make layer id unique, rename to iid Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/plugin.py | 10 +++++----- XYZHubConnector/xyz_qgis/layer/layer.py | 18 +++++++++--------- XYZHubConnector/xyz_qgis/layer/layer_props.py | 5 +++++ XYZHubConnector/xyz_qgis/loader/manager.py | 2 +- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/XYZHubConnector/plugin.py b/XYZHubConnector/plugin.py index b29e350..430e517 100644 --- a/XYZHubConnector/plugin.py +++ b/XYZHubConnector/plugin.py @@ -822,10 +822,10 @@ def _get_lst_reloading_con(self): for qnode in self.iter_visible_xyz_node(): if qnode.nodeType() == qnode.NodeLayer: layer = qnode.layer() - xlayer_id = get_customProperty_str(layer, QProps.UNIQUE_ID) + xlayer_id = QProps.get_iid(layer) is_editable = layer.isEditable() else: - xlayer_id = get_customProperty_str(qnode, QProps.UNIQUE_ID) + xlayer_id = QProps.get_iid(qnode) is_editable = None if xlayer_id in editing_xid: continue @@ -987,7 +987,7 @@ def init_layer_loader(self, qnode): def init_all_layer_loader(self): cnt = 0 for qnode in self.iter_update_all_xyz_node(): - xlayer_id = get_customProperty_str(qnode, QProps.UNIQUE_ID) + xlayer_id = QProps.get_iid(qnode) con = self.con_man.get_loader(xlayer_id) if con: continue @@ -1010,7 +1010,7 @@ def cb_qnode_visibility_changed(self, qnode): if is_xyz_supported_layer(vlayer): vlayer.reload() return - xlayer_id = get_customProperty_str(qnode, QProps.UNIQUE_ID) + xlayer_id = QProps.get_iid(qnode) con = self.con_man.get_interactive_loader(xlayer_id) if con: con.stop_loading() @@ -1022,7 +1022,7 @@ def cb_qnodes_deleting(self, parent, i0, i1): for i in range(i0, i1 + 1): qnode = lst[i] if is_parent_root and is_xyz_supported_node(qnode): - xlayer_id = get_customProperty_str(qnode, QProps.UNIQUE_ID) + xlayer_id = QProps.get_iid(qnode) self.pending_delete_qnodes.setdefault(key, list()).append(xlayer_id) self.con_man.remove_persistent_loader(xlayer_id) # is possible to handle vlayer delete here diff --git a/XYZHubConnector/xyz_qgis/layer/layer.py b/XYZHubConnector/xyz_qgis/layer/layer.py index 0922ebe..d956cf2 100644 --- a/XYZHubConnector/xyz_qgis/layer/layer.py +++ b/XYZHubConnector/xyz_qgis/layer/layer.py @@ -64,7 +64,7 @@ def __init__( conn_info, meta, tags="", - unique: str = None, + iid: str = None, loader_params: dict = None, group_name="XYZ Layer", ext="gpkg", @@ -74,7 +74,7 @@ def __init__( self.conn_info = conn_info self.meta = meta self.tags = tags - self.unique = str(unique or int(time.time() * 10)) + self.iid = str(iid or int(time.time() * 1000)) self.loader_params = loader_params or dict() self._base_group_name = group_name @@ -93,7 +93,7 @@ def load_from_qnode(cls, qnode): meta = get_customProperty_str(qnode, QProps.LAYER_META) conn_info = get_customProperty_str(qnode, QProps.CONN_INFO) tags = get_customProperty_str(qnode, QProps.TAGS) - unique = get_customProperty_str(qnode, QProps.UNIQUE_ID) + iid = get_customProperty_str(qnode, QProps.UNIQUE_ID) loader_params = get_customProperty_str(qnode, QProps.LOADER_PARAMS) meta = load_json_default(meta, default=dict()) conn_info = load_json_default(conn_info, default=dict()) @@ -102,7 +102,7 @@ def load_from_qnode(cls, qnode): name = qnode.name() obj = cls( - conn_info, meta, tags=tags, unique=unique, loader_params=loader_params, group_name=name + conn_info, meta, tags=tags, iid=iid, loader_params=loader_params, group_name=name ) obj.qgroups["main"] = qnode obj._update_group_name(qnode) @@ -173,7 +173,7 @@ def _save_meta_node(self, qnode): qnode.setCustomProperty( QProps.CONN_INFO, json.dumps(self.conn_info.to_project_dict(), ensure_ascii=False) ) - qnode.setCustomProperty(QProps.UNIQUE_ID, self.get_id()) + qnode.setCustomProperty(QProps.UNIQUE_ID, self.get_iid()) self._save_params_to_node(qnode) def _save_meta_vlayer(self, vlayer): @@ -400,14 +400,14 @@ def _layer_fname(self): returns file name of the sqlite db corresponds to xyz layer """ tags = self.tags.replace(",", "_") if len(self.tags) else "" - return "{id}_{tags}_{unique}".format( + return "{id}_{tags}_{iid}".format( id=self.meta.get("id", ""), tags=tags, - unique=self.unique, + iid=self.iid, ) - def get_id(self): - return self.unique + def get_iid(self): + return self.iid def get_map_fields(self): """returns reference to existing mutable map_fields""" diff --git a/XYZHubConnector/xyz_qgis/layer/layer_props.py b/XYZHubConnector/xyz_qgis/layer/layer_props.py index e36f2fe..585af36 100644 --- a/XYZHubConnector/xyz_qgis/layer/layer_props.py +++ b/XYZHubConnector/xyz_qgis/layer/layer_props.py @@ -32,6 +32,11 @@ def getProperty(qnode, key): val = qnode.customProperty(key) return val + @classmethod + def get_iid(cls, qnode): + val = cls.getProperty(qnode, QProps.UNIQUE_ID) + return val + @classmethod def updatePropsVersion(cls, qnode): cls._removeProperty(qnode, cls.EDIT_FLAG) # deprecated props diff --git a/XYZHubConnector/xyz_qgis/loader/manager.py b/XYZHubConnector/xyz_qgis/loader/manager.py index 86e2afd..208e887 100644 --- a/XYZHubConnector/xyz_qgis/loader/manager.py +++ b/XYZHubConnector/xyz_qgis/loader/manager.py @@ -154,7 +154,7 @@ def reset(self): def make_register_xyz_layer_cb(self, con, ptr): def _register_xyz_layer(): # assert con.layer is not None - self._layer_ptr[con.layer.get_id()] = ptr + self._layer_ptr[con.layer.get_iid()] = ptr return _register_xyz_layer From 7558bdf73ea4fafb98b2be3c57beb4db182cdd8e Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:01:04 +0100 Subject: [PATCH 31/42] improve install deps script (pip) + try QtWebEngineWidgets (not working) + confirm dialog before install deps + avoid pip error when sys.stderr is None (qgis python console not opened) + install deps to external folder + login.py: prepare usage QtWebEngineWidgets for future (not working now) + login_webengine.py: align PlatformUserAuthentication (qml) Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/__init__.py | 3 + XYZHubConnector/xyz_qgis/common/utils.py | 71 +++++++++++++--- XYZHubConnector/xyz_qgis/iml/network/login.py | 82 ++++++++++++------- .../xyz_qgis/iml/network/login_webengine.py | 20 +++-- .../xyz_qgis/iml/network/network.py | 7 +- 5 files changed, 131 insertions(+), 52 deletions(-) diff --git a/XYZHubConnector/__init__.py b/XYZHubConnector/__init__.py index 4e54b43..150ffb7 100644 --- a/XYZHubConnector/__init__.py +++ b/XYZHubConnector/__init__.py @@ -21,8 +21,11 @@ def classFactory(iface): """invoke plugin""" from . import config + from .xyz_qgis.common.utils import install_qml_dependencies config.load_external_lib() + install_qml_dependencies() + from .plugin import XYZHubConnector return XYZHubConnector(iface) diff --git a/XYZHubConnector/xyz_qgis/common/utils.py b/XYZHubConnector/xyz_qgis/common/utils.py index 16ba8bc..5f3da14 100644 --- a/XYZHubConnector/xyz_qgis/common/utils.py +++ b/XYZHubConnector/xyz_qgis/common/utils.py @@ -14,6 +14,7 @@ import shutil import gzip import sysconfig +from typing import List from qgis.PyQt.uic import loadUiType from . import config @@ -126,6 +127,29 @@ def read_properties_file(filepath, separator="=", commentcharacter="#") -> dict: return credentials_properties +def _confirm_with_dialog(package: str, extra_packages: List[str] = []) -> bool: + from qgis.PyQt.QtWidgets import QMessageBox + + message = ( + "The following Python packages are required to use the plugin" + f" {config.PLUGIN_NAME}:\n\n" + ) + message += "\n".join([package, *extra_packages]) + message += "\n\nWould you like to install them now? After installation please restart QGIS." + + reply = QMessageBox.question( + None, + "Missing Dependencies", + message, + QMessageBox.Yes | QMessageBox.No, + QMessageBox.No, + ) + + if reply == QMessageBox.Yes: + return True + return False + + def install_package( package, module_name="", package_version="", target_path="", extra_packages=[] ): @@ -140,10 +164,11 @@ def _reload(module_name): base_module_name = module_name.split(".")[0] base_module = importlib.import_module(base_module_name) importlib.reload(base_module) + importlib.import_module(module_name) try: - # _reload(module_name) - # importlib.import_module(module_name) + + _reload(module_name) # require for PyQt5 installed_version = importlib.metadata.version(package) print(package, installed_version) @@ -153,7 +178,10 @@ def _reload(module_name): print(repr(e)) do_install = True if do_install: - py_exec = os.path.join(sysconfig.get_path("scripts"), "python3") + do_install = _confirm_with_dialog(package, extra_packages) + + if do_install: + pip_exec = os.path.join(sysconfig.get_path("scripts"), "pip3") args = ( [ "install", @@ -169,23 +197,40 @@ def _reload(module_name): + extra_packages + (["-t", target_path] if target_path else []) ) + py_exec = os.path.join(sysconfig.get_path("scripts"), "..", "python3") + py_args = ["-m", "pip", *args] with open(config.PYTHON_LOG_FILE, "w") as f: pass - import pip - - ret = pip.main(args) - # import subprocess - # - # ret = subprocess.run([py_exec, "-m", "pip"] + args).returncode + installed = False + try: + cmd = f'"{pip_exec}" {" ".join(args)}' + # cmd = f'"{py_exec}" {" ".join(py_args)}' + # print(cmd); cmd += " && pause || pause" # debug + # print(cmd); cmd += "; read -n 1" # debug + ret = os.system(cmd) + print(ret) + if ret == 0: + installed = True + except Exception as e: + print(e) + if not installed: + try: + import subprocess + + subprocess.check_call([pip_exec, *args]) + # subprocess.check_call([py_exec, *py_args]) + except Exception as e: + print(e) - print(ret) if ret > 0: with open(config.PYTHON_LOG_FILE, "r") as f: txt = f.read() raise Exception(txt) + # _reload(module_name) + def install_qml_dependencies(): package_version = config.get_plugin_setting("PyQtWebEngine_version") @@ -193,7 +238,11 @@ def install_qml_dependencies(): # "PyQtWebEngine", "PyQt5.QtWebEngine", package_version, config.EXTERNAL_LIB_DIR # ) # , "5.15.2") install_package( - "PyQtWebEngine", "PyQt5.QtWebEngine", package_version, extra_packages=["PyQt5-Qt5"] + "PyQtWebEngine", + "PyQt5.QtWebEngineWidgets", + package_version, + config.EXTERNAL_LIB_DIR, + extra_packages=["PyQt5-Qt5"], ) diff --git a/XYZHubConnector/xyz_qgis/iml/network/login.py b/XYZHubConnector/xyz_qgis/iml/network/login.py index 1c01034..562ea1c 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login.py @@ -8,10 +8,9 @@ # ############################################################################### -# from PyQt5.QtWebEngineWidgets import QWebEngineView # import error import json import re -from typing import Optional, Dict +from typing import Optional, Dict, Callable from qgis.PyQt.QtCore import QUrl, QObject from qgis.PyQt.QtCore import Qt @@ -19,8 +18,19 @@ QNetworkAccessManager, QNetworkReply, ) -from qgis.PyQt.QtWebKit import QWebSettings -from qgis.PyQt.QtWebKitWidgets import QWebPage, QWebView, QWebInspector + +USE_WEBKIT = False + +if USE_WEBKIT: + from qgis.PyQt.QtWebKit import QWebSettings as _WebSettings + from qgis.PyQt.QtWebKitWidgets import QWebPage as _WebPage, QWebView as _WebView +else: + from PyQt5.QtWebEngineWidgets import ( + QWebEngineView as _WebView, + QWebEnginePage as _WebPage, + QWebEngineSettings as _WebSettings, + ) # import error + from qgis.PyQt.QtWidgets import QDialog, QGridLayout from .net_handler import IMLNetworkHandler @@ -35,24 +45,24 @@ ) -class WebPage(QWebPage): +class WebPage(_WebPage): def __init__(self, parent, *a, **kw): super().__init__(*a, **kw) self.parent_widget = parent settings = self.settings() - settings.setAttribute(QWebSettings.JavascriptEnabled, True) - settings.setAttribute(QWebSettings.LocalStorageEnabled, True) - settings.setAttribute(QWebSettings.JavascriptCanOpenWindows, True) - settings.setAttribute(QWebSettings.JavascriptCanCloseWindows, True) - # settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True) - settings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True) - settings.setAttribute(QWebSettings.WebGLEnabled, True) - settings.setAttribute(QWebSettings.PluginsEnabled, True) + settings.setAttribute(_WebSettings.JavascriptEnabled, True) + settings.setAttribute(_WebSettings.LocalStorageEnabled, True) + settings.setAttribute(_WebSettings.JavascriptCanOpenWindows, True) + settings.setAttribute(_WebSettings.JavascriptCanCloseWindows, True) + # settings.setAttribute(_WebSettings.PrivateBrowsingEnabled, True) + settings.setAttribute(_WebSettings.DeveloperExtrasEnabled, True) + settings.setAttribute(_WebSettings.WebGLEnabled, True) + settings.setAttribute(_WebSettings.PluginsEnabled, True) settings.setThirdPartyCookiePolicy(settings.AlwaysAllowThirdPartyCookies) def createWindow(self, window_type): - # WindowDialog is just a simple QDialog with a QWebView + # WindowDialog is just a simple QDialog with a web view parent = self.parent_widget dialog = QDialog(parent) @@ -70,7 +80,7 @@ def new_page(cls, dialog: QDialog): page = WebPage(parent) - view = QWebView(parent) + view = _WebView(parent) view.setPage(page) mainLayout = QGridLayout(parent) @@ -84,6 +94,8 @@ def new_page(cls, dialog: QDialog): @classmethod def attach_inspector(cls, dialog: QDialog, page: "WebPage"): + from qgis.PyQt.QtWebKitWidgets import QWebInspector + parent = page.parent_widget inspector = QWebInspector(parent) inspector.setPage(page) @@ -176,17 +188,9 @@ def auth(self, conn_info: SpaceConnectionInfo): conn_info.get_user_email(), conn_info.get_realm(), ) - token = self.get_access_token() - if token: - conn_info.set_(token=token) - else: - dialog = self.open_login_dialog(conn_info) - dialog.exec_() - token = self.get_access_token() - if token: - conn_info.set_(token=token) - else: - self.reset_auth(conn_info) + + conn_info = self.apply_token(conn_info) + return self.make_dummy_reply(parent, conn_info, **kw_prop) def reset_auth(self, conn_info: SpaceConnectionInfo): @@ -199,6 +203,12 @@ def _reset_auth(self, conn_info: SpaceConnectionInfo): realm = conn_info.get_realm() CookieUtils.remove_cookies_from_settings(API_TYPES.PLATFORM, api_env, email, realm) + @classmethod + def apply_token(self, conn_info: SpaceConnectionInfo) -> SpaceConnectionInfo: + token = self.get_access_token(conn_info) + conn_info.set_(token=token) + return token + # private def make_dummy_reply(self, parent, conn_info: SpaceConnectionInfo, **kw_prop): @@ -242,6 +252,11 @@ def cb_url_changed(self, url: QUrl, conn_info: SpaceConnectionInfo): dialog.close() # TODO: reset only auth of the input email self._reset_auth(conn_info) + if "/api/account/realm" in url.toString(): + m = self.REGEX_REALM.search(url) + if m and len(m.groups()): + realm = m.group(1) + self._update_conn_info(conn_info, realm=realm) def cb_auth_handler(self, reply, conn_info: SpaceConnectionInfo): try: @@ -285,7 +300,9 @@ def auth_handler(self, url: QUrl, conn_info: SpaceConnectionInfo): # reply.waitForReadyRead(1000) # self.cb_auth_handler(reply) - def open_login_dialog(self, conn_info: SpaceConnectionInfo, parent=None): + def open_login_dialog( + self, conn_info: SpaceConnectionInfo, parent=None, cb_login_view_closed: Callable = None + ): api_env = self.get_api_env(conn_info) email = conn_info.get_user_email() realm = conn_info.get_realm() @@ -312,12 +329,15 @@ def open_login_dialog(self, conn_info: SpaceConnectionInfo, parent=None): ] ) - page.networkAccessManager().finished.connect( - lambda reply: self.cb_response_handler(reply, conn_info) - ) - page.currentFrame().urlChanged.connect(lambda url: self.cb_url_changed(url, conn_info)) + # page.networkAccessManager().finished.connect( + # lambda reply: self.cb_response_handler(reply, conn_info) + # ) + page.urlChanged.connect(lambda url: self.cb_url_changed(url, conn_info)) dialog.finished.connect(lambda *a: self.cb_dialog_closed(conn_info)) + if cb_login_view_closed: + dialog.finished.connect(cb_login_view_closed) + dialog.open() view.load(QUrl(url)) else: diff --git a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py index 41bb064..1555d87 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py @@ -10,6 +10,7 @@ import json import os +from typing import Callable try: from PyQt5.Qt import PYQT_VERSION_STR @@ -120,12 +121,6 @@ def get_access_token(self, conn_info: SpaceConnectionInfo) -> str: token_obj = dict() return token_obj.get("accessToken", "") if isinstance(token_obj, dict) else "" - @classmethod - def apply_token(cls, conn_info: SpaceConnectionInfo) -> str: - token = cls.get_access_token(conn_info) - conn_info.set_(token=token) - return conn_info - @classmethod def create_qml_view(cls, login_url: str, title="", cb_login_view_closed=None): @@ -199,6 +194,7 @@ class PlatformUserAuthentication: def __init__(self, network: QNetworkAccessManager): self.signal = BasicSignal() self.network = network + self.platform_auth_view = PlatformAuthLoginView() # public @@ -238,3 +234,15 @@ def make_dummy_reply(self, parent, conn_info: SpaceConnectionInfo, **kw_prop): qobj = QObject(parent) set_qt_property(qobj, conn_info=conn_info, **kw_prop) return qobj + + # dialog, token + + def open_login_dialog( + self, conn_info: SpaceConnectionInfo, parent=None, cb_login_view_closed: Callable = None + ): + return self.platform_auth_view.open_login_view(conn_info, parent, cb_login_view_closed) + + def apply_token(self, conn_info: SpaceConnectionInfo) -> SpaceConnectionInfo: + token = self.platform_auth_view.get_access_token(conn_info) + conn_info.set_(token=token) + return conn_info diff --git a/XYZHubConnector/xyz_qgis/iml/network/network.py b/XYZHubConnector/xyz_qgis/iml/network/network.py index 9d14af8..dd18cc7 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/network.py +++ b/XYZHubConnector/xyz_qgis/iml/network/network.py @@ -13,7 +13,7 @@ import json from typing import Dict -from .login_webengine import PlatformUserAuthentication, PlatformAuthLoginView +from .login import PlatformUserAuthentication from .net_handler import IMLNetworkHandler from .platform_server import PlatformServer from ...common.crypter import decrypt_text @@ -169,7 +169,6 @@ def print_cookies(self): def __init__(self, parent): super().__init__(parent) self.user_auth_module = PlatformUserAuthentication(self.network) - self.platform_auth = PlatformAuthLoginView() self._connected_conn_info: Dict[str, SpaceConnectionInfo] = dict() self.load_all_connected_conn_info_from_settings() @@ -333,10 +332,10 @@ def apply_connected_conn_info(self, conn_info: SpaceConnectionInfo): return conn_info def open_login_view(self, conn_info: SpaceConnectionInfo, callback=None): - conn_info = self.platform_auth.apply_token(conn_info) + conn_info = self.user_auth_module.apply_token(conn_info) if not conn_info.has_token(): try: - self.platform_auth.open_login_view(conn_info, cb_login_view_closed=callback) + self.user_auth_module.open_login_dialog(conn_info, cb_login_view_closed=callback) except Exception as e: if callback: callback() From 7fc96c7abf7989e0a2ca0e06118fc380eb753ea9 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:22:03 +0100 Subject: [PATCH 32/42] use qml, QtWebEngine and QtQuick again Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/__init__.py | 2 -- XYZHubConnector/xyz_qgis/common/utils.py | 2 +- XYZHubConnector/xyz_qgis/iml/network/network.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/XYZHubConnector/__init__.py b/XYZHubConnector/__init__.py index 150ffb7..3a248d1 100644 --- a/XYZHubConnector/__init__.py +++ b/XYZHubConnector/__init__.py @@ -21,10 +21,8 @@ def classFactory(iface): """invoke plugin""" from . import config - from .xyz_qgis.common.utils import install_qml_dependencies config.load_external_lib() - install_qml_dependencies() from .plugin import XYZHubConnector diff --git a/XYZHubConnector/xyz_qgis/common/utils.py b/XYZHubConnector/xyz_qgis/common/utils.py index 5f3da14..9896d25 100644 --- a/XYZHubConnector/xyz_qgis/common/utils.py +++ b/XYZHubConnector/xyz_qgis/common/utils.py @@ -239,7 +239,7 @@ def install_qml_dependencies(): # ) # , "5.15.2") install_package( "PyQtWebEngine", - "PyQt5.QtWebEngineWidgets", + "PyQt5.QtWebEngine", # "PyQt5.QtWebEngineWidgets" package_version, config.EXTERNAL_LIB_DIR, extra_packages=["PyQt5-Qt5"], diff --git a/XYZHubConnector/xyz_qgis/iml/network/network.py b/XYZHubConnector/xyz_qgis/iml/network/network.py index dd18cc7..feadc0c 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/network.py +++ b/XYZHubConnector/xyz_qgis/iml/network/network.py @@ -13,7 +13,7 @@ import json from typing import Dict -from .login import PlatformUserAuthentication +from .login_webengine import PlatformUserAuthentication from .net_handler import IMLNetworkHandler from .platform_server import PlatformServer from ...common.crypter import decrypt_text From 05e38138f2f9b213a9797f872fb6c2f728f138ae Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 7 Mar 2024 22:18:46 +0100 Subject: [PATCH 33/42] update build script for win Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- makeBuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makeBuild.sh b/makeBuild.sh index bd34f3b..9a3a268 100755 --- a/makeBuild.sh +++ b/makeBuild.sh @@ -37,7 +37,7 @@ fi ( cd build if $( echo $ver | grep -q alpha ); then - sed -i "" -e "s/name=.*/name=HERE Maps for QGIS alpha/" \ + sed -i"" -e "s/name=.*/name=HERE Maps for QGIS alpha/" \ -e "s/version=.*/version=$ver/" \ ./$folder/metadata.txt fi From bd120cfdd0f1ffc0544cba28d3d4fbe297686dbb Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Thu, 7 Mar 2024 23:05:16 +0100 Subject: [PATCH 34/42] Fix error since QGIS 3.36.0+: Failed to create OpenGL context for format QSurfaceFormat(...). This is most likely caused by not having the necessary graphics drivers installed https://github.com/qt/qtdeclarative/blob/v5.15.3-lts-lgpl/src/quick/items/qquickwindow.cpp#L1457 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/login_webengine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py index 1555d87..a2c278d 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py @@ -20,8 +20,10 @@ except Exception as e: PYQT_STR = "PyQt (unknown){error}".format(error=" - " + repr(e)) +from PyQt5.QtGui import QSurfaceFormat from PyQt5.QtQuick import QQuickView + from qgis.PyQt.QtCore import QUrl, QObject from qgis.PyQt.QtCore import Qt from qgis.PyQt.QtNetwork import ( @@ -123,6 +125,7 @@ def get_access_token(self, conn_info: SpaceConnectionInfo) -> str: @classmethod def create_qml_view(cls, login_url: str, title="", cb_login_view_closed=None): + QSurfaceFormat.setDefaultFormat(QSurfaceFormat()) view = QQuickView() engine = view.engine() From 7567d06afb28293d84c2b7fdb0cda3971cdb667a Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:23:35 +0100 Subject: [PATCH 35/42] fix apply_token Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/login_webengine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py index a2c278d..de87e98 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py @@ -225,7 +225,7 @@ def auth(self, conn_info: SpaceConnectionInfo): # parent = self.network parent = None - PlatformAuthLoginView.apply_token(conn_info) + self.apply_token(conn_info) return self.make_dummy_reply(parent, conn_info, **kw_prop) def reset_auth(self, conn_info: SpaceConnectionInfo): From f6cdf2badfa0ef9321d3ccb9a80ea65a0cd0fb57 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:00:17 +0100 Subject: [PATCH 36/42] update build script for alpha version, folder suffix Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- makeBuild.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/makeBuild.sh b/makeBuild.sh index 9a3a268..00a883e 100755 --- a/makeBuild.sh +++ b/makeBuild.sh @@ -37,9 +37,14 @@ fi ( cd build if $( echo $ver | grep -q alpha ); then - sed -i"" -e "s/name=.*/name=HERE Maps for QGIS alpha/" \ + sed -i"" -e "s/name=.*/\0 alpha/" \ -e "s/version=.*/version=$ver/" \ ./$folder/metadata.txt fi +if [ "$folderSuffix" ]; then + sed -i"" -e "s/name=.*/\0 $folderSuffix/" \ + ./$folder/metadata.txt +fi + python ../zip_dir.py $folder QGIS-XYZ-Plugin-$ver.zip ) From 8a9a3ae943cc2ec452f8bb25abee0ec3a6de5d0b Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:41:58 +0100 Subject: [PATCH 37/42] test crypter with custom env variable Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- test/test_crypter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_crypter.py b/test/test_crypter.py index 06fde77..19c1015 100644 --- a/test/test_crypter.py +++ b/test/test_crypter.py @@ -77,11 +77,14 @@ def test_encrypt_platform_endpoint(self): encoded = encrypt_text(endpoint) print(name, encoded) - @debug_test_function def test_encrypt_string(self): - for text in []: + for text in os.environ.get("TEXT_TO_ENCRYPT", "example").split(","): print(text, encrypt_text(text)) + def test_decrypt_string(self): + for text in os.environ.get("TEXT_TO_DECRYPT", "UjIiXBI+Fg==").split(","): + print(text, decrypt_text(text)) + if __name__ == "__main__": unittest.main() From a0cdda96a167e5edb06ea4b8c122403746ea1c19 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 8 Mar 2024 20:17:43 +0100 Subject: [PATCH 38/42] improve check domain logic Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/common/config.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/common/config.py b/XYZHubConnector/xyz_qgis/common/config.py index cff3a4f..6bb480a 100644 --- a/XYZHubConnector/xyz_qgis/common/config.py +++ b/XYZHubConnector/xyz_qgis/common/config.py @@ -9,6 +9,7 @@ ############################################################################### import os +from typing import Iterable from ... import __version__ as version @@ -51,7 +52,24 @@ def _check_here_system(self): from .crypter import decrypt_text socket.setdefaulttimeout(1) - is_here_domain = decrypt_text("Vi5tWQcgFl88Wzg=") in socket.getfqdn() + + def _check_host(host: str) -> bool: + is_host_reachable = False + try: + ip = socket.gethostbyname(host) + is_host_reachable = len(ip.split(".")) == 4 + except: + pass + return is_host_reachable + + def _check_fqdn(hosts: Iterable[str]) -> bool: + fqdn = socket.getfqdn() + return any(host in fqdn for host in hosts) + + host1 = decrypt_text("Vi5tWQcgFl88Wzg=") + host2 = decrypt_text("XiRtWQcgFl88Wzg=") + + is_here_domain = _check_host(host1) or _check_host(host2) or _check_fqdn([host1, host2]) return is_here_domain def is_here_system(self): From a1594a2baa14416e3f933b9a13d85725df84a568 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:37:51 +0100 Subject: [PATCH 39/42] fix qml dependencies on mac Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/common/utils.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/common/utils.py b/XYZHubConnector/xyz_qgis/common/utils.py index 9896d25..74a4bdb 100644 --- a/XYZHubConnector/xyz_qgis/common/utils.py +++ b/XYZHubConnector/xyz_qgis/common/utils.py @@ -50,7 +50,9 @@ def add_qml_import_path(qml_engine): # Setup for the MAC OS X platform: # return if platform.system() == "Darwin" or os.name == "mac": - install_qml_dependencies() + install_qml_dependencies( + isolated=False + ) # non-isolated to avoid errors: icu data, sip, qml qml_engine.addImportPath(os.path.join(get_qml_import_base_path(), "qml")) qml_engine.addImportPath(os.path.join(get_qml_import_base_path(), "bin")) elif platform.system() == "Linux" or os.name == "posix": @@ -204,6 +206,7 @@ def _reload(module_name): pass installed = False + ret = 0 try: cmd = f'"{pip_exec}" {" ".join(args)}' # cmd = f'"{py_exec}" {" ".join(py_args)}' @@ -219,8 +222,10 @@ def _reload(module_name): try: import subprocess - subprocess.check_call([pip_exec, *args]) - # subprocess.check_call([py_exec, *py_args]) + ret = subprocess.check_call([pip_exec, *args]) + # ret = subprocess.check_call([py_exec, *py_args]) + print(ret) + except Exception as e: print(e) @@ -232,7 +237,7 @@ def _reload(module_name): # _reload(module_name) -def install_qml_dependencies(): +def install_qml_dependencies(isolated=True): package_version = config.get_plugin_setting("PyQtWebEngine_version") # install_package( # "PyQtWebEngine", "PyQt5.QtWebEngine", package_version, config.EXTERNAL_LIB_DIR @@ -241,7 +246,7 @@ def install_qml_dependencies(): "PyQtWebEngine", "PyQt5.QtWebEngine", # "PyQt5.QtWebEngineWidgets" package_version, - config.EXTERNAL_LIB_DIR, + config.EXTERNAL_LIB_DIR if isolated else "", extra_packages=["PyQt5-Qt5"], ) From 753ccccf22221184098708261ac58da896434178 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 12 Mar 2024 01:39:50 +0100 Subject: [PATCH 40/42] fix qml on mac Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/iml/network/login_webengine.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py index de87e98..9c69342 100644 --- a/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py +++ b/XYZHubConnector/xyz_qgis/iml/network/login_webengine.py @@ -10,6 +10,7 @@ import json import os +import platform from typing import Callable try: @@ -125,7 +126,10 @@ def get_access_token(self, conn_info: SpaceConnectionInfo) -> str: @classmethod def create_qml_view(cls, login_url: str, title="", cb_login_view_closed=None): - QSurfaceFormat.setDefaultFormat(QSurfaceFormat()) + os.environ["QML_USE_GLYPHCACHE_WORKAROUND"] = "1" + + if not (platform.system() == "Darwin" or os.name == "mac"): + QSurfaceFormat.setDefaultFormat(QSurfaceFormat()) # fix fot windows rdp, break mac view = QQuickView() engine = view.engine() From 9ef5cc97e3260587ff59a95fc9fd0dc917de42c1 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:29:44 +0100 Subject: [PATCH 41/42] cleanup Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- XYZHubConnector/xyz_qgis/layer/edit_buffer.py | 1 - XYZHubConnector/xyz_qgis/layer/queue.py | 8 ++++---- XYZHubConnector/xyz_qgis/models/filter_model.py | 4 +--- XYZHubConnector/xyz_qgis/models/token_model.py | 4 ++-- test/test_layer_loader.py | 3 ++- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/XYZHubConnector/xyz_qgis/layer/edit_buffer.py b/XYZHubConnector/xyz_qgis/layer/edit_buffer.py index 61d0538..3b50de2 100644 --- a/XYZHubConnector/xyz_qgis/layer/edit_buffer.py +++ b/XYZHubConnector/xyz_qgis/layer/edit_buffer.py @@ -12,7 +12,6 @@ from qgis.PyQt.QtCore import Qt from . import parser -from .layer_props import QProps from .layer_utils import ( get_feat_upload_from_iter, is_layer_committed, diff --git a/XYZHubConnector/xyz_qgis/layer/queue.py b/XYZHubConnector/xyz_qgis/layer/queue.py index a82ecb9..ec750e1 100644 --- a/XYZHubConnector/xyz_qgis/layer/queue.py +++ b/XYZHubConnector/xyz_qgis/layer/queue.py @@ -10,7 +10,7 @@ from collections import deque from . import bbox_utils -from typing import Iterable +from typing import Dict, List from ..common.utils import get_current_millis_time @@ -23,7 +23,7 @@ class ParamsQueue(object): if response error, retry with smaller limit from h0 to h1 """ - def __init__(self, params: Iterable, **kw): + def __init__(self, params: Dict, **kw): raise NotImplementedError() def has_next(self) -> bool: @@ -45,7 +45,7 @@ def has_retry(self) -> bool: class SimpleQueue(ParamsQueue): """Simple params queue with setter, getter""" - def __init__(self, params: list = None, key=None, **kw): + def __init__(self, params: List = None, key=None, **kw): self._queue = list() self.idx = 0 if params: @@ -151,7 +151,7 @@ class ParamsQueue_deque_v1(ParamsQueue): If response error, retry with smaller limit from h0 to h1. """ - def __init__(self, params, buffer_size=1): + def __init__(self, params: Dict, buffer_size=1): self._buffer_size = buffer_size self.retries = 0 self.limit = params.get("limit", 1) diff --git a/XYZHubConnector/xyz_qgis/models/filter_model.py b/XYZHubConnector/xyz_qgis/models/filter_model.py index 4aa4069..e4fa415 100644 --- a/XYZHubConnector/xyz_qgis/models/filter_model.py +++ b/XYZHubConnector/xyz_qgis/models/filter_model.py @@ -9,9 +9,7 @@ ############################################################################### -from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem -from qgis.PyQt.QtCore import QIdentityProxyModel, Qt, QVariant -from .token_model import EditableItemModel, UsedToken +from .token_model import EditableItemModel class FilterModel(EditableItemModel): diff --git a/XYZHubConnector/xyz_qgis/models/token_model.py b/XYZHubConnector/xyz_qgis/models/token_model.py index 3c6baec..1998a34 100644 --- a/XYZHubConnector/xyz_qgis/models/token_model.py +++ b/XYZHubConnector/xyz_qgis/models/token_model.py @@ -10,14 +10,14 @@ import configparser -from typing import List, Mapping +from typing import List, Dict from qgis.PyQt.QtCore import QIdentityProxyModel, Qt, QVariant from qgis.PyQt.QtGui import QStandardItem, QStandardItemModel from ..network import datahub_servers -GroupedData = Mapping[str, List[Mapping]] +GroupedData = Dict[str, List[Dict]] class UsedToken: diff --git a/test/test_layer_loader.py b/test/test_layer_loader.py index ba538bd..09599ab 100644 --- a/test/test_layer_loader.py +++ b/test/test_layer_loader.py @@ -148,7 +148,8 @@ def _load_layer(self, loader, conn_info, meta, **kw_start): timer.stop() lst_layer.append(loader.layer) - # with self.assertRaises(AllErrorsDuringTest, msg="stopping loader should emit error") as cm: + # with self.assertRaises( + # AllErrorsDuringTest, msg="stopping loader should emit error") as cm: # self._wait_async() # lst_err = cm.exception.args[0] # self.assertIsInstance(lst_err[0], ManualInterrupt) From 9151e5a0f31f979e6293668a61cc745142817e70 Mon Sep 17 00:00:00 2001 From: minff <16268924+minff@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:10:58 +0100 Subject: [PATCH 42/42] update changelog version 1.9.9 Signed-off-by: minff <16268924+minff@users.noreply.github.com> --- CHANGELOG.md | 4 +++- RELEASENOTE.md | 7 +++++-- XYZHubConnector/metadata.txt | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcaf363..c8b19ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Version 1.9.9 (2024-02-27) +## Version 1.9.9 (2024-03-12) #### Bug Fixes @@ -9,6 +9,8 @@ * Improved authorization * Improved stability * Deprecated Datahub servers +* Fixed OpenGL outdated driver error +* Show confirm dialog before installing dependencies ## Version 1.9.8 (2023-07-24) diff --git a/RELEASENOTE.md b/RELEASENOTE.md index 0806e3b..c9944ee 100644 --- a/RELEASENOTE.md +++ b/RELEASENOTE.md @@ -1,13 +1,16 @@ # Release Notes -## Version 1.9.9 (2024-02-27) +## Version 1.9.9 (2024-03-12) 🐛 FIXES 🐛 + * Updated Platform IML servers * Set single layering mode as default -* Improved authorization +* Improved authorization * Improved stability * Deprecated Datahub servers +* Fixed OpenGL outdated driver error +* Show confirm dialog before installing dependencies ## Version 1.9.8 (2023-07-24) diff --git a/XYZHubConnector/metadata.txt b/XYZHubConnector/metadata.txt index 738e0df..d6f7dc2 100644 --- a/XYZHubConnector/metadata.txt +++ b/XYZHubConnector/metadata.txt @@ -42,7 +42,7 @@ experimental=False # deprecated flag (applies to the whole plugin, not just a single version) deprecated=False -changelog=Version 1.9.9 (2024-02-27) +changelog=Version 1.9.9 (2024-03-12) 🐛 FIXES 🐛 * Updated Platform IML servers @@ -50,5 +50,7 @@ changelog=Version 1.9.9 (2024-02-27) * Improved authorization * Improved stability * Deprecated Datahub servers + * Fixed OpenGL outdated driver error + * Show confirm dialog before installing dependencies * .. more details on Github repos \ No newline at end of file