2023.3 updates (#72)

* MI 3.28 env

* styling peilvakken weer werkend

* vs code startup naar MI 3.28

* pygeos weg met gpd 0.12.2

* migrate sqlite before loading

* fix datachecker gdb to gpkg

* HR luchtfoto

* styling 0d1d

watergangen geen verhang pixel naar mm
kaart 1 bovenaan

* fix sqlitetest wateroppervlak

resultaat werd weggeschreven naar verkeerde laag

* update env

* raster edits deps

* load local hrt before htt.

Otherwise it leads to unexpected behaviour with htt using the installed hrt instead of the github one.

* datachecker env

* ThreeDiToolbox 2.5.3

* deps ThreeDiToolbox 2.5.3

* Create environment_services.yml

* remove local htt and hrt from path

instead these modules should now be installed using the bin/local_install.bat file in the repositories.

* remove localsettings hrt htt paths

* update env and version

* update envs

* update env

* Create mi_threedi_toolbox_upgrade.PNG

---------

Co-authored-by: Esse <[email protected]>
Co-authored-by: wietsevangerwen <[email protected]>
Co-authored-by: D2Hydro <[email protected]>
Co-authored-by: Daniel <[email protected]>

hhnk_threedi_plugin/__init__.py

@@ -36,29 +36,30 @@
 except ModuleNotFoundError:
     import hhnk_threedi_plugin.local_settings_default as local_settings
 
-if local_settings.DEBUG:
-    if local_settings.hhnk_threedi_tools_path not in sys.path:
-        sys.path.insert(0, local_settings.hhnk_threedi_tools_path)
+# if local_settings.DEBUG:
+#     try: 
+#         if local_settings.hhnk_research_tools_path not in sys.path:
+#             sys.path.insert(0, local_settings.hhnk_research_tools_path)
 
-        # reload hhnk_threedi_tools and all modules within when reloading the plugin in QGIS.
-        # Does not work with importlib.reload.
-        import hhnk_threedi_tools
-        for m in [i for i in sys.modules.keys() if i.startswith('hhnk_threedi_tools')]:
-            del(sys.modules[m])
-        import hhnk_threedi_tools
+#             # reload hhnk_threedi_tools and all modules within when reloading the plugin in QGIS.
+#             # Does not work with importlib.reload.
+#             import hhnk_research_tools
+#             for m in [i for i in sys.modules.keys() if i.startswith('hhnk_research_tools')]:
+#                 del(sys.modules[m])
+#             import hhnk_research_tools
+#     except:
+#         pass
+#     if local_settings.hhnk_threedi_tools_path not in sys.path:
+#         sys.path.insert(0, local_settings.hhnk_threedi_tools_path)
+
+#         # reload hhnk_threedi_tools and all modules within when reloading the plugin in QGIS.
+#         # Does not work with importlib.reload.
+#         import hhnk_threedi_tools
+#         for m in [i for i in sys.modules.keys() if i.startswith('hhnk_threedi_tools')]:
+#             del(sys.modules[m])
+#         import hhnk_threedi_tools
 
-    try: 
-        if local_settings.hhnk_research_tools_path not in sys.path:
-            sys.path.insert(0, local_settings.hhnk_research_tools_path)
 
-            # reload hhnk_threedi_tools and all modules within when reloading the plugin in QGIS.
-            # Does not work with importlib.reload.
-            import hhnk_research_tools
-            for m in [i for i in sys.modules.keys() if i.startswith('hhnk_research_tools')]:
-                del(sys.modules[m])
-            import hhnk_research_tools
-    except:
-        pass
 
     #sys.path.append(local_settings.threeditoolbox_path)
 

hhnk_threedi_plugin/dependencies.py

@@ -281,7 +281,7 @@ def _update_path(directories):
         dir_path = Path(dir_path)
         if dir_path.exists():
             if str(dir_path) not in sys.path:
-                sys.path.insert(0, str(dir_path))
+                sys.path.append(str(dir_path))
                 logger.info(f"{dir_path} added to sys.path")
         else:
             logger.warning(

hhnk_threedi_plugin/env/environment.yml

@@ -8,7 +8,7 @@ channels:
 dependencies:
   #QGIS preinstalled
   - python=3.9.5 # equals MI python version
-  - geopandas=0.11.1
+  - geopandas=0.12.2
   - pandas=1.1.3
   - scipy=1.6.2
 
@@ -41,19 +41,21 @@ dependencies:
   - iniconfig=2.0.0 #pytest dependency
   - tqdm=4.64.0
   - xarray=2022.3.0
-  - pygeos=0.14 #required for geopandas
-
 
   - pip: # 3Di packages are not conda installable and are therefore installed by PIP
     # - et-xmlfile==1.1.0
     - openpyxl==3.1.2
-    - threedi_modelchecker==2.1.0 #ThreediToolbox 2.5.2
-    - threedi_scenario_downloader==0.16
-    - threedi_schema==0.216.4 #ThreediToolbox 2.5.2
+    - threedi_modelchecker==2.2.4 #ThreediToolbox 2.5.3
+    - threedi_scenario_downloader==1.1 #Lizard API v4 only
+    - threedi_schema==0.217.5 #ThreediToolbox 2.5.3
     - threedi_raster_edits==0.26
-    - threedi-api-client==4.1.1
-    - threedigrid_builder==1.10.0 #ThreediToolbox 2.5.2
-
+    - threedi-api-client==4.1.2
+    - threedigrid_builder==1.11.4 #ThreediToolbox 2.5.3
+    - threedigrid==2.0.5
+    - threedidepth==0.5 #ThreediToolbox 2.5.2
+    - cached_property==1.5.2 #threedi_raster_edits dep
+    - rtree==1.0.1 #threedi_raster_edits dep 
+
     #Plugin
-    - hhnk_research_tools==2023.2
-    - hhnk_threedi_tools==2023.2
+    - hhnk_research_tools==2023.3
+    - hhnk_threedi_tools==2023.3

hhnk_threedi_plugin/env/environment_dev.yml

@@ -17,13 +17,13 @@ dependencies:
   - ipyfilechooser=0.6.0
   - jupyter=1.0.0
   - geoalchemy2=0.10.2
-  - geopandas=0.11.1
+  - geopandas=0.12.2
   - tomli=2.0.1
   - tzlocal=2.1
   - packaging=23.0
   - pandas=1.1.3
   - pluggy=1.0.0
-  - pygeos=0.14
+  # - pygeos=0.14 #required for geopandas, not for geopandas 0.12.2
   - python-dateutil=2.8.2
   - pytz=2020.1
   - pytz-deprecation-shim=0.1.0.post0
@@ -35,9 +35,13 @@ dependencies:
   - pip: # 3Di packages are not conda installable and are therefore installed by PIP
     - et-xmlfile==1.1.0
     - openpyxl==3.1.2
-    - threedi_modelchecker==2.1.0 #ThreediToolbox 2.5.2
-    - threedi_scenario_downloader==0.16
-    - threedi_schema==0.216.4 #ThreediToolbox 2.5.2
+    - threedi_modelchecker==2.2.4 #ThreediToolbox 2.5.3
+    - threedi_scenario_downloader==1.1 #Lizard API v4 only
+    - threedi_schema==0.217.5 #ThreediToolbox 2.5.3
     - threedi_raster_edits==0.26
-    - threedi-api-client==4.1.1
-    - threedigrid_builder==1.10.0 #ThreediToolbox 2.5.2
+    - threedi-api-client==4.1.2
+    - threedigrid_builder==1.11.4 #ThreediToolbox 2.5.3
+    - threedigrid==2.0.5
+    - threedidepth==0.5 #ThreediToolbox 2.5.2
+    - cached_property==1.5.2 #threedi_raster_edits dep
+    - rtree==1.0.1 #threedi_raster_edits dep 

hhnk_threedi_plugin/env/environment_services.yml

@@ -0,0 +1,65 @@
+# This is an environment for hhnk_qgis_plugin installation
+name: threedipy
+
+channels:
+  - conda-forge
+  - defaults
+
+dependencies:
+  #QGIS preinstalled
+  - python=3.9.5 # equals MI python version
+  - geopandas=0.12.2
+  - pandas=1.1.3
+  - scipy=1.6.2
+
+  # - attrs=22.2.0
+  # - apscheduler=3.10.1
+  # - click=8.1.3
+  # - click-plugins=1.1.1
+  # - colorama=0.4.6
+  # - exceptiongroup=1.1.1
+  # - iniconfig=2.0.0
+  # - ipyfilechooser=0.6.0
+  # - jupyter=1.0.0
+  # - geoalchemy2=0.10.2
+  # - tomli=2.0.1
+  # - tzlocal=2.1
+  # - packaging=23.0
+  # - python-dateutil=2.8.2
+  # - pytz=2020.1
+  # - pytz-deprecation-shim=0.1.0.post0
+  # - sqlalchemy=1.3.24
+
+  #User folder
+  - jupyterlab=3.6.3
+  - ipywidgets=8.0.6
+  - pytest=7.2.2
+  - pytest-cov
+  - pluggy=1.0.0 #pytest dependency
+  - exceptiongroup=1.1.1 #pytest dependency
+  - iniconfig=2.0.0 #pytest dependency
+  - tqdm=4.64.0
+  - xarray=2022.3.0
+  # - pygeos=0.14 #required for geopandas, not for geopandas 0.12.2
+
+  #Voor DataCheker
+  - flask # oude datachecker
+  - sqlparse # oude datachecker
+  - psycopg2 # oude datachecker
+  - fastapi # nieuwe datachecker
+  - uvicorn # nieuwe datachecker
+
+  #Voor Daniel
+  - spyder
+  - pip: # 3Di packages are not conda installable and are therefore installed by PIP
+    # - et-xmlfile==1.1.0
+    - openpyxl==3.1.2
+    - threedi_modelchecker==2.2.4 #ThreediToolbox 2.5.3
+    - threedi_scenario_downloader==1.1 #Lizard API v4 only
+    - threedi_schema==0.217.5 #ThreediToolbox 2.5.3
+    - threedi_raster_edits==0.26
+    - threedi-api-client==4.1.2
+    - threedigrid_builder==1.11.4 #ThreediToolbox 2.5.3
+    - threedigrid==2.0.5
+    - cached_property==1.5.2 #threedi_raster_edits dep
+    - rtree==1.0.1 #threedi_raster_edits dep 

hhnk_threedi_plugin/gui/load_layers_popup.py

@@ -20,7 +20,7 @@
 from qgis.utils import QgsMessageBar, QgsMessageLog, iface
 
 
-from hhnk_threedi_tools import SqliteCheck
+from hhnk_threedi_tools import SqliteCheck, MigrateSchema
 import hhnk_research_tools as hrt
 
 # new
@@ -226,7 +226,13 @@ def load_layers(self):
                     'klimaatsommen':''}
 
         if self.sqlite_selector.isChecked() == True:
-             load_layers_interaction.load_sqlite(filepath=self.caller.fenv.model.schema_base.sqlite_paths[0])
+
+            #Migrate sqlite to newest version
+            migrate_schema = MigrateSchema(filename=self.caller.fenv.model.schema_base.sqlite_paths[0])
+            migrate_schema.run()
+
+            #load in project
+            load_layers_interaction.load_sqlite(filepath=self.caller.fenv.model.schema_base.sqlite_paths[0])
 
         if self.grid_selector.isChecked() == True:
             sqlite_test = SqliteCheck(self.caller.fenv)

hhnk_threedi_plugin/local_settings_default.py

@@ -3,8 +3,6 @@
 
 DEBUG=True
 
-hhnk_threedi_tools_path = ''   #None crashes.
-hhnk_research_tools_path = ''
 threeditoolbox_path = os.path.join(os.getenv('APPDATA'), r'3Di\QGIS3\profiles\default\python\plugins\ThreeDiToolbox\deps')
 
 project_path = None

hhnk_threedi_plugin/metadata.txt

@@ -6,7 +6,7 @@
 name=HHNK 3Di toolbox
 qgisMinimumVersion=3.0
 description=Plugin voor watersysteemanalyse met 3Di
-version=2023.2
+version=2023.3
 author=Wietse van Gerwen, HHNK
 [email protected]