Add filters, use poetry, better use of pyinstaller (#23)

* Add more filters

* Use poetry for dependencies

* Best practices for pyinstaller (not working yet)

* Convert things to relative imports

* Add extra parameters to Chameleon.spec

* Move config dir creation to app initial run

* Remove unneeded hidden imports

* Fix caching

.gitignore

@@ -32,3 +32,4 @@ test/output_name.csv
 main.spec
 output_test.csv
 pip-selfcheck.json
+poetry.lock

Chameleon.spec

@@ -36,15 +36,13 @@ else:
 
 
 a = Analysis(
-    ["chameleon/qt/qt.py"],
+    ["pyinstaller_entry.py"],
     pathex=[],
     binaries=[],
     datas=added_files,
     hiddenimports=[
-        "chameleon.qt.design",
         "cmath",
         "pandas._libs.tslibs.timedeltas",
-        "pytest",
     ],
     hookspath=[],
     runtime_hooks=[],
@@ -77,9 +75,11 @@ exe = EXE(
     upx=True,
     console=False,
     disable_windowed_traceback=False,
+    argv_emulation=False,
     target_arch=None,
     codesign_identity=None,
     icon="chameleon/resources/chameleon.icns" if is_mac else None,
+    entitlements_file=None,
 )
 if is_mac:
     coll = COLLECT(
@@ -89,6 +89,7 @@ if is_mac:
         a.datas,
         strip=False,
         upx=True,
+        upx_exclude=[],
         name="Chameleon",
     )
     app = BUNDLE(

chameleon/core.py

@@ -23,7 +23,6 @@
 import requests_cache
 import yaml
 from more_itertools import chunked as pager
-from requests_cache.backends import sqlite
 
 pd.options.mode.chained_assignment = None
 
@@ -153,7 +152,7 @@ def get_column_from_intermediate(column: str) -> None:
                 # If neither had one, we just won't include in the output
                 pass
 
-        for column in ("name", "highway"):
+        for column in ("name", "highway", "barrier"):
             if self.chameleon_mode != column:
                 get_column_from_intermediate(column)
 
@@ -241,6 +240,8 @@ def group(self) -> ChameleonDataFrame:
             new_column_order += ["name"]
         if self.chameleon_mode != "highway":
             new_column_order += ["highway"]
+        if self.chameleon_mode != "barrier":
+            new_column_order += ["barrier"]
         new_column_order += [
             f"old_{self.chameleon_mode_cleaned}",
             f"new_{self.chameleon_mode_cleaned}",
@@ -282,6 +283,22 @@ def filter(self) -> ChameleonDataFrame:
                 )
             ]
 
+        # Drop access changes that don't affect routing
+        if self.chameleon_mode == "access":
+            self = self[self["highway"].notnull() | self["barrier"].notnull()]
+            self = self[
+                ~(
+                    self["old_access"].isin(["yes", np.nan])
+                    & self["new_access"].isin(["yes", np.nan])
+                )
+            ]
+
+        if self.chameleon_mode == "barrier":
+            self = self[self.index.str.startswith("n")]
+
+        if self.chameleon_mode in ["construction", "name"]:
+            self = self[self["highway"].notnull()]
+
         # Drop rows with Kaart users tagged
         if whitelist := self.config.get("user_whitelist", []):
             self = self[~self["user"].isin(whitelist)]
@@ -378,11 +395,12 @@ def setup_cache(self) -> None:
             CACHE_LOCATION.mkdir(exist_ok=True, parents=True)
             expiry = timedelta(hours=12)
             self.session = requests_cache.CachedSession(
-                backend=sqlite.DbCache(
-                    use_cache_dir=str(CACHE_LOCATION / "cache"),
-                    expire_after=expiry,
-                )
+                backend=requests_cache.backends.sqlite.SQLiteCache(
+                    db_path=CACHE_LOCATION / "cache.sqlite",
+                ),
+                expire_after=expiry,
             )
+            self.session.remove_expired_responses()
             logger.debug("Request caching enabled")
         except (OSError, OperationalError):
             logger.error(

chameleon/qt/qt.py

@@ -44,13 +44,13 @@
 from requests import HTTPError, Timeout
 
 # Import generated UI file
-from chameleon.core import (
+from ..core import (
     HIGH_DELETIONS_THRESHOLD,
     OVERPASS_TIMEOUT,
     ChameleonDataFrame,
     ChameleonDataFrameSet,
 )
-from chameleon.qt import design, favorite_edit, filter_config
+from . import design, favorite_edit, filter_config
 
 # Differentiate sys settings between pre and post-bundling
 RESOURCES_DIR = (
@@ -239,14 +239,6 @@ def run(self):
             logger.debug("Worker thread successfully exposed to debugger.")
         try:  # Global exception catcher
             # Saving paths to config for future loading
-            # Make directory if it doesn't exist
-            if not CONFIG_DIR.is_dir():
-                try:
-                    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
-                except FileExistsError:
-                    logger.debug("Config directory already exists")
-                except OSError:
-                    logger.debug("Config directory could not be created.")
             if self.files:
                 self.history_writer()
             cdf_set = ChameleonDataFrameSet(
@@ -663,6 +655,14 @@ def __init__(self, parent=None):
         self.work_thread = None
         self.worker = None
 
+        # Make directory if it doesn't exist
+        try:
+            CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+        except FileExistsError:
+            logger.debug("Config directory already exists")
+        except OSError:
+            logger.debug("Config directory could not be created.")
+
         # Set up application logo on main window
         self.logo = str((RESOURCES_DIR / "chameleon.png").resolve())
         self.setWindowIcon(QIcon(self.logo))
@@ -1943,8 +1943,12 @@ def tag_split(raw_label: str) -> list[str]:
     return sorted(splitter)
 
 
-if __name__ == "__main__":
+def main():
     app = QApplication(sys.argv)
     form = MainApp()
     form.show()
     app.exec()
+
+
+if __name__ == "__main__":
+    main()

pyinstaller_entry.py

@@ -0,0 +1,3 @@
+from chameleon.qt.qt import main
+
+main()

pyproject.toml

@@ -0,0 +1,45 @@
+[tool.poetry]
+name = "chameleon"
+version = "3.6.2"
+description = ""
+authors = ["Evan Derickson <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.11"
+appdirs = "^1.4.4"
+geojson = "^2.5.0"
+more-itertools = "^8.14.0"
+overpass = { git = "https://github.com/KaartGroup/overpass-api-python-wrapper", branch = "kaart_addons" }
+pandas = "^1.4.3"
+PyYAML = "^6.0"
+requests = "^2.28.1"
+requests-cache = "^1.0.0a2"
+XlsxWriter = "^3.0.3"
+
+[tool.poetry.group.qt.dependencies]
+bidict = "^0.22.0"
+pyinstaller = "^5.3"
+PySide6 = "^6.3.1"
+
+
+[tool.poetry.group.web.dependencies]
+celery = "^5.2.7"
+Flask = "^2.2.2"
+gevent = "^21.12.0"
+gunicorn = "^20.1.0"
+redis = "^4.3.4"
+SQLAlchemy = "^1.4.40"
+Werkzeug = "^2.2.2"
+
+
+[tool.poetry.group.dev.dependencies]
+black = { version = "^22.6.0", allow-prereleases = true }
+GitPython = "^3.1.27"
+ptvsd = "^4.3.2"
+pytest = "^7.1.2"
+pytest-qt = "^4.1.0"
+requests-mock = { extras = ["fixture"], version = "^1.9.3" }
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"