refactor!: organize project in source layout to make easily installab…

…le 🆕 (#45) * refactor: organize project in source layout to make easily installable * refactor: update installation of project * refator: add linting checks to build action * refactor: add pywinauto dependency back in * refactor: extend ruff to tools/
lxndrblz · Jan 9, 2024 · 5ed3d56 · 5ed3d56
1 parent e5eefab
commit 5ed3d56
Show file tree

Hide file tree

Showing 19 changed files with 260 additions and 124 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -15,7 +15,9 @@ jobs:
       - name: Install dependencies
         run: |
               python -m pip install --upgrade pip
-              python -m pip install -r requirements.txt pyinstaller
+              python -m pip install .[dev] pyinstaller
+      - name: Lint code 🦄
+        run: tox -e lint
       - name: Build binary 🔢
         run:  pyinstaller "main.spec"
       - name: Run conversion ↩️
@@ -24,9 +26,10 @@ jobs:
              .\dist\ms_teams_parser.exe -f ".\forensicsim-data\john_doe_v_1_4_00_11161\IndexedDB\https_teams.microsoft.com_0.indexeddb.leveldb" -o "john_doe.json"
       - name: Test calling script 📞
         run: |
-              python utils/dump_leveldb.py --help
-              python utils/dump_localstorage.py --help
-              python utils/dump_sessionstorage.py --help
+              python tools/main.py --help
+              python tools/dump_leveldb.py --help
+              python tools/dump_localstorage.py --help
+              python tools/dump_sessionstorage.py --help
 #              python utils/populate_teams.py --help
 #              python utils/populate_teams_2.py --help
 #              python utils/populate_skype.py --help

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -16,8 +16,8 @@ jobs:
     - name: Install dependencies
       run: |
             python -m pip install --upgrade pip
-            python -m pip install -r requirements.txt pyinstaller
-    - name: Build binary 🚧
+            python -m pip install . pyinstaller
+    - name: Build binary
       run:  pyinstaller "main.spec"
     - name: Zip files 🗜️
       run: |

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,5 +17,8 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.1.11
     hooks:
+    - id: ruff
+      args:
+      - --fix
     - id: ruff-format
 exclude: "^(export|populationdata|testdata)"
diff --git a/README.md b/README.md
@@ -121,7 +121,6 @@ as following. Simply specify the path to the database and where you want to outp
 
 usage: dump_leveldb.py [-h] -f FILEPATH -o OUTPUTPATH
 dump_leveldb.py: error: the following arguments are required: -f/--filepath, -o/--outputpath
-
 ```
 ---
 
@@ -131,24 +130,23 @@ dump_leveldb.py: error: the following arguments are required: -f/--filepath, -o/
 
 A wee script for populating *Skype for Desktop* in a lab environment. The script can be used like this:
 
-```
-utils\populate_skype.py -a 0 -f conversation.json
+```bash
+tools\populate_skype.py -a 0 -f conversation.json
 ```
 
 ## populate_teams.py
 
 A wee script for populating *Microsoft Teams* in a lab environment. The script can be used like this:
 
-```
-utils\populate_teams.py -a 0 -f conversation.json
+```bash
+tools\populate_teams.py -a 0 -f conversation.json
 ```
 
 ---
 # Datasets
 This repository comes with two datasets that allow reproducing the findings of this work. The `testdata` folder contains the *LevelDB* databases that have been extracted from two test clients. These can be used for benchmarking without having to perform a (lengthy) data population.
 
 The `populationdata` contains *JSON* files of the communication that has been populated into the testing environment. These can be used to reproduce the experiment from scratch. However, for a rerun, it will be essential to adjust the dates to future dates, as the populator script relies on sufficient breaks between the individual messages.
-
 ---
 
 # Acknowledgements & Thanks

diff --git a/main.spec b/main.spec
@@ -3,7 +3,7 @@
 block_cipher = None
 
 
-a = Analysis(['utils\\main.py'],
+a = Analysis(['tools\\main.py'],
              binaries=[],
              datas=[],
              hiddenimports=[],

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,114 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+
+[project]
+name = "forensicsim"
+description = "A forensic open-source parser module for Autopsy that allows extracting the messages, comments, posts, contacts, calendar entries and reactions from a Microsoft Teams IndexedDB LevelDB database."
+readme = "README.md"
+license = {file = "LICENSE.md"}
+requires-python = ">=3.9"
+authors = [
+    { name = "Alexander Bilz", email = "[email protected]" },
+    { name = "Markus Bilz", email = "[email protected]" }
+]
+dependencies = [
+"beautifulsoup4~=4.9.3",
+"click~=8.0.1",
+"chromedb @ git+https://github.com/karelze/ccl_chrome_indexeddb@master",
+"pause~=0.3",
+"pyautogui~=0.9.54",
+"pywinauto~=0.6.8"
+]
+
+dynamic = ["version"]
+
+[tool.setuptools.dynamic]
+version = {attr = "forensicsim.__version__"}
+
+[project.urls]
+"Homepage" = "https://forensics.im/"
+"Bug Tracker" = "https://github.com/lxndrblz/forensicsim/issues"
+
+[project.optional-dependencies]
+dev=[
+    "build",
+    "pre-commit",
+    "ruff",
+    "tox",
+]
+
+
+[tool.ruff]
+
+target-version = "py39"
+
+# See rules: https://beta.ruff.rs/docs/rules/
+select = [
+    "C",    # flake8-comprehensions
+    "F",    # pyflakes
+    "FURB", # refurb
+    "I",    # isort
+    "PIE",  # misc lints
+    "PT",   # pytest
+    "PGH",  # pygrep
+    "RUF",  # ruff-specific rules
+    "UP",   # pyupgrade
+    "SIM",  # flake8-simplify
+]
+
+include = ["*.py", "*.pyi", "**/pyproject.toml"]
+
+ignore = [
+    "C901",  # too complex
+    "E501",  # line too long, handled by black
+    "D206",  # indent with white space
+    "W191",  # tab identation
+]
+
+[tool.ruff.lint]
+preview = true
+# exclude = ["tools/**.py"]
+
+[tool.ruff.format]
+preview = true
+
+[tool.ruff.isort]
+known-first-party = ["forensicsim"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+
+[tool.tox]
+legacy_tox_ini = """
+
+[tox]
+envlist = format, lint, pre-commit
+skipdist = True
+isolated_build = True
+
+[testenv]
+deps = .[dev]
+
+# Cleanup tasks
+[testenv:clean]
+commands =
+    sh -c "rm -rf build cover dist .hypothesis .mypy_cache .pytest_cache site"
+
+# Auto Formatting
+[testenv:format]
+commands =
+    python -m ruff src tests --fix
+    python -m ruff format src
+
+# Syntax Checks
+[testenv:lint]
+commands =
+    python -m ruff --output-format=github src
+    python -m ruff format src --check
+
+# Pre-Commit
+[testenv:pre-commit]
+commands =
+    python -m pre-commit run --all-files --show-diff-on-failure
+
+"""
diff --git a/requirements.txt b/requirements.txt
diff --git a/src/forensicsim/__init__.py b/src/forensicsim/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.5.3"
diff --git a/utils/shared.py → src/forensicsim/backend.py b/utils/shared.py → src/forensicsim/backend.py
@@ -29,10 +29,10 @@
 from chromedb import (
     ccl_blink_value_deserializer,
     ccl_chromium_indexeddb,
-    ccl_v8_value_deserializer,
-    ccl_leveldb,
     ccl_chromium_localstorage,
     ccl_chromium_sessionstorage,
+    ccl_leveldb,
+    ccl_v8_value_deserializer,
 )
 from chromedb.ccl_chromium_indexeddb import (
     DatabaseMetadataType,
@@ -77,19 +77,18 @@ def fetch_data(self):
             if (
                 record.key.startswith(b"\x00\x00\x00\x00")
                 and record.state == ccl_leveldb.KeyState.Live
+            ) and (
+                record.key not in global_metadata_raw
+                or global_metadata_raw[record.key].seq < record.seq
             ):
-                if (
-                    record.key not in global_metadata_raw
-                    or global_metadata_raw[record.key].seq < record.seq
-                ):
-                    global_metadata_raw[record.key] = record
+                global_metadata_raw[record.key] = record
 
         # Convert the raw metadata to a nice GlobalMetadata Object
         global_metadata = ccl_chromium_indexeddb.GlobalMetadata(global_metadata_raw)
 
         # Loop through the database IDs
         for db_id in global_metadata.db_ids:
-            if None == db_id.dbid_no:
+            if db_id.dbid_no == None:
                 continue
 
             if db_id.dbid_no > 0x7F:
@@ -130,9 +129,11 @@ def fetch_data(self):
 
                     meta_type = record.key[len(prefix_objectstore) + len(varint_raw)]
 
-                    old_version = objectstore_metadata_raw.get(
-                        (db_id.dbid_no, objstore_id, meta_type)
-                    )
+                    old_version = objectstore_metadata_raw.get((
+                        db_id.dbid_no,
+                        objstore_id,
+                        meta_type,
+                    ))
 
                     if old_version is None or old_version.seq < record.seq:
                         objectstore_metadata_raw[
@@ -160,7 +161,7 @@ def iterate_records(self, do_not_filter=False):
         # Loop through the databases and object stores based on their ids
         for global_id in self.global_metadata.db_ids:
             # print(f"Processing database: {global_id.name}")
-            if None == global_id.dbid_no:
+            if global_id.dbid_no == None:
                 print(f"WARNING: Skipping database {global_id.name}")
                 continue
 
@@ -188,7 +189,7 @@ def iterate_records(self, do_not_filter=False):
                             if record.value == b"":
                                 continue
                             (
-                                value_version,
+                                _value_version,
                                 varint_raw,
                             ) = ccl_chromium_indexeddb.le_varint_from_bytes(
                                 record.value
@@ -201,7 +202,7 @@ def iterate_records(self, do_not_filter=False):
                             val_idx += 1
 
                             (
-                                blink_version,
+                                _blink_version,
                                 varint_raw,
                             ) = ccl_chromium_indexeddb.le_varint_from_bytes(
                                 record.value[val_idx:]
@@ -226,7 +227,7 @@ def iterate_records(self, do_not_filter=False):
                                     "state": record.state,
                                     "seq": record.seq,
                                 }
-                            except Exception as e:
+                            except Exception:
                                 # TODO Some proper error handling wouldn't hurt
                                 continue
                 # print(f"{datastore} {global_id.name} {records_per_object_store}")
@@ -280,7 +281,7 @@ def write_results_to_json(data, outputpath):
             json.dump(
                 data, f, indent=4, sort_keys=True, default=str, ensure_ascii=False
             )
-    except EnvironmentError as e:
+    except OSError as e:
         print(e)
 
 
@@ -290,5 +291,5 @@ def parse_json():
         with open("teams.json") as json_file:
             data = json.load(json_file)
             return data
-    except EnvironmentError as e:
+    except OSError as e:
         print(e)
diff --git a/utils/consts.py → src/forensicsim/consts.py b/utils/consts.py → src/forensicsim/consts.py
diff --git a/utils/main.py → src/forensicsim/parser.py b/utils/main.py → src/forensicsim/parser.py
@@ -26,11 +26,9 @@
 from datetime import datetime
 from pathlib import Path
 
-import click
 from bs4 import BeautifulSoup
 
-from shared import parse_db, write_results_to_json
-from consts import XTRACT_HEADER
+from forensicsim.backend import parse_db, write_results_to_json
 
 MESSAGE_TYPES = {
     "messages": {
@@ -253,7 +251,7 @@ def parse_reply_chain(reply_chains):
                         # Other types include ThreadActivity/TopicUpdate and ThreadActivity/AddMember
                         # -> ThreadActivity/TopicUpdate occurs for meeting updates
                         # -> ThreadActivity/AddMember occurs when someone gets added to a chat
-                    except UnicodeDecodeError or KeyError or NameError as e:
+                    except UnicodeDecodeError or KeyError or NameError:
                         print(
                             "Could not decode the following item in the reply chain (output is not deduplicated)."
                         )
@@ -273,19 +271,18 @@ def parse_conversations(conversations):
             # Include file origin for records
             x["origin_file"] = conversation["origin_file"]
             # Make first at sure that the conversation has a cachedDeduplicationKey
-            if "lastMessage" in conversation["value"]:
+            if "lastMessage" in conversation["value"]:  # noqa: SIM102
                 if (
                     hasattr(conversation["value"]["lastMessage"], "keys")
-                    and "cachedDeduplicationKey"
-                    in conversation["value"]["lastMessage"].keys()
+                    and "cachedDeduplicationKey" in conversation["value"]["lastMessage"]
                 ):
                     x["cachedDeduplicationKey"] = conversation["value"]["lastMessage"][
                         "cachedDeduplicationKey"
                     ]
                     # we are only interested in meetings for now
-                    if x["type"] == "Meeting":
+                    if x["type"] == "Meeting":  # noqa: SIM102
                         # assign the type for further processing as the object store might not be sufficient
-                        if "threadProperties" in x:
+                        if "threadProperties" in x:  # noqa: SIM102
                             if "meeting" in x["threadProperties"]:
                                 x["threadProperties"]["meeting"] = decode_and_loads(
                                     x["threadProperties"]["meeting"]
@@ -345,29 +342,3 @@ def process_db(input_path: Path, output_path: Path):
     extracted_values = parse_db(input_path)
     parsed_records = parse_records(extracted_values)
     write_results_to_json(parsed_records, output_path)
-
-
-@click.command()
-@click.option(
-    "-f",
-    "--filepath",
-    type=click.Path(
-        exists=True, readable=True, writable=False, dir_okay=True, path_type=Path
-    ),
-    required=True,
-    help="File path to the IndexedDB.",
-)
-@click.option(
-    "-o",
-    "--outputpath",
-    type=click.Path(writable=True, path_type=Path),
-    required=True,
-    help="File path to the processed output.",
-)
-def process_cmd(filepath, outputpath):
-    click.echo(XTRACT_HEADER)
-    process_db(filepath, outputpath)
-
-
-if __name__ == "__main__":
-    process_cmd()