test: add module test, some linting issues

.github/workflows/build-linux.yaml

@@ -51,23 +51,22 @@ jobs:
 #      - name: coveralls
 #        uses: coverallsapp/github-action@v2
 
-# Uncomment this if you want to run mypy
-#  mypy-linux:
-#    name: mypy check on linux
-#    runs-on: ubuntu-latest
-#    strategy:
-#      matrix:
-#        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-#    steps:
-#      - name: "Set up Python ${{ matrix.python_version }}"
-#        uses: actions/setup-python@v4
-#        with:
-#          python-version: "${{ matrix.python_version }}"
-#      - uses: actions/checkout@v3
-#      - name: install kiara_plugin.tropy
-#        run: pip install -U .[dev_testing]
-#      - name: Test with mypy
-#        run: make mypy
+  mypy-linux:
+    name: mypy check on linux
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - name: "Set up Python ${{ matrix.python_version }}"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python_version }}"
+      - uses: actions/checkout@v3
+      - name: install kiara_plugin.tropy
+        run: pip install -U .[dev_testing]
+      - name: Test with mypy
+        run: make mypy
 
   linting-linux:
     runs-on: ubuntu-latest
@@ -96,8 +95,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - test-linux
-#      - mypy-linux
-#      - linting-linux
+      - mypy-linux
+      - linting-linux
     steps:
       - name: Set up Python 3.11
         uses: actions/setup-python@v4
@@ -138,8 +137,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - test-linux
-      # - mypy-linux   # uncomment if this step is enabled
-      # - linting-linux   # uncomment if this step is enabled
+      - mypy-linux   # uncomment if this step is enabled
+      - linting-linux   # uncomment if this step is enabled
     steps:
       - name: "Set up Python 3.11"
         uses: actions/setup-python@v4

examples/jobs/assemble_journals_graph_directed.yaml

@@ -0,0 +1,8 @@
+operation: ${this_dir}/../pipelines/assemble_network_graph_from_paths.yaml
+inputs:
+  graph_type: directed
+  edges_path: ${this_dir}/../data/journals/JournalEdges1902.csv
+  nodes_path: ${this_dir}/../data/journals/JournalNodes1902.csv
+  node_id_column: Id
+  source_column: Source
+  target_column: Target

examples/jobs/assemble_journals_graph_directed_multi.yaml

@@ -0,0 +1,8 @@
+operation: ${this_dir}/../pipelines/assemble_network_graph_from_paths.yaml
+inputs:
+  graph_type: directed_multi
+  edges_path: ${this_dir}/../data/journals/JournalEdges1902.csv
+  nodes_path: ${this_dir}/../data/journals/JournalNodes1902.csv
+  node_id_column: Id
+  source_column: Source
+  target_column: Target

examples/jobs/assemble_journals_graph_undirected_multi.yaml

@@ -0,0 +1,8 @@
+operation: ${this_dir}/../pipelines/assemble_network_graph_from_paths.yaml
+inputs:
+  graph_type: undirected_multi
+  edges_path: ${this_dir}/../data/journals/JournalEdges1902.csv
+  nodes_path: ${this_dir}/../data/journals/JournalNodes1902.csv
+  node_id_column: Id
+  source_column: Source
+  target_column: Target

examples/jobs/init.yaml

@@ -0,0 +1,12 @@
+operation: "${this_dir}/../pipelines/init.yaml"
+inputs:
+  import_edges_table__path: "${this_dir}/../data/journals/JournalEdges1902.csv"
+  import_nodes_table__path: "${this_dir}/../data/journals/JournalNodes1902.csv"
+  assemble_network_graph__graph_type: "directed"
+  assemble_network_graph__node_id_column: "Id"
+  assemble_network_graph__source_column: "Source"
+  assemble_network_graph__target_column: "Target"
+save:
+  assemble_network_graph__network_graph: "journals_graph_directed"
+  import_edges_table__table: "journals_edges_table"
+  import_nodes_table__table: "journals_nodes_table"

examples/pipelines/init.yaml

@@ -0,0 +1,18 @@
+pipeline_name: init_tropy_plugin_data
+doc: Initialize some data to be used with example jobs.
+steps:
+  - module_type: import.table.from.local_file_path
+    step_id: import_edges_table
+    module_config:
+      constants:
+        first_row_is_header: true
+  - module_type: import.table.from.local_file_path
+    step_id: import_nodes_table
+    module_config:
+      constants:
+        first_row_is_header: true
+  - module_type: assemble.network_graph
+    step_id: assemble_network_graph
+    input_links:
+      edges: import_edges_table.table
+      nodes: import_nodes_table.table

pyproject.toml

@@ -172,7 +172,7 @@ select = [
     "PLW",
     "PIE",
 ]
-ignore = ["E501", "S101", "SIM118", "SIM108", "PLR2004", "PLR0913", "S110", "PLR0912", "PLR0915", "PIE810", "S608"]
+ignore = ["E501", "S101", "SIM118", "SIM108", "PLR2004", "PLR0913", "S110", "PLR0912", "PLR0915", "PIE810", "S608", "PD901"]
 
 fix = true
 fixable = ["E", "F", "RUF100", "I001", "Q"]
@@ -243,6 +243,7 @@ plugins = [
 module = [
     "appdirs.*",
     "pyarrow.*",
-    "ruamel.*"
+    "ruamel.*",
+    "networkx.*"
 ]
 ignore_missing_imports = true

src/kiara_plugin/tropy/data_types.py

@@ -39,15 +39,15 @@ def python_class(cls) -> Type:
 
     def parse_python_obj(self, data: Any) -> NetworkGraph:
 
-        return data
+        return data  # type: ignore
 
     def _validate(cls, value: Any) -> None:
         if not isinstance(value, NetworkGraph):
             raise ValueError(
                 f"Invalid type '{type(value)}': must be of 'NetworkGraph' (or a sub-class)."
             )
 
-    def serialize(self, data: NetworkGraph) -> Union[None, str, "SerializedData"]:
+    def serialize(self, data: NetworkGraph) -> Union[None, str, "SerializedData"]:  # type: ignore
 
         import pyarrow as pa
 

src/kiara_plugin/tropy/models.py

@@ -284,10 +284,10 @@ class GraphProperties(BaseModel):
 class NetworkGraphProperties(ValueMetadata):
     """Network data stats."""
 
-    _metadata_key: ClassVar[str] = "network_data"
+    _metadata_key: ClassVar[str] = "network_graph"
 
     number_of_nodes: int = Field(description="Number of nodes in the network graph.")
-    num_edges: int = Field(description="Number of edges in the network graph.")
+    number_of_edges: int = Field(description="Number of edges in the network graph.")
 
     @classmethod
     def retrieve_supported_data_types(cls) -> Iterable[str]:
@@ -296,13 +296,14 @@ def retrieve_supported_data_types(cls) -> Iterable[str]:
     @classmethod
     def create_value_metadata(cls, value: Value) -> "NetworkGraphProperties":
 
-        network_data: NetworkGraph = value.data
+        network_graph: NetworkGraph = value.data
 
-        num_rows = network_data.num_nodes
-        num_edges = network_data.num_edges
+        graph = network_graph.as_networkx_graph()
+        num_rows = graph.number_of_nodes()
+        num_edges = len(graph.edges())
 
         result = cls(
             number_of_nodes=num_rows,
-            num_edges=num_edges,
+            number_of_edges=num_edges,
         )
         return result

src/kiara_plugin/tropy/modules/centrality_measures_module.py

@@ -2,6 +2,7 @@
 from operator import itemgetter
 
 from kiara.api import KiaraModule
+from kiara.models.values.value import ValueMap
 
 KIARA_METADATA = {
     "authors": [
@@ -51,7 +52,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         import networkx as nx
         import pandas as pd
@@ -96,7 +97,7 @@ def result_func(list):
 
         df = pd.DataFrame(sorted_dict, columns=["Rank", "Node", "Degree"])
 
-        if wd == True:
+        if wd is True:
             if weight_name == "":
                 MG = network_data.as_networkx_graph()
 
@@ -194,7 +195,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         import networkx as nx
         import pandas as pd
@@ -236,15 +237,15 @@ def result_func(list):
         ]
 
         df = pd.DataFrame(sorted_dict)
-        df.columns = ["Rank", "Node", "Score"]
+        df.columns = pd.Index(["Rank", "Node", "Score"])
 
-        if wd == True:
+        if wd is True:
             graph = network_data.as_networkx_graph()
             edge_weight = nx.get_edge_attributes(graph, weight_name)
             for u, v, key in edge_weight:
                 nx.set_edge_attributes(graph, edge_weight, "weight")
 
-            if wm == True:
+            if wm is True:
                 for u, v, d in graph.edges(data=True):
                     d["weight"] == (1 / d["weight"])
 
@@ -307,7 +308,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         import networkx as nx
         import pandas as pd
@@ -350,15 +351,15 @@ def result_func(list):
         ]
 
         df = pd.DataFrame(sorted_dict)
-        df.columns = ["Rank", "Node", "Score"]
+        df.columns = pd.Index(["Rank", "Node", "Score"])
 
-        if wd == True:
+        if wd is True:
             graph = network_data.as_networkx_graph()
             edge_weight = nx.get_edge_attributes(graph, weight_name)
             for u, v, key in edge_weight:
                 nx.set_edge_attributes(graph, edge_weight, "weight")
 
-            if wm == False:
+            if wm is False:
                 for u, v, d in graph.edges(data=True):
                     d["weight"] == (1 / d["weight"])
 
@@ -422,7 +423,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         import networkx as nx
         import pandas as pd
@@ -464,15 +465,15 @@ def result_func(list):
         ]
 
         df = pd.DataFrame(sorted_dict)
-        df.columns = ["Rank", "Node", "Score"]
+        df.columns = pd.Index(["Rank", "Node", "Score"])
 
-        if wd == True:
+        if wd is True:
             graph = network_data.as_networkx_graph()
             edge_weight = nx.get_edge_attributes(graph, weight_name)
             for u, v, key in edge_weight:
                 nx.set_edge_attributes(graph, edge_weight, "weight")
 
-            if wm == True:
+            if wm is True:
                 for u, v, d in graph.edges(data=True):
                     d["weight"] == (1 / d["weight"])
 

src/kiara_plugin/tropy/modules/cutpoints_module.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from kiara.api import KiaraModule
+from kiara.models.values.value import ValueMap
 
 KIARA_METADATA = {
     "authors": [
@@ -38,7 +39,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         from kiara_plugin.tropy.models import NetworkGraph
 

src/kiara_plugin/tropy/modules/modularity.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from kiara.api import KiaraModule, ValueMapSchema
+from kiara.models.values.value import ValueMap
 
 KIARA_METADATA = {
     "authors": [
@@ -49,7 +50,7 @@ def create_outputs_schema(self):
             },
         }
 
-    def process(self, inputs, outputs):
+    def process(self, inputs: ValueMap, outputs: ValueMap):
 
         import networkx as nx
         from networkx.algorithms import community

tests/job_tests/assemble_journals_graph_directed/outputs.yaml

@@ -0,0 +1,4 @@
+network_graph::properties::metadata.tables::tables::edges::rows: 321
+network_graph::properties::metadata.tables::tables::nodes::rows: 276
+network_graph::properties::metadata.network_graph::number_of_nodes: 276
+network_graph::properties::metadata.network_graph::number_of_edges: 321

tests/job_tests/assemble_journals_graph_directed_multi/outputs.yaml

@@ -0,0 +1,4 @@
+network_graph::properties::metadata.tables::tables::edges::rows: 321
+network_graph::properties::metadata.tables::tables::nodes::rows: 276
+network_graph::properties::metadata.network_graph::number_of_nodes: 276
+network_graph::properties::metadata.network_graph::number_of_edges: 321

tests/job_tests/assemble_journals_graph_undirected/outputs.yaml

@@ -0,0 +1,4 @@
+network_graph::properties::metadata.tables::tables::edges::rows: 321
+network_graph::properties::metadata.tables::tables::nodes::rows: 276
+network_graph::properties::metadata.network_graph::number_of_nodes: 276
+network_graph::properties::metadata.network_graph::number_of_edges: 313

tests/job_tests/assemble_journals_graph_undirected_multi/outputs.yaml

@@ -0,0 +1,4 @@
+network_graph::properties::metadata.tables::tables::edges::rows: 321
+network_graph::properties::metadata.tables::tables::nodes::rows: 276
+network_graph::properties::metadata.network_graph::number_of_nodes: 276
+network_graph::properties::metadata.network_graph::number_of_edges: 321