Move retrieval of 'ETL_VERSION' to 'generate' commands and undo funct…

…ion signature changes

lambdas/commands.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Literal
 
 from lambdas import config, helpers
 
@@ -73,10 +72,10 @@ def generate_transform_commands(
     input_data: dict,
     run_date: str,
     timdex_bucket: str,
-    etl_version: Literal[1, 2],
 ) -> dict[str, list[dict]]:
     """Generate task run command for TIMDEX transform."""
     # NOTE: FEATURE FLAG: branching logic will be removed after v2 work is complete
+    etl_version = config.get_etl_version()
     match etl_version:
         case 1:
             return _etl_v1_generate_transform_commands_method(
@@ -120,14 +119,11 @@ def _etl_v2_generate_transform_commands_method() -> dict[str, list[dict]]:
 
 
 def generate_load_commands(
-    transform_output_files: list[str],
-    run_type: str,
-    source: str,
-    timdex_bucket: str,
-    etl_version: Literal[1, 2],
+    transform_output_files: list[str], run_type: str, source: str, timdex_bucket: str
 ) -> dict:
     """Generate task run command for loading records into OpenSearch."""
     # NOTE: FEATURE FLAG: branching logic will be removed after v2 work is complete
+    etl_version = config.get_etl_version()
     match etl_version:
         case 1:
             return _etl_v1_generate_load_commands_method(

lambdas/format_input.py

@@ -10,8 +10,6 @@
 def lambda_handler(event: dict, _context: dict) -> dict:
     """Format data into the necessary input for TIMDEX pipeline processing."""
     config.verify_env()
-    # NOTE: FEATURE FLAG: branching logic will be removed after v2 work is complete
-    etl_version = config.get_etl_version()
     verbose = config.check_verbosity(event.get("verbose", False))
     config.configure_logger(logging.getLogger(), verbose)
     logger.debug(json.dumps(event))
@@ -71,7 +69,7 @@ def lambda_handler(event: dict, _context: dict) -> dict:
 
         # NOTE: FEATURE FLAG: branching logic will be removed after v2 work is complete
         result["transform"] = commands.generate_transform_commands(
-            extract_output_files, event, run_date, timdex_bucket, etl_version
+            extract_output_files, event, run_date, timdex_bucket
         )
 
     elif next_step == "load":
@@ -91,7 +89,7 @@ def lambda_handler(event: dict, _context: dict) -> dict:
 
         # NOTE: FEATURE FLAG: branching logic will be removed after v2 work is complete
         result["load"] = commands.generate_load_commands(
-            transform_output_files, run_type, source, timdex_bucket, etl_version
+            transform_output_files, run_type, source, timdex_bucket
         )
 
     return result

tests/test_commands.py

@@ -88,7 +88,7 @@ def test_generate_transform_commands_required_input_fields():
         "testsource/testsource-2022-01-02-full-extracted-records-to-index.xml"
     ]
     assert commands.generate_transform_commands(
-        extract_output_files, input_data, "2022-01-02", "test-timdex-bucket", 1
+        extract_output_files, input_data, "2022-01-02", "test-timdex-bucket"
     ) == {
         "files-to-transform": [
             {
@@ -117,7 +117,7 @@ def test_generate_transform_commands_all_input_fields():
         "testsource/testsource-2022-01-02-daily-extracted-records-to-delete.xml",
     ]
     assert commands.generate_transform_commands(
-        extract_output_files, input_data, "2022-01-02", "test-timdex-bucket", 1
+        extract_output_files, input_data, "2022-01-02", "test-timdex-bucket"
     ) == {
         "files-to-transform": [
             {
@@ -158,7 +158,7 @@ def test_generate_load_commands_daily():
         "testsource/testsource-2022-01-02-daily-transformed-records-to-delete.txt",
     ]
     assert commands.generate_load_commands(
-        transform_output_files, "daily", "testsource", "test-timdex-bucket", 1
+        transform_output_files, "daily", "testsource", "test-timdex-bucket"
     ) == {
         "files-to-index": [
             {
@@ -200,7 +200,7 @@ def test_generate_load_commands_full_not_aliased():
         "testsource/testsource-2022-01-02-full-transformed-records-to-index.json"
     ]
     assert commands.generate_load_commands(
-        transform_output_files, "full", "testsource", "test-timdex-bucket", 1
+        transform_output_files, "full", "testsource", "test-timdex-bucket"
     ) == {
         "create-index-command": ["create", "--index", "testsource-2022-01-02t12-13-14"],
         "files-to-index": [
@@ -230,7 +230,7 @@ def test_generate_load_commands_full_aliased():
         "alma/alma-2022-01-02-full-transformed-records-to-index.json"
     ]
     assert commands.generate_load_commands(
-        transform_output_files, "full", "alma", "test-timdex-bucket", 1
+        transform_output_files, "full", "alma", "test-timdex-bucket"
     ) == {
         "create-index-command": ["create", "--index", "alma-2022-01-02t12-13-14"],
         "files-to-index": [
@@ -262,7 +262,7 @@ def test_generate_load_commands_full_with_deletes_logs_error(caplog):
         "alma/alma-2022-01-02-full-transformed-records-to-delete.txt"
     ]
     commands.generate_load_commands(
-        transform_output_files, "full", "alma", "test-timdex-bucket", 1
+        transform_output_files, "full", "alma", "test-timdex-bucket"
     )
     assert (
         "lambdas.commands",
@@ -277,5 +277,5 @@ def test_generate_load_commands_unexpected_input():
         "alma/alma-2022-01-02-full-transformed-records-to-index.json"
     ]
     assert commands.generate_load_commands(
-        transform_output_files, "wrong", "alma", "test-timdex-bucket", 1
+        transform_output_files, "wrong", "alma", "test-timdex-bucket"
     ) == {"failure": "Something unexpected went wrong. Please check input and try again."}