Support providing directory in file list (NVIDIA#262)

client/src/nv_ingest_client/util/util.py

@@ -300,14 +300,13 @@ def generate_matching_files(file_sources):
     It yields each matching file path, allowing for efficient processing of potentially large
     sets of files.
     """
-    files = [
-        file_path
-        for pattern in file_sources
-        for file_path in glob.glob(pattern, recursive=True)
-        if os.path.isfile(file_path)
-    ]
-    for file_path in files:
-        yield file_path
+    for pattern in file_sources:
+        if os.path.isdir(pattern):
+            pattern = os.path.join(pattern, "*")
+
+        for file_path in glob.glob(pattern, recursive=True):
+            if os.path.isfile(file_path):
+                yield file_path
 
 
 def create_job_specs_for_batch(files_batch: List[str]) -> List[JobSpec]:

tests/nv_ingest_client/util/test_util.py

@@ -89,6 +89,17 @@ def test_generate_matching_files(patterns, mock_files, expected):
         assert list(generate_matching_files(patterns)) == expected
 
 
+def test_generate_matching_directory():
+    patterns = ["docs"]
+    mock_files = ["docs/README.md", "docs/CHANGES.md"]
+    expected = ["docs/README.md", "docs/CHANGES.md"]
+    with patch("glob.glob", return_value=mock_files), patch(
+        "os.path.isfile", side_effect=lambda path: path in mock_files
+    ), patch("os.path.isdir", side_effect=lambda path: path == "docs"):
+
+        assert list(generate_matching_files(patterns)) == expected
+
+
 def test_filter_function_kwargs_with_matching_kwargs():
     def sample_func(a, b, c):
         pass