OZ-68: Fixed broken StreamJob and updated tests to only load files fo…

…r a single domain entity
ozone-his · May 15, 2024 · 894d644 · 894d644
1 parent 7dcd5ea
commit 894d644
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -74,7 +74,7 @@ export ANALYTICS_DESTINATION_TABLES_MIGRATIONS_PATH= path_to_folder_containing_l
 export ANALYTICS_DB_HOST=gateway.docker.internal; \
 export ANALYTICS_DB_PORT=5432; \
 export CONNECT_MYSQL_HOSTNAME=gateway.docker.internal; \
-export CONNECT_MYSQL_PORT=3307; \
+export CONNECT_MYSQL_PORT=3306; \
 export CONNECT_MYSQL_USER=root; \
 export CONNECT_MYSQL_PASSWORD=3cY8Kve4lGey; \
 export CONNECT_ODOO_DB_HOSTNAME=gateway.docker.internal; \

diff --git a/src/test/java/com/ozonehis/data/pipelines/BaseJobTest.java b/src/test/java/com/ozonehis/data/pipelines/BaseJobTest.java
@@ -14,6 +14,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
@@ -168,22 +169,21 @@ private void setupConfig() throws IOException {
         JdbcSinkConfig jdbcSinkCfg = new JdbcSinkConfig();
         jdbcSinkCfg.setJdbcCatalog(catalogName);
         jdbcSinkCfg.setDatabaseName(BaseTestDatabase.DB_NAME_ANALYTICS);
-        jdbcSinkCfg.setQueryPath(BaseJobTest.class
-                .getClassLoader()
-                .getResource("dsl/flattening/queries")
-                .getPath());
+        final String flattenQueryPath = testDir + "dsl/flattening/queries";
+        Files.createDirectories(Paths.get(flattenQueryPath));
+        addTestFile(getTestFilename() + ".sql", getResourcePath("dsl/flattening/queries"), flattenQueryPath);
+        jdbcSinkCfg.setQueryPath(flattenQueryPath);
         config.setJdbcSinks(List.of(jdbcSinkCfg));
         FileSinkConfig fileSinkCfg = new FileSinkConfig();
-        fileSinkCfg.setDestinationTableDefinitionsPath(BaseJobTest.class
-                .getClassLoader()
-                .getResource("dsl/export/tables")
-                .getPath());
+        final String exportQueryPath = testDir + "dsl/export/queries";
+        Files.createDirectories(Paths.get(exportQueryPath));
+        addTestFile(getTestFilename() + ".sql", getResourcePath("dsl/export/queries"), exportQueryPath);
+        fileSinkCfg.setQueryPath(exportQueryPath);
+        final String exportTablePath = testDir + "dsl/export/tables";
+        addTestFile(getTestFilename() + ".sql", getResourcePath("dsl/export/tables"), exportTablePath);
+        fileSinkCfg.setDestinationTableDefinitionsPath(exportTablePath);
         fileSinkCfg.setFormat("json");
         fileSinkCfg.setExportOutPutTag("h1");
-        fileSinkCfg.setQueryPath(BaseJobTest.class
-                .getClassLoader()
-                .getResource("dsl/export/queries")
-                .getPath());
         fileSinkCfg.setExportOutputPath(exportDir);
         config.setFileSinks(List.of(fileSinkCfg));
         final String configFile = testDir + "/config.yaml";
@@ -270,6 +270,8 @@ protected boolean requiresSourceSchema() {
 
     protected abstract String getAnalyticsLiquibaseFile();
 
+    protected abstract String getTestFilename();
+
     private void deleteAllData(Connection connection, boolean disableKeys) throws SQLException {
         List<String> tables = getTableNames(connection);
         Statement statement = connection.createStatement();
@@ -299,4 +301,12 @@ private List<String> getTableNames(Connection connection) throws SQLException {
         }
         return tableNames;
     }
+
+    private void addTestFile(String file, String sourcePath, String destinationPath) throws IOException {
+        Files.copy(Paths.get(sourcePath, file), Paths.get(destinationPath, file));
+    }
+
+    private String getResourcePath(String name) {
+        return BaseJobTest.class.getClassLoader().getResource(name).getPath();
+    }
 }
diff --git a/src/test/java/com/ozonehis/data/pipelines/batch/PatientBatchJobTest.java b/src/test/java/com/ozonehis/data/pipelines/batch/PatientBatchJobTest.java
@@ -20,6 +20,11 @@
 
 public class PatientBatchJobTest extends BaseOpenmrsJobTest {
 
+    @Override
+    protected String getTestFilename() {
+        return "patients";
+    }
+
     @Test
     public void execute_shouldLoadAllPatientsFromOpenmrsDbToAnalyticsDb() throws Exception {
         addTestDataToSourceDb("openmrs/initial.sql");

diff --git a/src/test/java/com/ozonehis/data/pipelines/batch/SaleOrderLineBatchJobTest.java b/src/test/java/com/ozonehis/data/pipelines/batch/SaleOrderLineBatchJobTest.java
@@ -8,6 +8,11 @@
 
 public class SaleOrderLineBatchJobTest extends BaseOdooJobTest {
 
+    @Override
+    protected String getTestFilename() {
+        return "sale_order_lines";
+    }
+
     @Override
     protected boolean requiresSourceSchema() {
         return true;

diff --git a/src/test/java/com/ozonehis/data/pipelines/batch/SaleOrderLineExportJobTest.java b/src/test/java/com/ozonehis/data/pipelines/batch/SaleOrderLineExportJobTest.java
@@ -16,6 +16,11 @@
 
 public class SaleOrderLineExportJobTest extends BaseOdooJobTest {
 
+    @Override
+    protected String getTestFilename() {
+        return "sale_order_lines";
+    }
+
     @Test
     public void execute_shouldExportAllSaleOrderLinesFromAnalyticsDbToAFile() throws Exception {
         addTestDataToAnalyticsDb("sale_order_line.sql");