Merge pull request #670 from bcgov/feat/task-225-debezium-upgrade

Feat/task 225 debezium upgrade
bcgov · Apr 17, 2024 · 70d0772 · 70d0772
2 parents 2ca3d20 + ddd672b
commit 70d0772
Show file tree

Hide file tree

Showing 8 changed files with 9 additions and 8 deletions.
diff --git a/etl/debezium-jdbc/Dockerfile b/etl/debezium-jdbc/Dockerfile
@@ -5,9 +5,9 @@ WORKDIR /app/
 COPY ./ ./
 RUN mvn clean install
 
-RUN echo "DEBEZIUM_VERSION is $DEBEZIUM_VERSION"
+RUN echo "DEBEZIUM_VERSION is " ${DEBEZIUM_VERSION}
 
-FROM debezium/connect:1.9
+FROM debezium/connect:2.5
 
 ARG ORACLE_JDBC_VERSION
 ARG ORACLE_JDBC_MD5

diff --git a/etl/debezium-jdbc/lib/kafka-connect-jdbc-10.7.2.jar b/etl/debezium-jdbc/lib/kafka-connect-jdbc-10.7.2.jar
diff --git a/etl/debezium-jdbc/lib/kafka-connect-jdbc-5.3.1.jar b/etl/debezium-jdbc/lib/kafka-connect-jdbc-5.3.1.jar
diff --git a/etl/debezium-kafka/Dockerfile b/etl/debezium-kafka/Dockerfile
@@ -1,5 +1,5 @@
 
-FROM debezium/kafka:1.9
+FROM debezium/kafka:2.5
 
 
 COPY connect-log4j.properties /kafka/config/

diff --git a/etl/register-oracle-jdbc-sink-connector.json b/etl/register-oracle-jdbc-sink-connector.json
@@ -3,7 +3,7 @@
     "config": {
         "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
         "tasks.max": "1",   
-        "topics.regex": "dbserver1.public.(.*)",
+        "topics.regex": "epd.public.(.*)",
         "connection.url": "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCPS)(PORT=1543)(HOST=nrcdb01.bcgov))(CONNECT_DATA=(SERVICE_NAME=SD57387.NRS.BCGOV))(SECURITY=(ssl_server_cert_dn=\"CN=nrcdb01.bcgov\")))",
         "connection.user": "JAITHOMA",
         "connection.password": "xxxxx",

diff --git a/etl/register-postgres-source-connector.json b/etl/register-postgres-source-connector.json
@@ -11,14 +11,15 @@
     "database.server.id": "184054",      
     "database.server.name": "dbserver1",
     "table.whitelist": "public.*",
+    "topic.prefix": "epd",
     "transforms": "unwrap",
     "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
     "transforms.unwrap.drop.tombstones": "false",
     "include.schema.changes": "false",
     "time.precision.mode":"connect",
     "heartbeat.action.query": "SELECT 1;",
     "errors.retry.timeout": -1,
-    "errors.max.retries": "5",
+    "errors.max.retries": "-1",
     "retry.backoff.ms": "30000",
     "retriable.restart.connector.wait.ms": "30000"
   }

diff --git a/openshift/templates/etl/debezium-jdbc.build.yaml b/openshift/templates/etl/debezium-jdbc.build.yaml
@@ -55,7 +55,7 @@ parameters:
   displayName: Environment name
   description: The name of the environment, in build names and image stream tags
   required: true
-  value: dev  
+  value:  dev 
 - name: GIT_REPO_URL
   displayName: Git Repo URL
   description: The URL to your GIT repo.
@@ -80,7 +80,7 @@ parameters:
   displayName: Debezium Version
   description: Debezium image version.
   required: false
-  value: "1.9"
+  value: "2.5"
 - name: ORACLE_JDBC_VERSION
   displayName: Oracle JDBC Version
   description: Oracle JDBC Version.

diff --git a/openshift/templates/etl/debezium-postgis.build.yaml b/openshift/templates/etl/debezium-postgis.build.yaml
@@ -46,7 +46,7 @@ parameters:
   displayName: Git Reference
   description: The git reference or branch.
   required: true
-  value: dev
+  value:  dev
 - name: SOURCE_CONTEXT_DIR
   displayName: Source Context Directory
   description: The source context directory.