diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6b3e0a4730..9a458299dc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -68,8 +68,7 @@ }, "forwardPorts": [ 2432, 3000, 3306, 3333, 4200, 4211, 5000, 5200, 5432, 5601, 7010, 7443, 7200, 7888, 8010, 8071, - 8000, 8080, 8081, 8082, 8084, 8085, 8086, 8090, 8200, 8443, 8888, 8889, 9090, 9104, 9200, 9411, - 27017, 18080 + 8000, 8080, 8081, 8082, 8084, 8085, 8086, 8090, 8200, 8888, 8889, 9090, 9104, 9200, 9411, 27017 ], "portsAttributes": { "2432": { @@ -164,10 +163,6 @@ "label": "openchallenges-config-server", "onAutoForward": "silent" }, - "8443": { - "label": "openchallenges-nifi", - "onAutoForward": "silent" - }, "8888": { "label": "openchallenges-notebook", "onAutoForward": "silent" @@ -195,10 +190,6 @@ "27017": { "label": "agora-mongo", "onAutoForward": "silent" - }, - "18080": { - "label": "openchallenges-nifi-registry", - "onAutoForward": "silent" } }, "remoteUser": "vscode", diff --git a/apps/openchallenges/challenge-service/src/main/resources/db/migration/V1.0.0__create_tables.sql b/apps/openchallenges/challenge-service/src/main/resources/db/migration/V1.0.0__create_tables.sql index f8819ad45e..0a1bac2017 100644 --- a/apps/openchallenges/challenge-service/src/main/resources/db/migration/V1.0.0__create_tables.sql +++ b/apps/openchallenges/challenge-service/src/main/resources/db/migration/V1.0.0__create_tables.sql @@ -123,9 +123,3 @@ CREATE TABLE `challenge_category` ( PRIMARY KEY (`id`), FOREIGN KEY (`challenge_id`) REFERENCES challenge (`id`) ); - -CREATE TABLE `nifi_demo` ( - `id` int NOT NULL AUTO_INCREMENT, - `name` varchar(120), - PRIMARY KEY (`id`) -); diff --git a/apps/openchallenges/nifi-registry/.env.example b/apps/openchallenges/nifi-registry/.env.example deleted file mode 100644 index a17418ab2c..0000000000 --- a/apps/openchallenges/nifi-registry/.env.example +++ /dev/null @@ -1 +0,0 @@ -LOG_LEVEL=INFO \ No newline at end of file diff --git a/apps/openchallenges/nifi-registry/Dockerfile b/apps/openchallenges/nifi-registry/Dockerfile deleted file mode 100644 index d5597ace44..0000000000 --- a/apps/openchallenges/nifi-registry/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM apache/nifi-registry:2.0.0 - diff --git a/apps/openchallenges/nifi-registry/README.md b/apps/openchallenges/nifi-registry/README.md deleted file mode 100644 index f65c2daa33..0000000000 --- a/apps/openchallenges/nifi-registry/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# OpenChallenges NiFi Registry - -## Build the Docker image - -```console -nx build-image openchallenges-nifi-registry -``` - -## Deploy the registry with Docker Compose - -```console -nx serve-detach openchallenges-nifi-registry -``` - -## Access the registry UI - -In your browser, navigate to http://localhost:18080/nifi-registry. - -## Create a bucket - -From the registry home page: - -1. Click on the button "Settings" in the toolbar. -2. Click on the button "NEW BUCKET". -3. Enter the bucket information. - - Name: openchallenges -4. Click on the button "CREATE". - -## Stop and remove the NiFi registry container - -```console -docker rm -f openchallenges-nifi-registry -``` - -## Data persistence - -The data created in the registry are stored in a Docker volume. diff --git a/apps/openchallenges/nifi-registry/nifi_demo-version-2.json b/apps/openchallenges/nifi-registry/nifi_demo-version-2.json deleted file mode 100644 index 9836566d26..0000000000 --- a/apps/openchallenges/nifi-registry/nifi_demo-version-2.json +++ /dev/null @@ -1,1634 +0,0 @@ -{ - "externalControllerServices": {}, - "flowContents": { - "comments": "", - "componentType": "PROCESS_GROUP", - "connections": [ - { - "backPressureDataSizeThreshold": "1 GB", - "backPressureObjectThreshold": 10000, - "bends": [], - "componentType": "CONNECTION", - "destination": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "fd86ff9c-c5c2-31ca-9aec-5d1419cc9455", - "instanceIdentifier": "c0755e5d-c0c0-3c01-e2b8-c8dfae3d7627", - "name": "PutDatabaseRecord", - "type": "PROCESSOR" - }, - "flowFileExpiration": "0 sec", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "89233f07-e092-398d-b77b-88222396cb75", - "instanceIdentifier": "23f7959f-2d98-38f2-3f0b-283675a2b354", - "labelIndex": 0, - "loadBalanceCompression": "DO_NOT_COMPRESS", - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "name": "", - "partitioningAttribute": "", - "prioritizers": [], - "selectedRelationships": ["splits"], - "source": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "93ffd07d-2ddd-3dc5-a3d9-a68a75e6dc04", - "instanceIdentifier": "c08f8e4a-3778-3559-1c72-b8c02757a135", - "name": "SplitRecord", - "type": "PROCESSOR" - }, - "zIndex": 0 - }, - { - "backPressureDataSizeThreshold": "1 GB", - "backPressureObjectThreshold": 10000, - "bends": [], - "componentType": "CONNECTION", - "destination": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "93ffd07d-2ddd-3dc5-a3d9-a68a75e6dc04", - "instanceIdentifier": "c08f8e4a-3778-3559-1c72-b8c02757a135", - "name": "SplitRecord", - "type": "PROCESSOR" - }, - "flowFileExpiration": "0 sec", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "c956bda6-bf4c-3885-a26a-7204ac2bab2d", - "instanceIdentifier": "00b3c4ce-b707-37c8-7cac-68528d318922", - "labelIndex": 0, - "loadBalanceCompression": "DO_NOT_COMPRESS", - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "name": "", - "partitioningAttribute": "", - "prioritizers": [], - "selectedRelationships": ["success"], - "source": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "2a9918eb-0433-37bf-97ae-12528859b99a", - "instanceIdentifier": "0f8ccffe-0bd3-3bda-5247-2ddf92337fe2", - "name": "GetCsvFile", - "type": "PROCESSOR" - }, - "zIndex": 0 - }, - { - "backPressureDataSizeThreshold": "1 GB", - "backPressureObjectThreshold": 10000, - "bends": [], - "componentType": "CONNECTION", - "destination": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "b63dfd38-939b-3a62-bd0f-ed2cc6e20674", - "instanceIdentifier": "ce2a1b66-97ab-3834-7ddd-b49341646684", - "name": "SetFilename", - "type": "PROCESSOR" - }, - "flowFileExpiration": "0 sec", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "f3bba195-58a1-3449-9178-0dc4a316fe01", - "instanceIdentifier": "9a98ec9f-c14d-3ec4-de6f-3d6e7e655d64", - "labelIndex": 0, - "loadBalanceCompression": "DO_NOT_COMPRESS", - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "name": "", - "partitioningAttribute": "", - "prioritizers": [], - "selectedRelationships": ["Response"], - "source": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "9cb3fc8c-2a8a-3a99-b104-3100f0e67538", - "instanceIdentifier": "28407726-80c4-3511-94e4-980672f02894", - "name": "GetChallenges", - "type": "PROCESSOR" - }, - "zIndex": 0 - }, - { - "backPressureDataSizeThreshold": "1 GB", - "backPressureObjectThreshold": 10000, - "bends": [], - "componentType": "CONNECTION", - "destination": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "dec7ffa2-e5fa-3a2a-aae8-7635e7c237f3", - "instanceIdentifier": "a8f6a6e7-33c9-3e5c-a252-7d94372de811", - "name": "PutFile", - "type": "PROCESSOR" - }, - "flowFileExpiration": "0 sec", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "4a17bb7f-1517-347f-93e8-5261f8432b0c", - "instanceIdentifier": "851d3ee4-8682-3a13-a820-cc0c941eca95", - "labelIndex": 0, - "loadBalanceCompression": "DO_NOT_COMPRESS", - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "name": "", - "partitioningAttribute": "", - "prioritizers": [], - "selectedRelationships": ["success"], - "source": { - "comments": "", - "groupId": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "id": "b63dfd38-939b-3a62-bd0f-ed2cc6e20674", - "instanceIdentifier": "ce2a1b66-97ab-3834-7ddd-b49341646684", - "name": "SetFilename", - "type": "PROCESSOR" - }, - "zIndex": 0 - } - ], - "controllerServices": [ - { - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-dbcp-service-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "CONTROLLER_SERVICE", - "controllerServiceApis": [ - { - "bundle": { - "artifact": "nifi-standard-services-api-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "type": "org.apache.nifi.dbcp.DBCPService" - } - ], - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "6650fa38-df10-35d3-85d1-14b73493f348", - "instanceIdentifier": "ac5c061a-5cd5-3409-a2df-5c20c6ea2270", - "name": "DBCPConnectionPool", - "properties": { - "dbcp-min-idle-conns": "0", - "Max Wait Time": "500 millis", - "Database Driver Class Name": "com.mysql.jdbc.Driver", - "dbcp-min-evictable-idle-time": "30 mins", - "Max Total Connections": "8", - "dbcp-max-conn-lifetime": "-1", - "Database Connection URL": "jdbc:mysql://openchallenges-mariadb:3306/challenge_service?allowLoadLocalInfile=true", - "dbcp-time-between-eviction-runs": "-1", - "Database User": "challenge_service", - "dbcp-soft-min-evictable-idle-time": "-1", - "database-driver-locations": "/opt/nifi/nifi-current/drivers/mysql-connector-j-9.1.0.jar", - "dbcp-max-idle-conns": "8" - }, - "propertyDescriptors": { - "dbcp-min-idle-conns": { - "displayName": "Minimum Idle Connections", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-min-idle-conns", - "sensitive": false - }, - "Max Wait Time": { - "displayName": "Max Wait Time", - "dynamic": false, - "identifiesControllerService": false, - "name": "Max Wait Time", - "sensitive": false - }, - "Database Driver Class Name": { - "displayName": "Database Driver Class Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "Database Driver Class Name", - "sensitive": false - }, - "dbcp-min-evictable-idle-time": { - "displayName": "Minimum Evictable Idle Time", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-min-evictable-idle-time", - "sensitive": false - }, - "Max Total Connections": { - "displayName": "Max Total Connections", - "dynamic": false, - "identifiesControllerService": false, - "name": "Max Total Connections", - "sensitive": false - }, - "dbcp-max-conn-lifetime": { - "displayName": "Max Connection Lifetime", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-max-conn-lifetime", - "sensitive": false - }, - "Validation-query": { - "displayName": "Validation query", - "dynamic": false, - "identifiesControllerService": false, - "name": "Validation-query", - "sensitive": false - }, - "Database Connection URL": { - "displayName": "Database Connection URL", - "dynamic": false, - "identifiesControllerService": false, - "name": "Database Connection URL", - "sensitive": false - }, - "dbcp-time-between-eviction-runs": { - "displayName": "Time Between Eviction Runs", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-time-between-eviction-runs", - "sensitive": false - }, - "Database User": { - "displayName": "Database User", - "dynamic": false, - "identifiesControllerService": false, - "name": "Database User", - "sensitive": false - }, - "kerberos-user-service": { - "displayName": "Kerberos User Service", - "dynamic": false, - "identifiesControllerService": true, - "name": "kerberos-user-service", - "sensitive": false - }, - "dbcp-soft-min-evictable-idle-time": { - "displayName": "Soft Minimum Evictable Idle Time", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-soft-min-evictable-idle-time", - "sensitive": false - }, - "database-driver-locations": { - "displayName": "Database Driver Location(s)", - "dynamic": false, - "identifiesControllerService": false, - "name": "database-driver-locations", - "resourceDefinition": { - "cardinality": "MULTIPLE", - "resourceTypes": ["URL", "FILE", "DIRECTORY"] - }, - "sensitive": false - }, - "dbcp-max-idle-conns": { - "displayName": "Max Idle Connections", - "dynamic": false, - "identifiesControllerService": false, - "name": "dbcp-max-idle-conns", - "sensitive": false - }, - "Password": { - "displayName": "Password", - "dynamic": false, - "identifiesControllerService": false, - "name": "Password", - "sensitive": true - } - }, - "scheduledState": "DISABLED", - "type": "org.apache.nifi.dbcp.DBCPConnectionPool" - }, - { - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-record-serialization-services-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "CONTROLLER_SERVICE", - "controllerServiceApis": [ - { - "bundle": { - "artifact": "nifi-standard-services-api-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "type": "org.apache.nifi.serialization.RecordSetWriterFactory" - } - ], - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "87d7b1c2-fd34-3cbc-b558-86b49fe72758", - "instanceIdentifier": "0be2a6ce-d467-3209-626f-cde34b6b47cc", - "name": "CSVRecordSetWriter", - "properties": { - "CSV Format": "custom", - "Include Header Line": "true", - "Escape Character": "\\", - "Trim Fields": "true", - "schema-access-strategy": "inherit-record-schema", - "csvutils-character-set": "UTF-8", - "schema-text": "${avro.schema}", - "csv-writer": "commons-csv", - "Quote Character": "\"", - "Value Separator": ",", - "Record Separator": "\\n", - "Schema Write Strategy": "no-schema", - "schema-name": "${schema.name}", - "Quote Mode": "MINIMAL", - "Include Trailing Delimiter": "false" - }, - "propertyDescriptors": { - "schema-reference-reader": { - "displayName": "Schema Reference Reader", - "dynamic": false, - "identifiesControllerService": true, - "name": "schema-reference-reader", - "sensitive": false - }, - "schema-branch": { - "displayName": "Schema Branch", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-branch", - "sensitive": false - }, - "CSV Format": { - "displayName": "CSV Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "CSV Format", - "sensitive": false - }, - "Include Header Line": { - "displayName": "Include Header Line", - "dynamic": false, - "identifiesControllerService": false, - "name": "Include Header Line", - "sensitive": false - }, - "schema-cache": { - "displayName": "Schema Cache", - "dynamic": false, - "identifiesControllerService": true, - "name": "schema-cache", - "sensitive": false - }, - "Escape Character": { - "displayName": "Escape Character", - "dynamic": false, - "identifiesControllerService": false, - "name": "Escape Character", - "sensitive": false - }, - "Date Format": { - "displayName": "Date Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Date Format", - "sensitive": false - }, - "Null String": { - "displayName": "Null String", - "dynamic": false, - "identifiesControllerService": false, - "name": "Null String", - "sensitive": false - }, - "Trim Fields": { - "displayName": "Trim Fields", - "dynamic": false, - "identifiesControllerService": false, - "name": "Trim Fields", - "sensitive": false - }, - "schema-registry": { - "displayName": "Schema Registry", - "dynamic": false, - "identifiesControllerService": true, - "name": "schema-registry", - "sensitive": false - }, - "Time Format": { - "displayName": "Time Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Time Format", - "sensitive": false - }, - "Comment Marker": { - "displayName": "Comment Marker", - "dynamic": false, - "identifiesControllerService": false, - "name": "Comment Marker", - "sensitive": false - }, - "schema-access-strategy": { - "displayName": "Schema Access Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-access-strategy", - "sensitive": false - }, - "schema-version": { - "displayName": "Schema Version", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-version", - "sensitive": false - }, - "csvutils-character-set": { - "displayName": "Character Set", - "dynamic": false, - "identifiesControllerService": false, - "name": "csvutils-character-set", - "sensitive": false - }, - "schema-text": { - "displayName": "Schema Text", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-text", - "sensitive": false - }, - "csv-writer": { - "displayName": "CSV Writer", - "dynamic": false, - "identifiesControllerService": false, - "name": "csv-writer", - "sensitive": false - }, - "Quote Character": { - "displayName": "Quote Character", - "dynamic": false, - "identifiesControllerService": false, - "name": "Quote Character", - "sensitive": false - }, - "Value Separator": { - "displayName": "Value Separator", - "dynamic": false, - "identifiesControllerService": false, - "name": "Value Separator", - "sensitive": false - }, - "Record Separator": { - "displayName": "Record Separator", - "dynamic": false, - "identifiesControllerService": false, - "name": "Record Separator", - "sensitive": false - }, - "Timestamp Format": { - "displayName": "Timestamp Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Timestamp Format", - "sensitive": false - }, - "Schema Write Strategy": { - "displayName": "Schema Write Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "Schema Write Strategy", - "sensitive": false - }, - "schema-name": { - "displayName": "Schema Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-name", - "sensitive": false - }, - "Quote Mode": { - "displayName": "Quote Mode", - "dynamic": false, - "identifiesControllerService": false, - "name": "Quote Mode", - "sensitive": false - }, - "Schema Reference Writer": { - "displayName": "Schema Reference Writer", - "dynamic": false, - "identifiesControllerService": true, - "name": "Schema Reference Writer", - "sensitive": false - }, - "Include Trailing Delimiter": { - "displayName": "Include Trailing Delimiter", - "dynamic": false, - "identifiesControllerService": false, - "name": "Include Trailing Delimiter", - "sensitive": false - } - }, - "scheduledState": "DISABLED", - "type": "org.apache.nifi.csv.CSVRecordSetWriter" - }, - { - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-record-serialization-services-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "CONTROLLER_SERVICE", - "controllerServiceApis": [ - { - "bundle": { - "artifact": "nifi-standard-services-api-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "type": "org.apache.nifi.serialization.RecordReaderFactory" - } - ], - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "73630c54-bf3f-397c-893b-0426090b7d92", - "instanceIdentifier": "58cccec5-00c5-33c6-84d4-ee4a488aed7a", - "name": "CSVReader", - "properties": { - "ignore-csv-header": "false", - "Trim double quote": "true", - "CSV Format": "custom", - "Quote Character": "\"", - "csvutils-allow-duplicate-header-names": "true", - "Value Separator": ",", - "Record Separator": "\\n", - "Escape Character": "\\", - "Skip Header Line": "true", - "Trim Fields": "true", - "schema-name": "${schema.name}", - "csv-reader-csv-parser": "commons-csv", - "schema-access-strategy": "infer-schema", - "csvutils-character-set": "UTF-8", - "schema-text": "${avro.schema}" - }, - "propertyDescriptors": { - "schema-reference-reader": { - "displayName": "Schema Reference Reader", - "dynamic": false, - "identifiesControllerService": true, - "name": "schema-reference-reader", - "sensitive": false - }, - "ignore-csv-header": { - "displayName": "Ignore CSV Header Column Names", - "dynamic": false, - "identifiesControllerService": false, - "name": "ignore-csv-header", - "sensitive": false - }, - "schema-branch": { - "displayName": "Schema Branch", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-branch", - "sensitive": false - }, - "Trim double quote": { - "displayName": "Trim double quote", - "dynamic": false, - "identifiesControllerService": false, - "name": "Trim double quote", - "sensitive": false - }, - "CSV Format": { - "displayName": "CSV Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "CSV Format", - "sensitive": false - }, - "Quote Character": { - "displayName": "Quote Character", - "dynamic": false, - "identifiesControllerService": false, - "name": "Quote Character", - "sensitive": false - }, - "csvutils-allow-duplicate-header-names": { - "displayName": "Allow Duplicate Header Names", - "dynamic": false, - "identifiesControllerService": false, - "name": "csvutils-allow-duplicate-header-names", - "sensitive": false - }, - "Value Separator": { - "displayName": "Value Separator", - "dynamic": false, - "identifiesControllerService": false, - "name": "Value Separator", - "sensitive": false - }, - "Record Separator": { - "displayName": "Record Separator", - "dynamic": false, - "identifiesControllerService": false, - "name": "Record Separator", - "sensitive": false - }, - "Timestamp Format": { - "displayName": "Timestamp Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Timestamp Format", - "sensitive": false - }, - "Escape Character": { - "displayName": "Escape Character", - "dynamic": false, - "identifiesControllerService": false, - "name": "Escape Character", - "sensitive": false - }, - "Date Format": { - "displayName": "Date Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Date Format", - "sensitive": false - }, - "Null String": { - "displayName": "Null String", - "dynamic": false, - "identifiesControllerService": false, - "name": "Null String", - "sensitive": false - }, - "Skip Header Line": { - "displayName": "Treat First Line as Header", - "dynamic": false, - "identifiesControllerService": false, - "name": "Skip Header Line", - "sensitive": false - }, - "Trim Fields": { - "displayName": "Trim Fields", - "dynamic": false, - "identifiesControllerService": false, - "name": "Trim Fields", - "sensitive": false - }, - "schema-name": { - "displayName": "Schema Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-name", - "sensitive": false - }, - "schema-registry": { - "displayName": "Schema Registry", - "dynamic": false, - "identifiesControllerService": true, - "name": "schema-registry", - "sensitive": false - }, - "csv-reader-csv-parser": { - "displayName": "CSV Parser", - "dynamic": false, - "identifiesControllerService": false, - "name": "csv-reader-csv-parser", - "sensitive": false - }, - "Time Format": { - "displayName": "Time Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "Time Format", - "sensitive": false - }, - "Comment Marker": { - "displayName": "Comment Marker", - "dynamic": false, - "identifiesControllerService": false, - "name": "Comment Marker", - "sensitive": false - }, - "schema-access-strategy": { - "displayName": "Schema Access Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-access-strategy", - "sensitive": false - }, - "schema-version": { - "displayName": "Schema Version", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-version", - "sensitive": false - }, - "csvutils-character-set": { - "displayName": "Character Set", - "dynamic": false, - "identifiesControllerService": false, - "name": "csvutils-character-set", - "sensitive": false - }, - "schema-text": { - "displayName": "Schema Text", - "dynamic": false, - "identifiesControllerService": false, - "name": "schema-text", - "sensitive": false - } - }, - "scheduledState": "DISABLED", - "type": "org.apache.nifi.csv.CSVReader" - } - ], - "defaultBackPressureDataSizeThreshold": "1 GB", - "defaultBackPressureObjectThreshold": 10000, - "defaultFlowFileExpiration": "0 sec", - "executionEngine": "INHERITED", - "flowFileConcurrency": "UNBOUNDED", - "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", - "funnels": [], - "identifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "inputPorts": [], - "instanceIdentifier": "60beeb61-0193-1000-174c-415892fdaaeb", - "labels": [], - "maxConcurrentTasks": 1, - "name": "GetChallengesProcessGroup", - "outputPorts": [], - "position": { "x": -656.0, "y": -376.0 }, - "processGroups": [], - "processors": [ - { - "autoTerminatedRelationships": ["original", "failure"], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "93ffd07d-2ddd-3dc5-a3d9-a68a75e6dc04", - "instanceIdentifier": "c08f8e4a-3778-3559-1c72-b8c02757a135", - "maxBackoffPeriod": "10 mins", - "name": "SplitRecord", - "penaltyDuration": "30 sec", - "position": { "x": 272.0, "y": -648.0 }, - "properties": { - "Record Writer": "87d7b1c2-fd34-3cbc-b558-86b49fe72758", - "Record Reader": "73630c54-bf3f-397c-893b-0426090b7d92", - "Records Per Split": "1" - }, - "propertyDescriptors": { - "Record Writer": { - "displayName": "Record Writer", - "dynamic": false, - "identifiesControllerService": true, - "name": "Record Writer", - "sensitive": false - }, - "Record Reader": { - "displayName": "Record Reader", - "dynamic": false, - "identifiesControllerService": true, - "name": "Record Reader", - "sensitive": false - }, - "Records Per Split": { - "displayName": "Records Per Split", - "dynamic": false, - "identifiesControllerService": false, - "name": "Records Per Split", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 0, - "scheduledState": "ENABLED", - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.standard.SplitRecord", - "yieldDuration": "1 sec" - }, - { - "autoTerminatedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-update-attribute-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "b63dfd38-939b-3a62-bd0f-ed2cc6e20674", - "instanceIdentifier": "ce2a1b66-97ab-3834-7ddd-b49341646684", - "maxBackoffPeriod": "10 mins", - "name": "SetFilename", - "penaltyDuration": "30 sec", - "position": { "x": 272.0, "y": -256.0 }, - "properties": { - "filename": "challenges.csv", - "Store State": "Do not store state", - "canonical-value-lookup-cache-size": "100" - }, - "propertyDescriptors": { - "Delete Attributes Expression": { - "displayName": "Delete Attributes Expression", - "dynamic": false, - "identifiesControllerService": false, - "name": "Delete Attributes Expression", - "sensitive": false - }, - "filename": { - "displayName": "filename", - "dynamic": true, - "identifiesControllerService": false, - "name": "filename", - "sensitive": false - }, - "Store State": { - "displayName": "Store State", - "dynamic": false, - "identifiesControllerService": false, - "name": "Store State", - "sensitive": false - }, - "canonical-value-lookup-cache-size": { - "displayName": "Cache Value Lookup Cache Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "canonical-value-lookup-cache-size", - "sensitive": false - }, - "Stateful Variables Initial Value": { - "displayName": "Stateful Variables Initial Value", - "dynamic": false, - "identifiesControllerService": false, - "name": "Stateful Variables Initial Value", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 25, - "scheduledState": "ENABLED", - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.attributes.UpdateAttribute", - "yieldDuration": "1 sec" - }, - { - "autoTerminatedRelationships": ["No Retry", "Retry", "Original", "Failure"], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "9cb3fc8c-2a8a-3a99-b104-3100f0e67538", - "instanceIdentifier": "28407726-80c4-3511-94e4-980672f02894", - "maxBackoffPeriod": "10 mins", - "name": "GetChallenges", - "penaltyDuration": "30 sec", - "position": { "x": -352.0, "y": -256.0 }, - "properties": { - "Request Content-Encoding": "DISABLED", - "Request Multipart Form-Data Filename Enabled": "true", - "Request Chunked Transfer-Encoding Enabled": "false", - "HTTP/2 Disabled": "False", - "Connection Timeout": "5 secs", - "Response Cookie Strategy": "DISABLED", - "Socket Read Timeout": "15 secs", - "Socket Idle Connections": "5", - "Request Body Enabled": "true", - "HTTP URL": "https://raw.githubusercontent.com/Sage-Bionetworks/sage-monorepo/refs/heads/main/apps/openchallenges/challenge-service/src/main/resources/db/challenges.csv", - "Socket Idle Timeout": "5 mins", - "Response Redirects Enabled": "True", - "Socket Write Timeout": "15 secs", - "Response FlowFile Naming Strategy": "RANDOM", - "Response Cache Enabled": "false", - "Request Date Header Enabled": "True", - "Request Failure Penalization Enabled": "false", - "Response Body Attribute Size": "256", - "Response Generation Required": "false", - "Response Header Request Attributes Enabled": "false", - "HTTP Method": "GET", - "Request Content-Type": "${mime.type}", - "Request Digest Authentication Enabled": "false", - "Response Cache Size": "10MB", - "Response Body Ignored": "false" - }, - "propertyDescriptors": { - "Request Content-Encoding": { - "displayName": "Request Content-Encoding", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Content-Encoding", - "sensitive": false - }, - "proxy-configuration-service": { - "displayName": "Proxy Configuration Service", - "dynamic": false, - "identifiesControllerService": true, - "name": "proxy-configuration-service", - "sensitive": false - }, - "Request Multipart Form-Data Filename Enabled": { - "displayName": "Request Multipart Form-Data Filename Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Multipart Form-Data Filename Enabled", - "sensitive": false - }, - "Request Chunked Transfer-Encoding Enabled": { - "displayName": "Request Chunked Transfer-Encoding Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Chunked Transfer-Encoding Enabled", - "sensitive": false - }, - "HTTP/2 Disabled": { - "displayName": "HTTP/2 Disabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "HTTP/2 Disabled", - "sensitive": false - }, - "Connection Timeout": { - "displayName": "Connection Timeout", - "dynamic": false, - "identifiesControllerService": false, - "name": "Connection Timeout", - "sensitive": false - }, - "Response Cookie Strategy": { - "displayName": "Response Cookie Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Cookie Strategy", - "sensitive": false - }, - "Request Password": { - "displayName": "Request Password", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Password", - "sensitive": true - }, - "Socket Read Timeout": { - "displayName": "Socket Read Timeout", - "dynamic": false, - "identifiesControllerService": false, - "name": "Socket Read Timeout", - "sensitive": false - }, - "Socket Idle Connections": { - "displayName": "Socket Idle Connections", - "dynamic": false, - "identifiesControllerService": false, - "name": "Socket Idle Connections", - "sensitive": false - }, - "Request Body Enabled": { - "displayName": "Request Body Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Body Enabled", - "sensitive": false - }, - "HTTP URL": { - "displayName": "HTTP URL", - "dynamic": false, - "identifiesControllerService": false, - "name": "HTTP URL", - "sensitive": false - }, - "Request OAuth2 Access Token Provider": { - "displayName": "Request OAuth2 Access Token Provider", - "dynamic": false, - "identifiesControllerService": true, - "name": "Request OAuth2 Access Token Provider", - "sensitive": false - }, - "Socket Idle Timeout": { - "displayName": "Socket Idle Timeout", - "dynamic": false, - "identifiesControllerService": false, - "name": "Socket Idle Timeout", - "sensitive": false - }, - "Response Redirects Enabled": { - "displayName": "Response Redirects Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Redirects Enabled", - "sensitive": false - }, - "Socket Write Timeout": { - "displayName": "Socket Write Timeout", - "dynamic": false, - "identifiesControllerService": false, - "name": "Socket Write Timeout", - "sensitive": false - }, - "Request Header Attributes Pattern": { - "displayName": "Request Header Attributes Pattern", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Header Attributes Pattern", - "sensitive": false - }, - "Response FlowFile Naming Strategy": { - "displayName": "Response FlowFile Naming Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response FlowFile Naming Strategy", - "sensitive": false - }, - "Response Cache Enabled": { - "displayName": "Response Cache Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Cache Enabled", - "sensitive": false - }, - "Request Date Header Enabled": { - "displayName": "Request Date Header Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Date Header Enabled", - "sensitive": false - }, - "Request Failure Penalization Enabled": { - "displayName": "Request Failure Penalization Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Failure Penalization Enabled", - "sensitive": false - }, - "Response Body Attribute Size": { - "displayName": "Response Body Attribute Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Body Attribute Size", - "sensitive": false - }, - "SSL Context Service": { - "displayName": "SSL Context Service", - "dynamic": false, - "identifiesControllerService": true, - "name": "SSL Context Service", - "sensitive": false - }, - "Response Generation Required": { - "displayName": "Response Generation Required", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Generation Required", - "sensitive": false - }, - "Request User-Agent": { - "displayName": "Request User-Agent", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request User-Agent", - "sensitive": false - }, - "Response Header Request Attributes Enabled": { - "displayName": "Response Header Request Attributes Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Header Request Attributes Enabled", - "sensitive": false - }, - "HTTP Method": { - "displayName": "HTTP Method", - "dynamic": false, - "identifiesControllerService": false, - "name": "HTTP Method", - "sensitive": false - }, - "Request Username": { - "displayName": "Request Username", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Username", - "sensitive": false - }, - "Request Content-Type": { - "displayName": "Request Content-Type", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Content-Type", - "sensitive": false - }, - "Response Body Attribute Name": { - "displayName": "Response Body Attribute Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Body Attribute Name", - "sensitive": false - }, - "Request Digest Authentication Enabled": { - "displayName": "Request Digest Authentication Enabled", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Digest Authentication Enabled", - "sensitive": false - }, - "Request Multipart Form-Data Name": { - "displayName": "Request Multipart Form-Data Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "Request Multipart Form-Data Name", - "sensitive": false - }, - "Response Cache Size": { - "displayName": "Response Cache Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Cache Size", - "sensitive": false - }, - "Response Body Ignored": { - "displayName": "Response Body Ignored", - "dynamic": false, - "identifiesControllerService": false, - "name": "Response Body Ignored", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 0, - "scheduledState": "ENABLED", - "schedulingPeriod": "300 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.standard.InvokeHTTP", - "yieldDuration": "1 sec" - }, - { - "autoTerminatedRelationships": ["success", "failure"], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "dec7ffa2-e5fa-3a2a-aae8-7635e7c237f3", - "instanceIdentifier": "a8f6a6e7-33c9-3e5c-a252-7d94372de811", - "maxBackoffPeriod": "10 mins", - "name": "PutFile", - "penaltyDuration": "30 sec", - "position": { "x": 952.0, "y": -256.0 }, - "properties": { - "Create Missing Directories": "true", - "Directory": "/home/nifi/openchallenges", - "Conflict Resolution Strategy": "fail" - }, - "propertyDescriptors": { - "Group": { - "displayName": "Group", - "dynamic": false, - "identifiesControllerService": false, - "name": "Group", - "sensitive": false - }, - "Owner": { - "displayName": "Owner", - "dynamic": false, - "identifiesControllerService": false, - "name": "Owner", - "sensitive": false - }, - "Create Missing Directories": { - "displayName": "Create Missing Directories", - "dynamic": false, - "identifiesControllerService": false, - "name": "Create Missing Directories", - "sensitive": false - }, - "Permissions": { - "displayName": "Permissions", - "dynamic": false, - "identifiesControllerService": false, - "name": "Permissions", - "sensitive": false - }, - "Maximum File Count": { - "displayName": "Maximum File Count", - "dynamic": false, - "identifiesControllerService": false, - "name": "Maximum File Count", - "sensitive": false - }, - "Last Modified Time": { - "displayName": "Last Modified Time", - "dynamic": false, - "identifiesControllerService": false, - "name": "Last Modified Time", - "sensitive": false - }, - "Directory": { - "displayName": "Directory", - "dynamic": false, - "identifiesControllerService": false, - "name": "Directory", - "sensitive": false - }, - "Conflict Resolution Strategy": { - "displayName": "Conflict Resolution Strategy", - "dynamic": false, - "identifiesControllerService": false, - "name": "Conflict Resolution Strategy", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 0, - "scheduledState": "ENABLED", - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.standard.PutFile", - "yieldDuration": "1 sec" - }, - { - "autoTerminatedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "2a9918eb-0433-37bf-97ae-12528859b99a", - "instanceIdentifier": "0f8ccffe-0bd3-3bda-5247-2ddf92337fe2", - "maxBackoffPeriod": "10 mins", - "name": "GetCsvFile", - "penaltyDuration": "30 sec", - "position": { "x": -352.0, "y": -648.0 }, - "properties": { - "Keep Source File": "true", - "Minimum File Age": "0 sec", - "Polling Interval": "0 sec", - "Input Directory": "/home/nifi/input", - "Batch Size": "10", - "Minimum File Size": "0 B", - "Ignore Hidden Files": "true", - "Recurse Subdirectories": "true", - "File Filter": "[^\\.].*" - }, - "propertyDescriptors": { - "Keep Source File": { - "displayName": "Keep Source File", - "dynamic": false, - "identifiesControllerService": false, - "name": "Keep Source File", - "sensitive": false - }, - "Minimum File Age": { - "displayName": "Minimum File Age", - "dynamic": false, - "identifiesControllerService": false, - "name": "Minimum File Age", - "sensitive": false - }, - "Polling Interval": { - "displayName": "Polling Interval", - "dynamic": false, - "identifiesControllerService": false, - "name": "Polling Interval", - "sensitive": false - }, - "Input Directory": { - "displayName": "Input Directory", - "dynamic": false, - "identifiesControllerService": false, - "name": "Input Directory", - "sensitive": false - }, - "Maximum File Age": { - "displayName": "Maximum File Age", - "dynamic": false, - "identifiesControllerService": false, - "name": "Maximum File Age", - "sensitive": false - }, - "Batch Size": { - "displayName": "Batch Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "Batch Size", - "sensitive": false - }, - "Maximum File Size": { - "displayName": "Maximum File Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "Maximum File Size", - "sensitive": false - }, - "Minimum File Size": { - "displayName": "Minimum File Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "Minimum File Size", - "sensitive": false - }, - "Ignore Hidden Files": { - "displayName": "Ignore Hidden Files", - "dynamic": false, - "identifiesControllerService": false, - "name": "Ignore Hidden Files", - "sensitive": false - }, - "Recurse Subdirectories": { - "displayName": "Recurse Subdirectories", - "dynamic": false, - "identifiesControllerService": false, - "name": "Recurse Subdirectories", - "sensitive": false - }, - "File Filter": { - "displayName": "File Filter", - "dynamic": false, - "identifiesControllerService": false, - "name": "File Filter", - "sensitive": false - }, - "Path Filter": { - "displayName": "Path Filter", - "dynamic": false, - "identifiesControllerService": false, - "name": "Path Filter", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 0, - "scheduledState": "ENABLED", - "schedulingPeriod": "30 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.standard.GetFile", - "yieldDuration": "1 sec" - }, - { - "autoTerminatedRelationships": ["success", "failure", "retry"], - "backoffMechanism": "PENALIZE_FLOWFILE", - "bulletinLevel": "WARN", - "bundle": { - "artifact": "nifi-standard-nar", - "group": "org.apache.nifi", - "version": "2.0.0" - }, - "comments": "", - "componentType": "PROCESSOR", - "concurrentlySchedulableTaskCount": 1, - "executionNode": "ALL", - "groupIdentifier": "95113416-c5d8-3e3a-b3fc-f49cec0d01a9", - "identifier": "fd86ff9c-c5c2-31ca-9aec-5d1419cc9455", - "instanceIdentifier": "c0755e5d-c0c0-3c01-e2b8-c8dfae3d7627", - "maxBackoffPeriod": "10 mins", - "name": "PutDatabaseRecord", - "penaltyDuration": "30 sec", - "position": { "x": 936.0, "y": -648.0 }, - "properties": { - "put-db-record-allow-multiple-statements": "false", - "table-schema-cache-size": "100", - "put-db-record-quoted-table-identifiers": "false", - "put-db-record-unmatched-column-behavior": "Fail on Unmatched Columns", - "put-db-record-translate-field-names": "true", - "put-db-record-dcbp-service": "6650fa38-df10-35d3-85d1-14b73493f348", - "put-db-record-query-timeout": "0 seconds", - "rollback-on-failure": "false", - "put-db-record-statement-type": "UPSERT", - "put-db-record-binary-format": "UTF-8", - "db-type": "MySQL", - "put-db-record-quoted-identifiers": "false", - "put-db-record-table-name": "nifi_demo", - "put-db-record-unmatched-field-behavior": "Ignore Unmatched Fields", - "put-db-record-max-batch-size": "1000", - "put-db-record-record-reader": "73630c54-bf3f-397c-893b-0426090b7d92", - "database-session-autocommit": "false" - }, - "propertyDescriptors": { - "put-db-record-allow-multiple-statements": { - "displayName": "Allow Multiple SQL Statements", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-allow-multiple-statements", - "sensitive": false - }, - "table-schema-cache-size": { - "displayName": "Table Schema Cache Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "table-schema-cache-size", - "sensitive": false - }, - "put-db-record-schema-name": { - "displayName": "Schema Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-schema-name", - "sensitive": false - }, - "put-db-record-field-containing-sql": { - "displayName": "Field Containing SQL", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-field-containing-sql", - "sensitive": false - }, - "put-db-record-quoted-table-identifiers": { - "displayName": "Quote Table Identifiers", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-quoted-table-identifiers", - "sensitive": false - }, - "Statement Type Record Path": { - "displayName": "Statement Type Record Path", - "dynamic": false, - "identifiesControllerService": false, - "name": "Statement Type Record Path", - "sensitive": false - }, - "put-db-record-unmatched-column-behavior": { - "displayName": "Unmatched Column Behavior", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-unmatched-column-behavior", - "sensitive": false - }, - "Delete Keys": { - "displayName": "Delete Keys", - "dynamic": false, - "identifiesControllerService": false, - "name": "Delete Keys", - "sensitive": false - }, - "put-db-record-catalog-name": { - "displayName": "Catalog Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-catalog-name", - "sensitive": false - }, - "put-db-record-translate-field-names": { - "displayName": "Translate Field Names", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-translate-field-names", - "sensitive": false - }, - "put-db-record-dcbp-service": { - "displayName": "Database Connection Pooling Service", - "dynamic": false, - "identifiesControllerService": true, - "name": "put-db-record-dcbp-service", - "sensitive": false - }, - "put-db-record-query-timeout": { - "displayName": "Max Wait Time", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-query-timeout", - "sensitive": false - }, - "rollback-on-failure": { - "displayName": "Rollback On Failure", - "dynamic": false, - "identifiesControllerService": false, - "name": "rollback-on-failure", - "sensitive": false - }, - "put-db-record-statement-type": { - "displayName": "Statement Type", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-statement-type", - "sensitive": false - }, - "put-db-record-binary-format": { - "displayName": "Binary String Format", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-binary-format", - "sensitive": false - }, - "db-type": { - "displayName": "Database Type", - "dynamic": false, - "identifiesControllerService": false, - "name": "db-type", - "sensitive": false - }, - "put-db-record-update-keys": { - "displayName": "Update Keys", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-update-keys", - "sensitive": false - }, - "put-db-record-quoted-identifiers": { - "displayName": "Quote Column Identifiers", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-quoted-identifiers", - "sensitive": false - }, - "put-db-record-table-name": { - "displayName": "Table Name", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-table-name", - "sensitive": false - }, - "put-db-record-unmatched-field-behavior": { - "displayName": "Unmatched Field Behavior", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-unmatched-field-behavior", - "sensitive": false - }, - "put-db-record-max-batch-size": { - "displayName": "Maximum Batch Size", - "dynamic": false, - "identifiesControllerService": false, - "name": "put-db-record-max-batch-size", - "sensitive": false - }, - "put-db-record-record-reader": { - "displayName": "Record Reader", - "dynamic": false, - "identifiesControllerService": true, - "name": "put-db-record-record-reader", - "sensitive": false - }, - "Data Record Path": { - "displayName": "Data Record Path", - "dynamic": false, - "identifiesControllerService": false, - "name": "Data Record Path", - "sensitive": false - }, - "database-session-autocommit": { - "displayName": "Database Session AutoCommit", - "dynamic": false, - "identifiesControllerService": false, - "name": "database-session-autocommit", - "sensitive": false - } - }, - "retriedRelationships": [], - "retryCount": 10, - "runDurationMillis": 0, - "scheduledState": "ENABLED", - "schedulingPeriod": "0 sec", - "schedulingStrategy": "TIMER_DRIVEN", - "style": {}, - "type": "org.apache.nifi.processors.standard.PutDatabaseRecord", - "yieldDuration": "1 sec" - } - ], - "remoteProcessGroups": [], - "scheduledState": "ENABLED", - "statelessFlowTimeout": "1 min" - }, - "flowEncodingVersion": "1.0", - "parameterContexts": {}, - "parameterProviders": {}, - "snapshotMetadata": { - "author": "anonymous", - "comments": "Update the name of the data input folder", - "timestamp": 1732495279528, - "version": 2 - } -} diff --git a/apps/openchallenges/nifi-registry/project.json b/apps/openchallenges/nifi-registry/project.json deleted file mode 100644 index 7585a58fc1..0000000000 --- a/apps/openchallenges/nifi-registry/project.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "openchallenges-nifi-registry", - "$schema": "../../../node_modules/nx/schemas/project-schema.json", - "sourceRoot": "apps/openchallenges-nifi-registry/src", - "projectType": "application", - "targets": { - "create-config": { - "executor": "nx:run-commands", - "options": { - "command": "cp -n .env.example .env", - "cwd": "{projectRoot}" - } - }, - "serve-detach": { - "executor": "nx:run-commands", - "options": { - "command": "docker/openchallenges/serve-detach.sh {projectName}" - } - } - }, - "tags": ["type:service", "scope:backend"] -} diff --git a/apps/openchallenges/nifi/.env.example b/apps/openchallenges/nifi/.env.example deleted file mode 100644 index b0c3a297cd..0000000000 --- a/apps/openchallenges/nifi/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -SINGLE_USER_CREDENTIALS_USERNAME=openchallenges -SINGLE_USER_CREDENTIALS_PASSWORD=changemechangeme \ No newline at end of file diff --git a/apps/openchallenges/nifi/Dockerfile b/apps/openchallenges/nifi/Dockerfile deleted file mode 100644 index 76ad9ab8c2..0000000000 --- a/apps/openchallenges/nifi/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM apache/nifi:2.0.0 - -ENV MYSQL_CONNECTOR_VERSION=9.1.0 -ENV MYSQL_CONNECTOR_URL=https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-j-${MYSQL_CONNECTOR_VERSION}.tar.gz - -# Create the drivers directory -RUN mkdir -p /opt/nifi/nifi-current/drivers/ - -# Download and extract the MySQL JDBC connector, then move the JAR to the drivers directory -RUN curl -L -o /tmp/mysql-connector-j.tar.gz "${MYSQL_CONNECTOR_URL}" \ - && tar -xzf /tmp/mysql-connector-j.tar.gz -C /tmp \ - && mv /tmp/mysql-connector-j-${MYSQL_CONNECTOR_VERSION}/mysql-connector-j-${MYSQL_CONNECTOR_VERSION}.jar /opt/nifi/nifi-current/drivers/ \ - && rm -rf /tmp/mysql-connector-j.tar.gz /tmp/mysql-connector-j-${MYSQL_CONNECTOR_VERSION} \ No newline at end of file diff --git a/apps/openchallenges/nifi/README.md b/apps/openchallenges/nifi/README.md deleted file mode 100644 index 69beb11082..0000000000 --- a/apps/openchallenges/nifi/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# OpenChallenges NiFi - -## Build the Docker image - -Build the Docker image of NiFi and the NiFi registry. - -```console -nx run-many -t build-image -p openchallenges-{nifi,nifi-registry} -``` - -## Deploy NiFi with Docker Compose - -```console -nx serve-detach openchallenges-nifi -``` - -## Access NiFi UI - -1. In your browser, navigate to http://localhost:8443/nifi. -2. Accept the security risk (in the development environment). -3. Login with the credentials specified in the config file `.env`. - -The NiFi canvas should now be visible. - -## Connect NiFi to the NiFi registry - -Configuring a NiFi registry client enable to version control the Process Groups created in NiFi. - -1. Click on the three-line menu button. -2. Click on "Controller Settings". -3. Click on the tab "Registry Clients". -4. Click on the button "+". -5. Add the registry client: - - Name: openchallenges-nifi-registry. - - Type: NifiRegistryFlowRegistryClient. - - Click on "Apply". -6. Configure the registry client: - - Click on the three-dot button. - - Click on "Edit". - - Click on the tab "Properties". - - Set the URL to `http://openchallenges-nifi-registry:18080`. - - Click on "Apply". -7. Click on the NiFi logo in the navigation bar to return to the canvas. - -Here is a quick way to check that NiFi can connect to the registry: - -1. Add a Process Group to the canvas. -2. Right-click on the PG and select "Version" > "Start Version Control". -3. The "Save Flow Version" dialog should open without any error messages visible. - -## Stop and remove the NiFi container - -```console -docker rm -f openchallenges-nifi-registry -``` - -## Data persistence - -Process Groups created in NiFi can be saved and version-controlled using the NiFi Registry. diff --git a/apps/openchallenges/nifi/input/nifi_demo.csv b/apps/openchallenges/nifi/input/nifi_demo.csv deleted file mode 100644 index 540407b355..0000000000 --- a/apps/openchallenges/nifi/input/nifi_demo.csv +++ /dev/null @@ -1,4 +0,0 @@ -"id","name" -"1","BRCA1" -"2","TP53" -"3","CFTR" \ No newline at end of file diff --git a/apps/openchallenges/nifi/project.json b/apps/openchallenges/nifi/project.json deleted file mode 100644 index 434a8c9d3c..0000000000 --- a/apps/openchallenges/nifi/project.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "openchallenges-nifi", - "$schema": "../../../node_modules/nx/schemas/project-schema.json", - "sourceRoot": "apps/openchallenges-nifi/src", - "projectType": "application", - "targets": { - "create-config": { - "executor": "nx:run-commands", - "options": { - "command": "cp -n .env.example .env", - "cwd": "{projectRoot}" - } - }, - "serve-detach": { - "executor": "nx:run-commands", - "options": { - "command": "docker/openchallenges/serve-detach.sh {projectName}" - } - } - }, - "tags": ["type:service", "scope:backend"] -} diff --git a/docker/openchallenges/serve-detach.sh b/docker/openchallenges/serve-detach.sh index 8478d476c3..f6344fc51c 100755 --- a/docker/openchallenges/serve-detach.sh +++ b/docker/openchallenges/serve-detach.sh @@ -15,8 +15,6 @@ args=( --file docker/openchallenges/services/kafka.yml --file docker/openchallenges/services/mariadb.yml --file docker/openchallenges/services/mysqld-exporter.yml - --file docker/openchallenges/services/nifi-registry.yml - --file docker/openchallenges/services/nifi.yml --file docker/openchallenges/services/organization-service.yml --file docker/openchallenges/services/prometheus.yml --file docker/openchallenges/services/service-registry.yml diff --git a/docker/openchallenges/services/nifi-registry.yml b/docker/openchallenges/services/nifi-registry.yml deleted file mode 100644 index 26312d77db..0000000000 --- a/docker/openchallenges/services/nifi-registry.yml +++ /dev/null @@ -1,17 +0,0 @@ -services: - openchallenges-nifi-registry: - image: ghcr.io/sage-bionetworks/openchallenges-nifi-registry:${OPENCHALLENGES_VERSION:-local} - container_name: openchallenges-nifi-registry - restart: always - env_file: - - ../../../apps/openchallenges/nifi-registry/.env - networks: - - openchallenges - ports: - - '18080:18080' - volumes: - - openchallenges-nifi-registry-data:/opt/nifi-registry/nifi-registry-current - deploy: - resources: - limits: - memory: 1G diff --git a/docker/openchallenges/services/nifi.yml b/docker/openchallenges/services/nifi.yml deleted file mode 100644 index 30fa426c16..0000000000 --- a/docker/openchallenges/services/nifi.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - openchallenges-nifi: - image: ghcr.io/sage-bionetworks/openchallenges-nifi:${OPENCHALLENGES_VERSION:-local} - container_name: openchallenges-nifi - restart: always - env_file: - - ../../../apps/openchallenges/nifi/.env - networks: - - openchallenges - ports: - - '8443:8443' - volumes: - - ../../../apps/openchallenges/nifi/input:/home/nifi/input - depends_on: - openchallenges-nifi-registry: - condition: service_started - deploy: - resources: - limits: - memory: 2G diff --git a/docker/openchallenges/volumes.yml b/docker/openchallenges/volumes.yml index 4f53a6bbc5..5f6f3319e6 100644 --- a/docker/openchallenges/volumes.yml +++ b/docker/openchallenges/volumes.yml @@ -5,8 +5,5 @@ volumes: openchallenges-grafana-data: name: openchallenges-grafana-data - openchallenges-nifi-registry-data: - name: openchallenges-nifi-registry-data - openchallenges-thumbor-data: name: openchallenges-thumbor-data diff --git a/tools/configure-hostnames.sh b/tools/configure-hostnames.sh index 5bfc482acf..39b1563414 100755 --- a/tools/configure-hostnames.sh +++ b/tools/configure-hostnames.sh @@ -20,8 +20,6 @@ declare -a hostnames=( "127.0.0.1 openchallenges-image-service" "127.0.0.1 openchallenges-mariadb" "127.0.0.1 openchallenges-mysqld-exporter" - "127.0.0.1 openchallenges-nifi-registry" - "127.0.0.1 openchallenges-nifi" "127.0.0.1 openchallenges-opensearch" "127.0.0.1 openchallenges-organization-service" "127.0.0.1 openchallenges-prometheus"