Update ADLS Upload documentation (#157)

* Update ADLS Upload documentation * Update Delete.java * Update List.java * Update Read.java * Update Reads.java * change data example * change data example
kestra-io · Nov 28, 2024 · b6c926d · b6c926d
1 parent 22c3f3c
commit b6c926d
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 48 deletions.
diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/Delete.java b/src/main/java/io/kestra/plugin/azure/storage/adls/Delete.java
@@ -27,18 +27,14 @@
             code = """
                 id: azure_storage_datalake_delete
                 namespace: company.team
-
-                pluginDefaults:
-                  - type: io.kestra.plugin.azure.storage.adls
-                    values:
-                      connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
-                      fileSystem: "tasks"
-                      endpoint: "https://yourblob.blob.core.windows.net"
-
+                      
                 tasks:
                   - id: delete_file
                     type: io.kestra.plugin.azure.storage.adls.Delete
                     filePath: "full/path/to/file.txt"
+                    fileSystem: "tasks"
+                    endpoint: "https://yourblob.blob.core.windows.net"
+                    connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
                 """
         )
     }

diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/List.java b/src/main/java/io/kestra/plugin/azure/storage/adls/List.java
@@ -33,16 +33,12 @@
                 id: azure_data_lake_storage_list
                 namespace: company.team
 
-                pluginDefaults:
-                  - type: io.kestra.plugin.azure.storage.adls
-                    values:
-                      connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
-                      fileSystem: "tasks"
-                      endpoint: "https://yourblob.blob.core.windows.net"
-
                 tasks:
                   - id: list_files_in_dir
                     type: io.kestra.plugin.azure.storage.adls.List
+                    connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
+                    fileSystem: "tasks"
+                    endpoint: "https://yourblob.blob.core.windows.net"
                     directoryPath: "path/to/my/directory/"
 
                   - id: for_each_file

diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/Read.java b/src/main/java/io/kestra/plugin/azure/storage/adls/Read.java
@@ -30,16 +30,12 @@
                 id: azure_storage_datalake_read
                 namespace: company.team
 
-                pluginDefaults:
-                  - type: io.kestra.plugin.azure.storage.adls
-                    values:
-                      connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
-                      fileSystem: "tasks"
-                      endpoint: "https://yourblob.blob.core.windows.net"
-
                 tasks:
                   - id: read_file
                     type: io.kestra.plugin.azure.storage.adls.Read
+                    connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
+                    fileSystem: "tasks"
+                    endpoint: "https://yourblob.blob.core.windows.net"
                     filePath: "full/path/to/file.txt"
 
                   - id: log_size

diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/Reads.java b/src/main/java/io/kestra/plugin/azure/storage/adls/Reads.java
@@ -37,16 +37,12 @@
                 id: azure_storage_datalake_readq
                 namespace: company.team
 
-                pluginDefaults:
-                  - type: io.kestra.plugin.azure.storage.adls
-                    values:
-                      connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
-                      fileSystem: "tasks"
-                      endpoint: "https://yourblob.blob.core.windows.net"
-
                 tasks:
                   - id: read_file
                     type: io.kestra.plugin.azure.storage.adls.Reads
+                    connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
+                    fileSystem: "tasks"
+                    endpoint: "https://yourblob.blob.core.windows.net"
                     directoryPath: "path/to/my/directory/"
                 """
         )

diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/SharedAccess.java b/src/main/java/io/kestra/plugin/azure/storage/adls/SharedAccess.java
@@ -47,20 +47,20 @@
                 tasks:
                     - id: download_request
                       type: io.kestra.plugin.core.http.Download
-                      uri: https://pokeapi.co/api/v2/pokemon/pikachu
+                      uri: https://dummyjson.com/products
 
                     - id: to_ion
                       type: io.kestra.plugin.serdes.json.JsonToIon
                       from: "{{ outputs.download_request.uri }}"
 
                     - id: upload_file
                       type: io.kestra.plugin.azure.storage.adls.Upload
-                      fileName: "adls/pokemon/pikachu.json"
+                      fileName: "adls/product_data/product.json"
                       from: "{{ outputs.to_ion.uri }}"
 
                     - id: shared_access
                       type: io.kestra.plugin.azure.storage.adls.SharedAccess
-                      fileName: "adls/pokemon/pikachu.json"
+                      fileName: "adls/product_data/product.json"
                       expirationDate: "{{ now() | dateAdd(1, 'DAYS') }}"
                       permissions:
                         - READ

diff --git a/src/main/java/io/kestra/plugin/azure/storage/adls/Upload.java b/src/main/java/io/kestra/plugin/azure/storage/adls/Upload.java
@@ -29,31 +29,23 @@
     examples = {
         @Example(
             full = true,
-            title = "Download a json file, convert it to ion format, finally upload it to Azure Data Lake Storage.",
+            title = "Download a json file and upload it to Azure Data Lake Storage.",
             code = """
                 id: azure_data_lake_storage_upload
                 namespace: company.team
 
-                pluginDefaults:
-                  - type: io.kestra.plugin.azure.storage.adls
-                    values:
-                      connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
-                      fileSystem: "tasks"
-                      endpoint: "https://yourblob.blob.core.windows.net"
-
                 tasks:
                   - id: download_request
                     type: io.kestra.plugin.core.http.Download
-                    uri: https://pokeapi.co/api/v2/pokemon/pikachu
-
-                  - id: to_ion
-                    type: io.kestra.plugin.serdes.json.JsonToIon
-                    from: "{{ outputs.download_request.uri }}"
-
+                    uri: adls/product_data/product.json
+                
                   - id: upload_file
                     type: io.kestra.plugin.azure.storage.adls.Upload
-                    filePath: "path/to/file/pikachu.json"
-                    from: "{{ outputs.to_ion.uri }}"
+                    filePath: "path/to/file/product.json"
+                    from: "{{ outputs.download_request.uri }}"
+                    connectionString: "{{ secret('AZURE_CONNECTION_STRING') }}"
+                    fileSystem: "tasks"
+                    endpoint: "https://yourblob.blob.core.windows.net"
                 """
         )
     }