Merge pull request #148 from cloud-gov/reporting

Fixing index-pattern issue and optimize upload
cloud-gov · Dec 11, 2024 · 2722e21 · 2722e21
2 parents a5d2be4 + fa35085
commit 2722e21
Show file tree

Hide file tree

Showing 8 changed files with 7 additions and 414 deletions.
diff --git a/jobs/upload-dashboards-objects/spec b/jobs/upload-dashboards-objects/spec
@@ -25,7 +25,6 @@ consumes:
   optional: true
 
 templates:
-  bin/generate-objects: bin/generate-objects
   bin/generate-tenants: bin/generate-tenants
   bin/generate-ndjson: bin/generate-ndjson
   bin/import-objects: bin/import-objects
@@ -38,8 +37,6 @@ templates:
 
   # OpenSearch Dashboards objects (defaults).
   dashboards-objects/index-pattern/logs-app.json.erb:       dashboards-objects/index-pattern/logs-app*.json
-  dashboards-objects/index-pattern/logs.json.erb:           dashboards-objects/index-pattern/logs-*.json
-  dashboards-objects/index-pattern/logs-platform.json.erb:  dashboards-objects/index-pattern/logs-platform*.json
 
   dashboards-objects/search/app-all.json.erb:                         dashboards-objects/search/app-all.json
   dashboards-objects/search/app-all-messages.json.erb:                dashboards-objects/search/app-all-messages.json

diff --git a/jobs/upload-dashboards-objects/templates/bin/generate-ndjson b/jobs/upload-dashboards-objects/templates/bin/generate-ndjson
@@ -5,7 +5,8 @@ import json
 folders = [
 "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/dashboard",
 "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/search",
-"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/visualization"
+"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/visualization",
+"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/index-pattern"
 ]
 output_file = "/var/vcap/jobs/upload-dashboards-objects/dashboard.ndjson"
 

diff --git a/jobs/upload-dashboards-objects/templates/bin/generate-objects b/jobs/upload-dashboards-objects/templates/bin/generate-objects
diff --git a/jobs/upload-dashboards-objects/templates/bin/generate-tenants b/jobs/upload-dashboards-objects/templates/bin/generate-tenants
@@ -10,4 +10,5 @@ export STORE_DIR="/var/vcap/store/$JOB_NAME"
 cf api "<%= api %>"
 cf auth "<%= client %>" "<%= password %>" --client-credentials
 mkdir -p $STORE_DIR
-cf orgs| tail -n +4 > "$STORE_DIR/org-tenants.txt"
+cf orgs| tail -n +4 > "$STORE_DIR/org-tenants.txt"
+echo "admin_tenant" >> "$STORE_DIR/org-tenants.txt"
diff --git a/jobs/upload-dashboards-objects/templates/bin/run b/jobs/upload-dashboards-objects/templates/bin/run
@@ -10,9 +10,6 @@ export PYTHONPATH="/var/vcap/packages/python3/lib/python3.12/site-packages${PYTH
 export LD_LIBRARY_PATH="/var/vcap/packages/python3/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 export REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
 
-echo "[$(date)] Calling 'generate-objects'..."
-$(dirname $0)/generate-objects
-
 echo "[$(date)] Calling 'generate-tenants'..."
 $(dirname $0)/generate-tenants
 

diff --git a/jobs/upload-dashboards-objects/templates/dashboards-objects/index-pattern/logs-app.json.erb b/jobs/upload-dashboards-objects/templates/dashboards-objects/index-pattern/logs-app.json.erb
@@ -175,5 +175,7 @@ end
     "title": "<%= p('opensearch_config.app_index_prefix') %>*",
     "timeFieldName": "@timestamp",
     "fields": "<%= fields_json.to_json.gsub(/"/) { '\"' } %>"
-  }
+  },
+  "id": "<%= p('opensearch_config.app_index_prefix') %>*",
+  "type":"index-pattern"
 }
diff --git a/...load-dashboards-objects/templates/dashboards-objects/index-pattern/logs-platform.json.erb b/...load-dashboards-objects/templates/dashboards-objects/index-pattern/logs-platform.json.erb