From 17adb9993dd4037642b3285bd55748b163f05b98 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 16 Dec 2024 10:36:45 -0500 Subject: [PATCH 1/2] removing curator --- config/blobs.yml | 76 ---------------- jobs/curator/monit | 0 jobs/curator/spec | 91 ------------------- jobs/curator/templates/bin/pre-start | 43 --------- jobs/curator/templates/bin/run-curator | 13 --- jobs/curator/templates/config/actions.yml.erb | 44 --------- jobs/curator/templates/config/admin-crt.erb | 3 - jobs/curator/templates/config/admin-pem.erb | 3 - jobs/curator/templates/config/ca.erb | 3 - jobs/curator/templates/config/config.yml.erb | 19 ---- packages/curator/packaging | 22 ----- packages/curator/spec | 9 -- 12 files changed, 326 deletions(-) delete mode 100644 jobs/curator/monit delete mode 100644 jobs/curator/spec delete mode 100644 jobs/curator/templates/bin/pre-start delete mode 100644 jobs/curator/templates/bin/run-curator delete mode 100644 jobs/curator/templates/config/actions.yml.erb delete mode 100644 jobs/curator/templates/config/admin-crt.erb delete mode 100644 jobs/curator/templates/config/admin-pem.erb delete mode 100644 jobs/curator/templates/config/ca.erb delete mode 100644 jobs/curator/templates/config/config.yml.erb delete mode 100644 packages/curator/packaging delete mode 100644 packages/curator/spec diff --git a/config/blobs.yml b/config/blobs.yml index c44ca4bb..a5fb8d4e 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -122,82 +122,6 @@ auth-proxy/werkzeug-3.1.2-py3-none-any.whl: size: 224352 object_id: 45af49a7-9728-4fdb-5576-6627b8e4fbb7 sha: sha256:4f7d1a5de312c810a8a2c6f0b47e9f6a7cffb7c8322def35e4d4d9841ff85597 -curator_opensearch-0.0.13.tar.gz: - size: 236782 - object_id: 9c602cc3-eb88-4279-57b4-eded33d1338f - sha: sha256:c97f45a038e19c71227f0d88baa370d737f2b3a18fe1416d448c21c3a1bf2714 -curator_opensearch/vendor/Events-0.5-py3-none-any.whl: - size: 6758 - object_id: e97f361f-72f4-448f-490a-886f65c053b5 - sha: sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd -curator_opensearch/vendor/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl: - size: 767542 - object_id: 1a94d4cb-82e9-4c5a-60d4-ef18ebbea1b4 - sha: sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 -curator_opensearch/vendor/boto3-1.35.55-py3-none-any.whl: - size: 139179 - object_id: f7813215-70ab-4077-484f-95f825f894f3 - sha: sha256:c7a0a0bc5ae3bed5d38e8bfe5a56b31621e79bdd7c1ea6e5ba4326d820cde3a5 -curator_opensearch/vendor/botocore-1.35.55-py3-none-any.whl: - size: 12713468 - object_id: 2f075282-05fc-48c7-49f9-71903bd9de95 - sha: sha256:3d54739e498534c9d7a6e9732ae2d17ed29c7d5e29fe36c956d8488b859538b0 -curator_opensearch/vendor/certifi-2024.8.30-py3-none-any.whl: - size: 167321 - object_id: f9859e14-b0ef-4d3d-7fd5-3b49f604943c - sha: sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 -curator_opensearch/vendor/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl: - size: 143838 - object_id: b20e567d-d590-474e-69c0-6d8e9090bea5 - sha: sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15 -curator_opensearch/vendor/click-7.1.2-py2.py3-none-any.whl: - size: 82780 - object_id: 2e64f942-e0c1-4b7f-664b-038aad18952e - sha: sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc -curator_opensearch/vendor/curator_opensearch-0.0.12-py2.py3-none-any.whl: - size: 107475 - object_id: a56efef8-a39f-4de1-73a8-d2def3f13e70 - sha: sha256:a284addcb616e2257f8042d764195917c22c046ae930011150717040412f890a -curator_opensearch/vendor/idna-3.10-py3-none-any.whl: - size: 70442 - object_id: fe17826c-dea7-4b41-654f-4539287d8205 - sha: sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 -curator_opensearch/vendor/jmespath-1.0.1-py3-none-any.whl: - size: 20256 - object_id: 1ad75ca2-ebe1-460e-6213-3b719aae5692 - sha: sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 -curator_opensearch/vendor/opensearch_py-2.7.1-py3-none-any.whl: - size: 325380 - object_id: b36abe12-8ee5-4a5f-4026-79750b5f9010 - sha: sha256:5417650eba98a1c7648e502207cebf3a12beab623ffe0ebbf55f9b1b4b6e44e9 -curator_opensearch/vendor/python_dateutil-2.9.0.post0-py2.py3-none-any.whl: - size: 229892 - object_id: 004b1e17-d221-440e-7af4-57179d0e035a - sha: sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 -curator_opensearch/vendor/requests-2.32.3-py3-none-any.whl: - size: 64928 - object_id: 0d47779d-ea69-4667-7732-1baf054e7101 - sha: sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 -curator_opensearch/vendor/requests_aws4auth-1.3.1-py3-none-any.whl: - size: 24584 - object_id: a35ce536-1c21-4236-4247-79dcb4c9ad81 - sha: sha256:2969b5379ae6e60ee666638caf6cb94a32d67033f6bfcf0d50c95cd5474f2419 -curator_opensearch/vendor/s3transfer-0.10.3-py3-none-any.whl: - size: 82625 - object_id: 8400dcee-b10f-483a-66cc-24e7b07cec11 - sha: sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d -curator_opensearch/vendor/six-1.16.0-py2.py3-none-any.whl: - size: 11053 - object_id: 83cb138b-eeb0-4dd0-62d3-2ca2db6f1f2f - sha: sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 -curator_opensearch/vendor/urllib3-1.26.20-py2.py3-none-any.whl: - size: 144225 - object_id: 32797f40-0359-4ef8-6cfd-eb59af93d98f - sha: sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e -curator_opensearch/vendor/voluptuous-0.15.2-py3-none-any.whl: - size: 31349 - object_id: cab6f4d3-79bd-4d78-4ff3-f10cfddb69c5 - sha: sha256:016348bc7788a9af9520b1764ebd4de0df41fe2138ebe9e06fa036bf86a65566 logstash/logstash-8.16.1-linux-x86_64.tar.gz: size: 428558276 object_id: 6448195b-fb5b-4a22-5f16-759c29296cf6 diff --git a/jobs/curator/monit b/jobs/curator/monit deleted file mode 100644 index e69de29b..00000000 diff --git a/jobs/curator/spec b/jobs/curator/spec deleted file mode 100644 index 4001e699..00000000 --- a/jobs/curator/spec +++ /dev/null @@ -1,91 +0,0 @@ ---- -name: curator - -description: This job runs Curator process to purge outdated info from opensearch cluster - -packages: - - curator - - python3 - -templates: - bin/pre-start: bin/pre-start - bin/run-curator: bin/run-curator - config/actions.yml.erb: config/actions.yml - config/config.yml.erb: config/config.yml - config/admin-crt.erb: config/ssl/opensearch-admin.crt - config/admin-pem.erb: config/ssl/opensearch-admin.pem - config/ca.erb: config/ssl/opensearch.ca - - -consumes: -- name: opensearch - type: opensearch - -provides: -- name: curator - type: curator - -properties: - opensearch.snapshots.repository: - description: Repository name for automatic snapshots - opensearch.path_repo: - description: | - Shared file system to store snapshots. - In order to register the shared file system repository it is - necessary to mount the same shared filesystem to the same location - on all master and data nodes. - default: '' - opensearch.cloud.aws.bucket: - description: Bucket name on S3 where to keep snapshots - default: '' - - curator.execute.hourly: - description: "Execute Curator hourly?" - default: true - curator.execute.daily: - description: "Execute Curator daily?" - default: false - curator.execute.weekly: - description: "Execute Curator weekly?" - default: false - - curator.opensearch.hosts: - description: IP address of opensearch host to proxy requests for (eg, 127.0.0.1) - default: - - "localhost" - curator.opensearch.port: - description: Port address of opensearch host to proxy requests for (eg, 9200) - default: 9200 - curator.purge_logs.unit: - description: "One of the following values: seconds, minutes, hours, days, weeks, months, years" - default: "days" - curator.purge_logs.retention_period: - description: The value of this setting will be used as a multiplier for unit - default: 30 - curator.loglevel: - description: Set the minimum acceptable log severity to display. - default: INFO - curator.logformat: - description: This should default, json, logstash. - default: default - curator.actions: - description: List of opensearch Curator actions in YAML format. - example: | - - action: delete_indices - description: >- - Delete indices older than 45 days (based on index name), for logstash- - prefixed indices. Ignore the error if the filter does not result in an - actionable list of indices (ignore_empty_list) and exit cleanly. - options: - ignore_empty_list: True - disable_action: True - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 45 \ No newline at end of file diff --git a/jobs/curator/templates/bin/pre-start b/jobs/curator/templates/bin/pre-start deleted file mode 100644 index 24f4cc06..00000000 --- a/jobs/curator/templates/bin/pre-start +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -<% if p('curator.execute.hourly') %> -# Execute Curator hourly -cp /var/vcap/jobs/curator/bin/run-curator /etc/cron.hourly -<% else %> -# Remove pre-existing hourly script -rm -f /etc/cron.hourly/run-curator -<% end %> - -<% if p('curator.execute.daily') %> -# Execute Curator daily -cp /var/vcap/jobs/curator/bin/run-curator /etc/cron.daily -<% else %> -# Remove pre-existing daily script -rm -f /etc/cron.daily/run-curator -<% end %> - -<% if p('curator.execute.weekly') %> -# Execute curator weekly -cp /var/vcap/jobs/curator/bin/run-curator /etc/cron.weekly -<% else %> -# Remove pre-existing weekly script -rm -f /etc/cron.weekly/run-curator -<% end %> - -export JOB_NAME=curator -export JOB_DIR=/var/vcap/jobs/$JOB_NAME - -<% if_link('opensearch') do |curator| %> -<% curator.if_p('opensearch.admin.private_key') do %> - openssl pkcs8 -v1 "PBE-SHA1-3DES" \ - -in "${JOB_DIR}/config/ssl/opensearch-admin.pem" -topk8 \ - -out "${JOB_DIR}/config/ssl/opensearch-admin.key" -nocrypt - chmod 600 ${JOB_DIR}/config/ssl/opensearch*.key -<% end %> -<% end %> - -if [ -d ${JOB_DIR}/config/ssl ]; then - chown -R vcap:vcap ${JOB_DIR}/config/ssl -fi \ No newline at end of file diff --git a/jobs/curator/templates/bin/run-curator b/jobs/curator/templates/bin/run-curator deleted file mode 100644 index 94a1746b..00000000 --- a/jobs/curator/templates/bin/run-curator +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x -export LC_ALL=en_US.UTF-8 -export LANG=en_US.UTF-8 - -export PATH=/var/vcap/packages/curator/bin:$PATH -export CONFIG_DIR=/var/vcap/jobs/curator/config -export PYTHONPATH=/var/vcap/packages/curator/lib/python3.12/site-packages/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python3/lib - -curator --config $CONFIG_DIR/config.yml $CONFIG_DIR/actions.yml \ No newline at end of file diff --git a/jobs/curator/templates/config/actions.yml.erb b/jobs/curator/templates/config/actions.yml.erb deleted file mode 100644 index ecc90948..00000000 --- a/jobs/curator/templates/config/actions.yml.erb +++ /dev/null @@ -1,44 +0,0 @@ ---- -actions: - <% if_p('opensearch.snapshots.repository') do |repo| %> - 1: - action: snapshot - description: >- - Create snapshot for all indicies - options: - disable_action: <% if p('opensearch.path_repo') != '' or p('opensearch.cloud.aws.bucket') != '' %>False<% else %>True<% end %> - repository: <%= repo %> - wait_for_completion: True - ignore_empty_list: True - filters: - - filtertype: pattern - kind: prefix - value: logs- - - filtertype: age - source: creation_date - direction: older - unit: <%= p('curator.purge_logs.unit') %> - unit_count: <%= p('curator.purge_logs.retention_period') %> - <% end %> - 2: - action: delete_indices - description: >- - Delete indices older than <%= p('curator.purge_logs.retention_period') %> <%= p('curator.purge_logs.unit') %> (based on index name), for logs- - prefixed indices. Ignore the error if the filter does not result in an - actionable list of indices (ignore_empty_list) and exit cleanly. - options: - ignore_empty_list: True - disable_action: False - filters: - ## removed until the future when logs are created - # - filtertype: pattern - # kind: regex - # value: logs* - - filtertype: age - source: creation_date - direction: older - unit: <%= p('curator.purge_logs.unit') %> - unit_count: <%= p('curator.purge_logs.retention_period') %> -<% p("curator.actions", []).each_with_index do | action, index | %> - <%= index + 3 %>:<%= action.to_yaml.gsub("---", "").gsub("\n", "\n ") %> -<% end %> \ No newline at end of file diff --git a/jobs/curator/templates/config/admin-crt.erb b/jobs/curator/templates/config/admin-crt.erb deleted file mode 100644 index 9c59c1d5..00000000 --- a/jobs/curator/templates/config/admin-crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_link('opensearch') do |opensearchconfig| %> -<%= opensearchconfig.p('opensearch.admin.certificate', '') %> -<% end %> diff --git a/jobs/curator/templates/config/admin-pem.erb b/jobs/curator/templates/config/admin-pem.erb deleted file mode 100644 index 39d3495c..00000000 --- a/jobs/curator/templates/config/admin-pem.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_link('opensearch') do |opensearchconfig| %> -<%= opensearchconfig.p('opensearch.admin.private_key', '') %> -<% end %> diff --git a/jobs/curator/templates/config/ca.erb b/jobs/curator/templates/config/ca.erb deleted file mode 100644 index e490c88d..00000000 --- a/jobs/curator/templates/config/ca.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_link('opensearch') do |opensearchconfig| %> -<%= opensearchconfig.p('opensearch.node.ssl.ca', '') %> -<% end %> diff --git a/jobs/curator/templates/config/config.yml.erb b/jobs/curator/templates/config/config.yml.erb deleted file mode 100644 index 15f695df..00000000 --- a/jobs/curator/templates/config/config.yml.erb +++ /dev/null @@ -1,19 +0,0 @@ ---- -client: - hosts: <%= p("curator.opensearch.hosts") %> - port: <%= p("curator.opensearch.port") %> - url_prefix: - use_ssl: True - certificate: /var/vcap/jobs/curator/config/ssl/opensearch.ca - client_cert: /var/vcap/jobs/curator/config/ssl/opensearch-admin.crt - client_key: /var/vcap/jobs/curator/config/ssl/opensearch-admin.key - ssl_no_validate: False - http_auth: - timeout: 30 - master_only: False - -logging: - loglevel: <%= p('curator.loglevel') %> - logfile: - logformat: <%= p('curator.logformat') %> - blacklist: ['opensearch', 'urllib3'] \ No newline at end of file diff --git a/packages/curator/packaging b/packages/curator/packaging deleted file mode 100644 index d373665d..00000000 --- a/packages/curator/packaging +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e - -export PATH="/var/vcap/packages/python3/bin:${PATH}" -export LD_LIBRARY_PATH="/var/vcap/packages/python3/lib:${LD_LIBRARY_PATH}" -export C_INCLUDE_PATH="/var/vcap/packages/python3/include:${C_INCLUDE_PATH}" - -# To download updated blobs for this package, see bin/download-wheels.sh. For example: -# -# ./bin/download-wheels/curator_opensearch -# -# Then, add the downloaded blobs to the release: -# -# ./bin/add-wheel-blobs.sh vendor/curator_opensearch curator_opensearch/vendor - -# --no-index prevents contacting pypi to download packages -# --find-links tells pip where to look for the dependancies -# --prefix installation prefix where lib, bin and other top-level folders are placed - -python3 -m pip install "--prefix=${BOSH_INSTALL_TARGET}" --no-index curator_opensearch/vendor/*.whl -tar xzvf curator_opensearch-0.0.13.tar.gz -C "$BOSH_INSTALL_TARGET" --strip-components 1 diff --git a/packages/curator/spec b/packages/curator/spec deleted file mode 100644 index aa983c93..00000000 --- a/packages/curator/spec +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: curator - -dependencies: - - python3 - -files: - - curator_opensearch-0.0.13.tar.gz - - curator_opensearch/vendor/* From 1806888af94926c32b3c779447daf2508ba7192f Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 16 Dec 2024 10:37:22 -0500 Subject: [PATCH 2/2] removing curator --- .../src/logstash-filters/deployment_lookup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/base-logstash-filters/src/logstash-filters/deployment_lookup.yml b/src/base-logstash-filters/src/logstash-filters/deployment_lookup.yml index 88cf341f..f987240a 100644 --- a/src/base-logstash-filters/src/logstash-filters/deployment_lookup.yml +++ b/src/base-logstash-filters/src/logstash-filters/deployment_lookup.yml @@ -1,5 +1,4 @@ opensearch*: logs-opensearch -curator.*: logs-opensearch ingestor.*: logs-opensearch archiver.*: logs-opensearch parser.*: logs-opensearch