From d5bd7a0be3f315ba05d53a1162a0cde74c54580f Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Thu, 14 Dec 2023 09:39:43 -0600 Subject: [PATCH] Remove commented out code and update field conversion in ps-rtt.conf --- configs/es-output.conf | 12 +- configs/ps-rtt.conf | 2 +- example_rtt_new.json | 605 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 612 insertions(+), 7 deletions(-) create mode 100644 example_rtt_new.json diff --git a/configs/es-output.conf b/configs/es-output.conf index 3d96d8a..e080aeb 100644 --- a/configs/es-output.conf +++ b/configs/es-output.conf @@ -2,17 +2,17 @@ input { pipeline { address => "es-output" } } output { + + + # stdout { + # codec => json + # } if "ps_rtt_write" in [@metadata][type]{ stdout { codec => json } } - - # stdout { - # codec => json - # } - - if "_jsonparsefailure" in [tags] { + else if "_jsonparsefailure" in [tags] { file { path => "/var/log/logstash/json_parse_failure.txt" } diff --git a/configs/ps-rtt.conf b/configs/ps-rtt.conf index c2b9451..7fadb9f 100644 --- a/configs/ps-rtt.conf +++ b/configs/ps-rtt.conf @@ -55,7 +55,7 @@ filter { if ![tags] { # for problematic things don't remove fields mutate{ - remove_field => ["type", "test", "result", "@timestamp"] + # remove_field => ["type", "test", "result", "@timestamp"] convert => { "ipv6" => "boolean" } } } diff --git a/example_rtt_new.json b/example_rtt_new.json new file mode 100644 index 0000000..4f0c10d --- /dev/null +++ b/example_rtt_new.json @@ -0,0 +1,605 @@ +{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "pic", + "lost": 10, + "src_site": "pic", + "dest_VO": "UNKNOWN", + "src_host": "picperfsonar-bandwidth.pic.es", + "@timestamp": "2023-12-14T15:33:11.244242176Z", + "loss": 1.0, + "destination": { + "ipv6": "2400:4500:0:2:0:0:0:164c", + "ipv4": "202.140.176.76" + }, + "reorders": 0, + "dest_rcsite": "Taiwan-LCG2", + "source": { + "ipv6": "2001:67c:1148:204:0:0:0:250", + "ipv4": "193.109.172.250" + }, + "src_netsite": "pic-LHCOPNE", + "dest": "2400:4500:0:2:0:0:0:164c", + "src_VO": "ATLAS", + "src": "2001:67c:1148:204:0:0:0:250", + "dest_site": "unknown", + "sent": 10, + "push": true, + "dest_host": "lhc-bandwidth.twgrid.org", + "@version": "1", + "received": 0, + "dest_netsite": "Taiwan-LCG2-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:10.218932748Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:09+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "pic", + "lost": 10, + "src_site": "pic", + "dest_VO": "ATLAS", + "src_host": "picperfsonar-latency.pic.es", + "@timestamp": "2023-12-14T15:33:12.350975387Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:660:5009:9:193:48:99:76", + "ipv4": "193.48.99.76" + }, + "reorders": 0, + "dest_rcsite": "IN2P3-CC", + "source": { + "ipv6": "2001:67c:1148:203:0:0:0:242", + "ipv4": "193.109.172.242" + }, + "src_netsite": "pic-LHCOPNE", + "dest": "2001:660:5009:9:193:48:99:76", + "src_VO": "ATLAS", + "src": "2001:67c:1148:203:0:0:0:242", + "dest_site": "IN2P3-CC-T3", + "sent": 10, + "push": true, + "dest_host": "ccperfsonar2.in2p3.fr", + "@version": "1", + "received": 0, + "dest_netsite": "IN2P3-CC-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:11.678161574Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:11+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "pic", + "lost": 10, + "src_site": "pic", + "dest_VO": "ATLAS", + "src_host": "picperfsonar-bandwidth.pic.es", + "@timestamp": "2023-12-14T15:33:18.071318524Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:660:5009:9:193:48:99:76", + "ipv4": "193.48.99.76" + }, + "reorders": 0, + "dest_rcsite": "IN2P3-CC", + "source": { + "ipv6": "2001:67c:1148:204:0:0:0:250", + "ipv4": "193.109.172.250" + }, + "src_netsite": "pic-LHCOPNE", + "dest": "2001:660:5009:9:193:48:99:76", + "src_VO": "ATLAS", + "src": "2001:67c:1148:204:0:0:0:250", + "dest_site": "IN2P3-CC-T3", + "sent": 10, + "push": true, + "dest_host": "ccperfsonar2.in2p3.fr", + "@version": "1", + "received": 0, + "dest_netsite": "IN2P3-CC-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:15.509326438Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:14+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "NDGF-T1", + "lost": 10, + "src_site": "NDGF-T1", + "dest_VO": "ATLAS", + "src_host": "perfsonar-ps.ndgf.org", + "@timestamp": "2023-12-14T15:33:21.955478757Z", + "loss": 1.0, + "destination": { + "ipv6": "2a00:139c:5:585:0:41:2:6", + "ipv4": "192.108.47.6" + }, + "reorders": 0, + "dest_rcsite": "FZK-LCG2", + "source": { + "ipv6": "2001:948:40:3:0:0:0:86", + "ipv4": "109.105.124.86" + }, + "src_netsite": "NDGF-T1-LHCOPNE", + "dest": "2a00:139c:5:585:0:41:2:6", + "src_VO": "ATLAS", + "src": "2001:948:40:3:0:0:0:86", + "dest_site": "FZK-LCG2", + "sent": 10, + "push": true, + "dest_host": "perfsonar-de-kit.gridka.de", + "@version": "1", + "received": 0, + "dest_netsite": "FZK-LCG2-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:21.579621015Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:20+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "INFN-T1", + "lost": 10, + "src_site": "INFN-T1", + "dest_VO": "UNKNOWN", + "src_host": "perfsonar-ps.cnaf.infn.it", + "@timestamp": "2023-12-14T15:33:22.508240182Z", + "loss": 1.0, + "destination": { + "ipv6": "2a05:81c5:301:228:0:0:0:e597", + "ipv4": "159.93.229.151" + }, + "reorders": 0, + "dest_rcsite": "JINR-T1", + "source": { + "ipv6": "2001:760:4205:254:0:0:0:11", + "ipv4": "131.154.254.11" + }, + "src_netsite": "INFN-T1-LHCOPNE", + "dest": "2a05:81c5:301:228:0:0:0:e597", + "src_VO": "ATLAS", + "src": "2001:760:4205:254:0:0:0:11", + "dest_site": "unknown", + "sent": 10, + "push": true, + "dest_host": "t1-pfsn2.jinr-t1.ru", + "@version": "1", + "received": 0, + "dest_netsite": "JINR-T1-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:21.931519696Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:20+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "AGLT2", + "lost": 10, + "src_site": "AGLT2", + "dest_VO": "ATLAS", + "src_host": "psmsu01.aglt2.org", + "@timestamp": "2023-12-14T15:33:27.613056452Z", + "loss": 1.0, + "destination": { + "ipv6": "2602:fd0f:0:1001:0:0:0:2", + "ipv4": "129.107.255.29" + }, + "reorders": 0, + "dest_rcsite": "SWT2_CPB", + "source": { + "ipv6": "2001:48a8:68f7:8001:192:41:236:31", + "ipv4": "192.41.236.31" + }, + "src_netsite": "AGLT2_MSU", + "dest": "2602:fd0f:0:1001:0:0:0:2", + "src_VO": "ATLAS", + "src": "2001:48a8:68f7:8001:192:41:236:31", + "dest_site": "SWT2_CPB", + "sent": 10, + "push": true, + "dest_host": "psuta01.atlas-swt2.org", + "@version": "1", + "received": 0, + "dest_netsite": "UTA_SWT2-LHCONE", + "ingest_timestamp": "2023-12-14T15:33:27.048313829Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:26+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "USCMS-FNAL-WC1", + "lost": 10, + "src_site": "T1_US_FNAL", + "dest_VO": "ATLAS", + "src_host": "psonar4.fnal.gov", + "@timestamp": "2023-12-14T15:33:29.316712179Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:630:58:1820:0:0:82f6:b06d", + "ipv4": "130.246.176.109" + }, + "reorders": 0, + "dest_rcsite": "RAL-LCG2", + "source": { + "ipv6": "2620:6a:0:8420:0:0:0:45", + "ipv4": "131.225.205.12" + }, + "src_netsite": "USCMS-FNAL-WC1-LHCOPNE", + "dest": "2001:630:58:1820:0:0:82f6:b06d", + "src_VO": "CMS", + "src": "2620:6a:0:8420:0:0:0:45", + "dest_site": "RAL-LCG2-ECHO", + "sent": 10, + "push": true, + "dest_host": "lcgps01.gridpp.rl.ac.uk", + "@version": "1", + "received": 0, + "dest_netsite": "RAL-LCG2-LHCOPN", + "ingest_timestamp": "2023-12-14T15:33:28.518017955Z", + "ipv6": true, + "timestamp": "2023-12-14T09:33:27-06:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "Taiwan-LCG2", + "lost": 10, + "src_site": "unknown", + "dest_VO": "ATLAS", + "src_host": "lhc-latency.twgrid.org", + "@timestamp": "2023-12-14T15:33:44.388149567Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:630:58:1820:0:0:82f6:b06e", + "ipv4": "130.246.176.110" + }, + "reorders": 0, + "dest_rcsite": "RAL-LCG2", + "source": { + "ipv6": "2400:4500:0:2:0:0:0:164b", + "ipv4": "202.140.176.75" + }, + "src_netsite": "Taiwan-LCG2-LHCOPNE", + "dest": "2001:630:58:1820:0:0:82f6:b06e", + "src_VO": "UNKNOWN", + "src": "2400:4500:0:2:0:0:0:164b", + "dest_site": "RAL-LCG2-ECHO", + "sent": 10, + "push": true, + "dest_host": "lcgps02.gridpp.rl.ac.uk", + "@version": "1", + "received": 0, + "dest_netsite": "RAL-LCG2-LHCOPN", + "ingest_timestamp": "2023-12-14T15:33:43.502387665Z", + "ipv6": true, + "timestamp": "2023-12-14T15:33:30Z" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "TRIUMF-LCG2", + "lost": 10, + "src_site": "TRIUMF-LCG2", + "dest_VO": "ATLAS", + "src_host": "ps-bandwidth.lhcmon.triumf.ca", + "@timestamp": "2023-12-14T15:33:47.785942835Z", + "loss": 1.0, + "destination": { + "ipv6": "2a00:139c:5:585:0:41:2:12", + "ipv4": "192.108.47.12" + }, + "reorders": 0, + "dest_rcsite": "FZK-LCG2", + "source": { + "ipv6": "2607:f8f0:660:2:0:0:0:2", + "ipv4": "206.12.9.1" + }, + "src_netsite": "TRIUMF-LCG2-LHCOPNE", + "dest": "2a00:139c:5:585:0:41:2:12", + "src_VO": "ATLAS", + "src": "2607:f8f0:660:2:0:0:0:2", + "dest_site": "FZK-LCG2", + "sent": 10, + "push": true, + "dest_host": "perfsonar2-de-kit.gridka.de", + "@version": "1", + "received": 0, + "dest_netsite": "FZK-LCG2-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:47.631838877Z", + "ipv6": true, + "timestamp": "2023-12-14T07:33:47-08:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "IN2P3-CC", + "lost": 10, + "src_site": "IN2P3-CC-T3", + "dest_VO": "ATLAS", + "src_host": "ccperfsonar1.in2p3.fr", + "@timestamp": "2023-12-14T15:33:48.849264577Z", + "loss": 1.0, + "destination": { + "ipv6": "2607:f8f0:660:2:0:0:0:3", + "ipv4": "206.12.9.2" + }, + "reorders": 0, + "dest_rcsite": "TRIUMF-LCG2", + "source": { + "ipv6": "2001:660:5009:9:193:48:99:77", + "ipv4": "193.48.99.77" + }, + "src_netsite": "IN2P3-CC-LHCOPNE", + "dest": "2607:f8f0:660:2:0:0:0:3", + "src_VO": "ATLAS", + "src": "2001:660:5009:9:193:48:99:77", + "dest_site": "TRIUMF-LCG2", + "sent": 10, + "push": true, + "dest_host": "ps-latency.lhcmon.triumf.ca", + "@version": "1", + "received": 0, + "dest_netsite": "TRIUMF-LCG2-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:33:47.941790318Z", + "ipv6": true, + "timestamp": "2023-12-14T16:33:47+01:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "SARA-MATRIX", + "lost": 10, + "src_site": "SARA-MATRIX", + "dest_VO": "ATLAS", + "src_host": "perfsonar-latency.grid.surfsara.nl", + "@timestamp": "2023-12-14T15:34:03.947053945Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:610:108:203a:0:0:0:32", + "ipv4": "145.100.32.32" + }, + "reorders": 0, + "dest_rcsite": "SARA-MATRIX", + "source": { + "ipv6": "2001:610:108:203a:0:0:0:31", + "ipv4": "145.100.32.31" + }, + "src_netsite": "NLT1-SARA-LHCOPNE", + "dest": "2001:610:108:203a:0:0:0:32", + "src_VO": "ATLAS", + "src": "2001:610:108:203a:0:0:0:31", + "dest_site": "SARA-MATRIX", + "sent": 10, + "push": true, + "dest_host": "perfsonar-bandwidth.grid.surfsara.nl", + "@version": "1", + "received": 0, + "dest_netsite": "NLT1-SARA-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:03.440531918Z", + "ipv6": true, + "timestamp": "2023-12-14T16:33:02+01:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "TRIUMF-LCG2", + "lost": 10, + "src_site": "TRIUMF-LCG2", + "dest_VO": "ATLAS", + "src_host": "ps-latency.lhcmon.triumf.ca", + "@timestamp": "2023-12-14T15:34:03.947078451Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:67c:1148:204:0:0:0:250", + "ipv4": "193.109.172.250" + }, + "reorders": 0, + "dest_rcsite": "pic", + "source": { + "ipv6": "2607:f8f0:660:2:0:0:0:3", + "ipv4": "206.12.9.2" + }, + "src_netsite": "TRIUMF-LCG2-LHCOPNE", + "dest": "2001:67c:1148:204:0:0:0:250", + "src_VO": "ATLAS", + "src": "2607:f8f0:660:2:0:0:0:3", + "dest_site": "pic", + "sent": 10, + "push": true, + "dest_host": "picperfsonar-bandwidth.pic.es", + "@version": "1", + "received": 0, + "dest_netsite": "pic-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:03.647687852Z", + "ipv6": true, + "timestamp": "2023-12-14T07:34:03-08:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "TRIUMF-LCG2", + "lost": 10, + "src_site": "TRIUMF-LCG2", + "dest_VO": "ATLAS", + "src_host": "ps-latency.lhcmon.triumf.ca", + "@timestamp": "2023-12-14T15:34:14.448957408Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:1458:301:86:0:0:100:6", + "ipv4": "128.142.208.134" + }, + "reorders": 0, + "dest_rcsite": "CERN-PROD", + "source": { + "ipv6": "2607:f8f0:660:2:0:0:0:3", + "ipv4": "206.12.9.2" + }, + "src_netsite": "TRIUMF-LCG2-LHCOPNE", + "dest": "2001:1458:301:86:0:0:100:6", + "src_VO": "ATLAS", + "src": "2607:f8f0:660:2:0:0:0:3", + "dest_site": "CERN-PROD", + "sent": 10, + "push": true, + "dest_host": "psl01-gva.cern.ch", + "@version": "1", + "received": 0, + "dest_netsite": "CERN-PROD-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:13.886215976Z", + "ipv6": true, + "timestamp": "2023-12-14T07:34:13-08:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "TRIUMF-LCG2", + "lost": 10, + "src_site": "TRIUMF-LCG2", + "dest_VO": "ATLAS", + "src_host": "ps-latency.lhcmon.triumf.ca", + "@timestamp": "2023-12-14T15:34:15.936305586Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:630:58:1820:0:0:82f6:b06e", + "ipv4": "130.246.176.110" + }, + "reorders": 0, + "dest_rcsite": "RAL-LCG2", + "source": { + "ipv6": "2607:f8f0:660:2:0:0:0:3", + "ipv4": "206.12.9.2" + }, + "src_netsite": "TRIUMF-LCG2-LHCOPNE", + "dest": "2001:630:58:1820:0:0:82f6:b06e", + "src_VO": "ATLAS", + "src": "2607:f8f0:660:2:0:0:0:3", + "dest_site": "RAL-LCG2-ECHO", + "sent": 10, + "push": true, + "dest_host": "lcgps02.gridpp.rl.ac.uk", + "@version": "1", + "received": 0, + "dest_netsite": "RAL-LCG2-LHCOPN", + "ingest_timestamp": "2023-12-14T15:34:15.457344527Z", + "ipv6": true, + "timestamp": "2023-12-14T07:34:15-08:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "INFN-T1", + "lost": 10, + "src_site": "INFN-T1", + "dest_VO": "ATLAS", + "src_host": "perfsonar-ps.cnaf.infn.it", + "@timestamp": "2023-12-14T15:34:22.484028974Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:610:108:203a:0:0:0:32", + "ipv4": "145.100.32.32" + }, + "reorders": 0, + "dest_rcsite": "SARA-MATRIX", + "source": { + "ipv6": "2001:760:4205:254:0:0:0:11", + "ipv4": "131.154.254.11" + }, + "src_netsite": "INFN-T1-LHCOPNE", + "dest": "2001:610:108:203a:0:0:0:32", + "src_VO": "ATLAS", + "src": "2001:760:4205:254:0:0:0:11", + "dest_site": "SARA-MATRIX", + "sent": 10, + "push": true, + "dest_host": "perfsonar-bandwidth.grid.surfsara.nl", + "@version": "1", + "received": 0, + "dest_netsite": "NLT1-SARA-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:21.645758149Z", + "ipv6": true, + "timestamp": "2023-12-14T15:34:20+00:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "TRIUMF-LCG2", + "lost": 10, + "src_site": "TRIUMF-LCG2", + "dest_VO": "ATLAS", + "src_host": "ps-latency.lhcmon.triumf.ca", + "@timestamp": "2023-12-14T15:34:48.891116586Z", + "loss": 1.0, + "destination": { + "ipv6": "2001:660:5009:9:193:48:99:77", + "ipv4": "193.48.99.77" + }, + "reorders": 0, + "dest_rcsite": "IN2P3-CC", + "source": { + "ipv6": "2607:f8f0:660:2:0:0:0:3", + "ipv4": "206.12.9.2" + }, + "src_netsite": "TRIUMF-LCG2-LHCOPNE", + "dest": "2001:660:5009:9:193:48:99:77", + "src_VO": "ATLAS", + "src": "2607:f8f0:660:2:0:0:0:3", + "dest_site": "IN2P3-CC-T3", + "sent": 10, + "push": true, + "dest_host": "ccperfsonar1.in2p3.fr", + "@version": "1", + "received": 0, + "dest_netsite": "IN2P3-CC-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:26.935986657Z", + "ipv6": true, + "timestamp": "2023-12-14T07:34:26-08:00" +}{ + "dest_production": true, + "src_production": true, + "duplicates": 0, + "src_rcsite": "pic", + "lost": 10, + "src_site": "pic", + "dest_VO": "ATLAS", + "src_host": "picperfsonar-bandwidth.pic.es", + "@timestamp": "2023-12-14T15:34:49.202708859Z", + "loss": 1.0, + "destination": { + "ipv6": "2620:0:210:1:0:0:0:23", + "ipv4": "192.12.15.23" + }, + "reorders": 0, + "dest_rcsite": "BNL-ATLAS", + "source": { + "ipv6": "2001:67c:1148:204:0:0:0:250", + "ipv4": "193.109.172.250" + }, + "src_netsite": "pic-LHCOPNE", + "dest": "2620:0:210:1:0:0:0:23", + "src_VO": "ATLAS", + "src": "2001:67c:1148:204:0:0:0:250", + "dest_site": "BNLLAKE", + "sent": 10, + "push": true, + "dest_host": "lhcmon.bnl.gov", + "@version": "1", + "received": 0, + "dest_netsite": "BNL-ATLAS-LHCOPNE", + "ingest_timestamp": "2023-12-14T15:34:35.241616549Z", + "ipv6": true, + "timestamp": "2023-12-14T15:34:34+00:00" +}[ + 2023-12-14T15: 34: 49, + 363 +][INFO +][logstash.outputs.file +][es-output +][fb147c310201af18f294da40faa574ec85a24127272cd053daef31235934b4d4 +] Opening file { + :path=>"/var/log/logstash/extra.json" +} \ No newline at end of file