From ede17dcb152838a300f127e5acdd95269482d1ad Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 15 Apr 2015 12:45:55 +0200 Subject: [PATCH 1/8] add couchbase support --- README.rst | 7 +++++++ docker/base/newrelic-plugin-agent.cfg | 6 ++++++ etc/newrelic/newrelic-plugin-agent.cfg | 6 ++++++ newrelic_plugin_agent/plugins/__init__.py | 1 + newrelic_plugin_agent/plugins/couchbase.py | 16 ++++++++++++++++ 5 files changed, 36 insertions(+) create mode 100644 newrelic_plugin_agent/plugins/couchbase.py diff --git a/README.rst b/README.rst index 917bf8f..6b515d1 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,7 @@ NewRelic platform. Currently supported backend systems are: - Alternative PHP Cache - Apache HTTP Server - CouchDB +- Couchbase - Elasticsearch - HAProxy - Memcached @@ -298,6 +299,12 @@ Configuration Example #username: foo #password: bar + couchbase: + name: localhost + host: localhost + port: 8091 + path: pools/default/buckets/test_bucket/stats + elasticsearch: name: clustername host: localhost diff --git a/docker/base/newrelic-plugin-agent.cfg b/docker/base/newrelic-plugin-agent.cfg index f957d04..fa4de07 100644 --- a/docker/base/newrelic-plugin-agent.cfg +++ b/docker/base/newrelic-plugin-agent.cfg @@ -22,6 +22,12 @@ Application: # username: foo # password: bar + #couchbase: + # name: localhost + # host: localhost + # port: 8091 + # path: pools/default/buckets/test_bucket/stats + #elasticsearch: # name: Clustername # host: localhost diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index f957d04..fa4de07 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -22,6 +22,12 @@ Application: # username: foo # password: bar + #couchbase: + # name: localhost + # host: localhost + # port: 8091 + # path: pools/default/buckets/test_bucket/stats + #elasticsearch: # name: Clustername # host: localhost diff --git a/newrelic_plugin_agent/plugins/__init__.py b/newrelic_plugin_agent/plugins/__init__.py index 98c5cb9..1a87a63 100644 --- a/newrelic_plugin_agent/plugins/__init__.py +++ b/newrelic_plugin_agent/plugins/__init__.py @@ -6,6 +6,7 @@ available = { 'apache_httpd': 'newrelic_plugin_agent.plugins.apache_httpd.ApacheHTTPD', 'couchdb': 'newrelic_plugin_agent.plugins.couchdb.CouchDB', + 'couchbase': 'newrelic_plugin_agent.plugins.couchbase.Couchbase', 'edgecast': 'newrelic_plugin_agent.plugins.edgecast.Edgecast', 'elasticsearch': 'newrelic_plugin_agent.plugins.elasticsearch.ElasticSearch', diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py new file mode 100644 index 0000000..3dde52b --- /dev/null +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -0,0 +1,16 @@ +""" +couchbase + +""" + +from newrelic_plugin_agent.plugins import base + + +class Couchbase(base.JSONStatsPlugin): + + GUID = 'com.meetme.couchbase' + + def add_datapoints(self, stats): + for key, values in stats['op']['samples'].items(): + for value in values: + self.add_derive_value(key, 'requests', value) From ee8001c6a4d36181acee19da19bbd1d7c4d29e76 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 15 Apr 2015 13:19:35 +0200 Subject: [PATCH 2/8] log polling URL on info --- newrelic_plugin_agent/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic_plugin_agent/plugins/base.py b/newrelic_plugin_agent/plugins/base.py index 2aa3980..7341f74 100644 --- a/newrelic_plugin_agent/plugins/base.py +++ b/newrelic_plugin_agent/plugins/base.py @@ -345,7 +345,7 @@ def http_get(self, url=None): :rtype: requests.models.Response """ - LOGGER.debug('Polling %s Stats at %s', + LOGGER.info('Polling %s Stats at %s', self.__class__.__name__, url or self.stats_url) req_kwargs = self.request_kwargs req_kwargs.update({'url': url} if url else {}) From 2d0930c14bc029aed9940d434d09e70526f715fa Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 16 Apr 2015 11:34:20 +0200 Subject: [PATCH 3/8] add nagiorelic --- README.rst | 8 +++++ docker/base/newrelic-plugin-agent.cfg | 3 ++ etc/newrelic/newrelic-plugin-agent.cfg | 3 ++ newrelic_plugin_agent/plugins/__init__.py | 3 +- newrelic_plugin_agent/plugins/nagiorelic.py | 35 +++++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 newrelic_plugin_agent/plugins/nagiorelic.py diff --git a/README.rst b/README.rst index 6b515d1..34a164c 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,11 @@ NewRelic platform. Currently supported backend systems are: - Redis - Riak - uWSGI +- NagioRelic + - this is not a backend + - it just executes programs (usually bash scripts) in a directory + - the program name is used as the metric name + - the output of the program must be numeric Base Requirements ----------------- @@ -414,6 +419,9 @@ Configuration Example port: 8098 #verify_ssl_cert: true + nagiorelic: + path: /etc/newrelic/nagiorelic + Daemon: user: newrelic pidfile: /var/run/newrelic/newrelic-plugin-agent.pid diff --git a/docker/base/newrelic-plugin-agent.cfg b/docker/base/newrelic-plugin-agent.cfg index fa4de07..86f9e0c 100644 --- a/docker/base/newrelic-plugin-agent.cfg +++ b/docker/base/newrelic-plugin-agent.cfg @@ -157,6 +157,9 @@ Application: # port: 1717 # path: /path/to/unix/socket + #nagiorelic: + # path: /etc/newrelic/nagiorelic + Daemon: user: newrelic pidfile: /var/run/newrelic/newrelic-plugin-agent.pid diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index fa4de07..86f9e0c 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -157,6 +157,9 @@ Application: # port: 1717 # path: /path/to/unix/socket + #nagiorelic: + # path: /etc/newrelic/nagiorelic + Daemon: user: newrelic pidfile: /var/run/newrelic/newrelic-plugin-agent.pid diff --git a/newrelic_plugin_agent/plugins/__init__.py b/newrelic_plugin_agent/plugins/__init__.py index 1a87a63..5b80596 100644 --- a/newrelic_plugin_agent/plugins/__init__.py +++ b/newrelic_plugin_agent/plugins/__init__.py @@ -21,4 +21,5 @@ 'rabbitmq': 'newrelic_plugin_agent.plugins.rabbitmq.RabbitMQ', 'redis': 'newrelic_plugin_agent.plugins.redis.Redis', 'riak': 'newrelic_plugin_agent.plugins.riak.Riak', - 'uwsgi': 'newrelic_plugin_agent.plugins.uwsgi.uWSGI'} + 'uwsgi': 'newrelic_plugin_agent.plugins.uwsgi.uWSGI', + 'nagiorelic': 'newrelic_plugin_agent.plugins.nagiorelic.NagioRelic'} diff --git a/newrelic_plugin_agent/plugins/nagiorelic.py b/newrelic_plugin_agent/plugins/nagiorelic.py new file mode 100644 index 0000000..d521396 --- /dev/null +++ b/newrelic_plugin_agent/plugins/nagiorelic.py @@ -0,0 +1,35 @@ +""" +PostgreSQL Plugin + +""" +import logging +import os +import subprocess +from subprocess import CalledProcessError + +from newrelic_plugin_agent.plugins import base + + +LOGGER = logging.getLogger(__name__) + + +class NagioRelic(base.Plugin): + + GUID = 'com.meetme.nagiorelic' + + def poll(self): + self.initialize() + + root = self.config['path'] + for script in os.listdir(root): + try: + output = subprocess.check_output(os.path.join(root, script)).decode().strip() + if output.isdigit(): + metric = '%s' % (os.path.splitext(os.path.basename(script))[0]) + self.add_gauge_value(metric, None, int(output)) + else: + LOGGER.error('command did not return a number %s: %s' % (script, output)) + except CalledProcessError as e: + LOGGER.error('command returned an error %s: %s: %s' % (script, e.returncode, e.output)) + + self.finish() From b3a3b7b9fc6973acc921ad81d757ccb6f763cf12 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 16 Apr 2015 11:35:40 +0200 Subject: [PATCH 4/8] add nagiorelic --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 34a164c..468c9d6 100644 --- a/README.rst +++ b/README.rst @@ -21,10 +21,10 @@ NewRelic platform. Currently supported backend systems are: - Riak - uWSGI - NagioRelic - - this is not a backend - - it just executes programs (usually bash scripts) in a directory - - the program name is used as the metric name - - the output of the program must be numeric + + this is not a backend + + it just executes programs (usually bash scripts) in a directory + + the program name is used as the metric name + + the output of the program must be numeric Base Requirements ----------------- From b983f7f1f89c1f8e080983d2fdea2ed4ad411918 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 16 Apr 2015 11:44:33 +0200 Subject: [PATCH 5/8] add nagiorelic --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 468c9d6..296933b 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,7 @@ NewRelic platform. Currently supported backend systems are: - Riak - uWSGI - NagioRelic + + this is not a backend + it just executes programs (usually bash scripts) in a directory + the program name is used as the metric name From eb58aa6391e747903f9c76fdbb622a64d75dc8c3 Mon Sep 17 00:00:00 2001 From: Fabian Brosig Date: Thu, 17 Sep 2015 16:32:42 +0200 Subject: [PATCH 6/8] Couchbase plugin: read basicStats from all couchbase buckets --- README.rst | 2 +- docker/base/newrelic-plugin-agent.cfg | 2 +- etc/newrelic/newrelic-plugin-agent.cfg | 2 +- newrelic_plugin_agent/plugins/couchbase.py | 4 ++-- newrelic_plugin_agent/plugins/nagiorelic.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 296933b..5d4826b 100644 --- a/README.rst +++ b/README.rst @@ -309,7 +309,7 @@ Configuration Example name: localhost host: localhost port: 8091 - path: pools/default/buckets/test_bucket/stats + path: pools/default/buckets elasticsearch: name: clustername diff --git a/docker/base/newrelic-plugin-agent.cfg b/docker/base/newrelic-plugin-agent.cfg index 86f9e0c..a0b1695 100644 --- a/docker/base/newrelic-plugin-agent.cfg +++ b/docker/base/newrelic-plugin-agent.cfg @@ -26,7 +26,7 @@ Application: # name: localhost # host: localhost # port: 8091 - # path: pools/default/buckets/test_bucket/stats + # path: pools/default/buckets #elasticsearch: # name: Clustername diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index 86f9e0c..a0b1695 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -26,7 +26,7 @@ Application: # name: localhost # host: localhost # port: 8091 - # path: pools/default/buckets/test_bucket/stats + # path: pools/default/buckets #elasticsearch: # name: Clustername diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py index 3dde52b..aca53c2 100644 --- a/newrelic_plugin_agent/plugins/couchbase.py +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -11,6 +11,6 @@ class Couchbase(base.JSONStatsPlugin): GUID = 'com.meetme.couchbase' def add_datapoints(self, stats): - for key, values in stats['op']['samples'].items(): - for value in values: + for _bucket in stats: + for key, value in _bucket['basicStats'].items(): self.add_derive_value(key, 'requests', value) diff --git a/newrelic_plugin_agent/plugins/nagiorelic.py b/newrelic_plugin_agent/plugins/nagiorelic.py index d521396..f0d4011 100644 --- a/newrelic_plugin_agent/plugins/nagiorelic.py +++ b/newrelic_plugin_agent/plugins/nagiorelic.py @@ -1,5 +1,5 @@ """ -PostgreSQL Plugin +Nagiorelic Plugin """ import logging From 4e0695788575805e351863384eacf4ab1af1e708 Mon Sep 17 00:00:00 2001 From: Fabian Brosig Date: Thu, 17 Sep 2015 18:38:02 +0200 Subject: [PATCH 7/8] Couchbase plugin: basicStats now per bucket sent to newrelic --- newrelic_plugin_agent/plugins/couchbase.py | 50 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py index aca53c2..5c8f6df 100644 --- a/newrelic_plugin_agent/plugins/couchbase.py +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -11,6 +11,50 @@ class Couchbase(base.JSONStatsPlugin): GUID = 'com.meetme.couchbase' def add_datapoints(self, stats): - for _bucket in stats: - for key, value in _bucket['basicStats'].items(): - self.add_derive_value(key, 'requests', value) + for bucket_data in stats: + bucket_name = bucket_data.get('name', default='') + bucket_stats = bucket_data.get('basicStats', default=None) + if bucket_stats: + """ Example JSON + "basicStats":{ + "quotaPercentUsed":7.548735046386719, + "opsPerSec":0, + "diskFetches":0, + "itemCount":2222, + "diskUsed":229366539, + "dataUsed":203511808, + "memUsed":79154224 + } + """ + self.add_gauge_bucket_metric( + bucket_name, 'quotaPercentUsed', 'percent', bucket_stats.get('quotaPercentUsed', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'opsPerSec', 'ops', bucket_stats.get('opsPerSec', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'diskFetches', 'count', bucket_stats.get('diskFetches', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'itemCount', 'count', bucket_stats.get('itemCount', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'diskUsed', 'Byte', bucket_stats.get('diskUsed', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'dataUsed', 'Byte', bucket_stats.get('dataUsed', default=None) + ) + self.add_gauge_bucket_metric( + bucket_name, 'memUsed', 'Byte', bucket_stats.get('memUsed', default=None) + ) + + # Summary metrics + self.add_gauge_value('Summary/%s/quotaPercentUsed' % bucket_name, 'percent', + bucket_stats.get('quotaPercentUsed'), + min_val=0, max_val=0) + self.add_gauge_value('Summary/%s/diskUsed' % bucket_name, 'byte', + bucket_stats.get('diskUsed')) + + def add_gauge_bucket_metric(self, bucket_name, metric_name, units, metric_value): + if metric_value: + self.add_gauge_value('%s/%s' % (bucket_name, metric_name), units, metric_value) From 33c9162f920cd821c6d051302b73ebb2ab27419d Mon Sep 17 00:00:00 2001 From: Fabian Brosig Date: Thu, 17 Sep 2015 18:43:18 +0200 Subject: [PATCH 8/8] Couchbase plugin: fix JSON access --- newrelic_plugin_agent/plugins/couchbase.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py index 5c8f6df..35c3f72 100644 --- a/newrelic_plugin_agent/plugins/couchbase.py +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -12,8 +12,8 @@ class Couchbase(base.JSONStatsPlugin): def add_datapoints(self, stats): for bucket_data in stats: - bucket_name = bucket_data.get('name', default='') - bucket_stats = bucket_data.get('basicStats', default=None) + bucket_name = bucket_data['name'] + bucket_stats = bucket_data['basicStats'] if bucket_stats: """ Example JSON "basicStats":{ @@ -27,33 +27,33 @@ def add_datapoints(self, stats): } """ self.add_gauge_bucket_metric( - bucket_name, 'quotaPercentUsed', 'percent', bucket_stats.get('quotaPercentUsed', default=None) + bucket_name, 'quotaPercentUsed', 'percent', bucket_stats['quotaPercentUsed'] ) self.add_gauge_bucket_metric( - bucket_name, 'opsPerSec', 'ops', bucket_stats.get('opsPerSec', default=None) + bucket_name, 'opsPerSec', 'ops', bucket_stats['opsPerSec'] ) self.add_gauge_bucket_metric( - bucket_name, 'diskFetches', 'count', bucket_stats.get('diskFetches', default=None) + bucket_name, 'diskFetches', 'count', bucket_stats['diskFetches'] ) self.add_gauge_bucket_metric( - bucket_name, 'itemCount', 'count', bucket_stats.get('itemCount', default=None) + bucket_name, 'itemCount', 'count', bucket_stats['itemCount'] ) self.add_gauge_bucket_metric( - bucket_name, 'diskUsed', 'Byte', bucket_stats.get('diskUsed', default=None) + bucket_name, 'diskUsed', 'Byte', bucket_stats['diskUsed'] ) self.add_gauge_bucket_metric( - bucket_name, 'dataUsed', 'Byte', bucket_stats.get('dataUsed', default=None) + bucket_name, 'dataUsed', 'Byte', bucket_stats['dataUsed'] ) self.add_gauge_bucket_metric( - bucket_name, 'memUsed', 'Byte', bucket_stats.get('memUsed', default=None) + bucket_name, 'memUsed', 'Byte', bucket_stats['memUsed'] ) # Summary metrics self.add_gauge_value('Summary/%s/quotaPercentUsed' % bucket_name, 'percent', - bucket_stats.get('quotaPercentUsed'), + bucket_stats['quotaPercentUsed'], min_val=0, max_val=0) self.add_gauge_value('Summary/%s/diskUsed' % bucket_name, 'byte', - bucket_stats.get('diskUsed')) + bucket_stats['diskUsed']) def add_gauge_bucket_metric(self, bucket_name, metric_name, units, metric_value): if metric_value: