initial release of graylog 2.2.2

.gitignore

@@ -0,0 +1,16 @@
+config/dev.yml
+config/private.yml
+releases/*.tgz
+releases/**/*.tgz
+dev_releases
+.blobs
+blobs
+.dev_builds
+.idea
+.DS_Store
+.final_builds/jobs/**/*.tgz
+.final_builds/packages/**/*.tgz
+*.swp
+*~
+*#
+#*

README.md

@@ -0,0 +1,29 @@
+# graylog-boshrelease
+BOSH release for graylog (https://www.graylog.org/) - tool for centrally collecting logging events for your infrastructure and applications.
+
+NOTE: this is initial development release and is not recommended for production environments.  it has currently only been tested with BOSH-lite.
+
+### Creating a development bosh release
+```
+bosh create release --force
+bosh upload release
+```
+
+### Deploying to *bosh-lite*.
+a sample cloud-config and deployment manifest is provided in the `templates` directory.
+
+```
+bosh update cloud-config templates/bosh-lite-cloud-config.yml
+bosh deployment templates/bosh-lite-deployment.yml
+bosh deploy
+```
+
+
+### attribution
+This BOSH release for Graylog was heavily inspired by an existing BOSH release for the ELK stack - https://github.com/cloudfoundry-community/logsearch-boshrelease.  
+A large amount of the code for the _elasticsearch_ jobs and packages has been copied from the ELK BOSH release and re-used here.
+
+
+### TODO
+- add authentication to mongodb
+- add better management of multiple graylog instances

config/blobs.yml

@@ -0,0 +1,17 @@
+---
+elasticsearch/elasticsearch-2.4.4.tar.gz:
+  object_id: 06f85038-ba22-46f7-a7c7-058d0d6aa7c2
+  sha: cdb5068d1baa07388e522c3bc04cca38aa8f3048
+  size: 27343272
+mongodb/mongodb-linux-x86_64-3.4.2.tgz:
+  object_id: a68edeb2-27a4-4532-96e1-09b00145f9c7
+  sha: 540ed237905235ac4ca40bbfa0a78324f5d1db4f
+  size: 86473759
+openjdk/openjdk-1.8.0_111.tar.gz:
+  object_id: d3e89d5f-e28e-41f8-8633-988803c304c0
+  sha: d5ca703f6115cb0564651c2ab7f962030f70d594
+  size: 77747700
+graylog/graylog-2.2.2.tgz:
+  object_id: dee996d9-8ed8-453e-85cb-27ea5362ff0c
+  sha: 571289739173b9644a43e94d526912c1d0e1e4a9
+  size: 99956086

config/final.yml

@@ -0,0 +1,8 @@
+---
+blobstore:
+  provider: s3
+  options:
+    region: ap-southeast-2
+    endpoint: https://s3-ap-southeast-2.amazonaws.com
+    bucket_name: dta-graylog-boshrelease
+final_name: graylog

jobs/elasticsearch/monit

@@ -0,0 +1,11 @@
+check process elasticsearch
+  with pidfile /var/vcap/sys/run/elasticsearch/elasticsearch.pid
+  start program "/var/vcap/jobs/elasticsearch/bin/monit_debugger elasticsearch_ctl '/var/vcap/jobs/elasticsearch/bin/elasticsearch_ctl start'" with timeout 120 seconds
+  stop program "/var/vcap/jobs/elasticsearch/bin/monit_debugger elasticsearch_ctl '/var/vcap/jobs/elasticsearch/bin/elasticsearch_ctl stop'"
+  group vcap
+
+check device elasticsearch-ephemeral_disk with path /var/vcap/data
+  if SPACE usage > 80% then alert
+
+check device elasticsearch-persistent_disk with path /var/vcap/store
+  if SPACE usage > 80% then alert

jobs/elasticsearch/spec

@@ -0,0 +1,56 @@
+---
+name: elasticsearch
+packages:
+- elasticsearch
+- java
+
+templates:
+  bin/drain.erb: bin/drain
+  bin/post-deploy.erb: bin/post-deploy
+  bin/elasticsearch_ctl: bin/elasticsearch_ctl
+  bin/monit_debugger: bin/monit_debugger
+  config/config.yml.erb: config/elasticsearch.yml
+  config/logging.yml.erb: config/logging.yml
+  data/properties.sh.erb: data/properties.sh
+  helpers/ctl_setup.sh: helpers/ctl_setup.sh
+  helpers/ctl_utils.sh: helpers/ctl_utils.sh
+
+properties:
+  elasticsearch.master_hosts:
+    description: The list of elasticsearch master node IPs
+  elasticsearch.cluster_name:
+    description: The name of the elastic search cluster
+  elasticsearch.log_level:
+    description: The default logging level (e.g. WARN, DEBUG, INFO)
+    default: INFO
+  elasticsearch.node.allow_master:
+    description: Allow node to become master? (true / false)
+    default: false
+  elasticsearch.node.allow_data:
+    description: Allow node to store data? (true / false)
+    default: false
+  elasticsearch.node.tags:
+    description: A hash of additional tags for the node
+  elasticsearch.exec.environment:
+    description: A hash of additional environment variables for the process
+  elasticsearch.exec.options:
+    description: An array of additional options to pass when starting elasticsearch
+    default: []
+  elasticsearch.limits.fd:
+    description: Maximum file descriptors
+    default: 65536
+  elasticsearch.discovery.minimum_master_nodes:
+    description: The minimum number of master eligible nodes a node should "see" in order to operate within the cluster. Recommended to set it to a higher value than 1 when running more than 2 nodes in the cluster.
+    default: 1
+  elasticsearch.config_options:
+    description: "Additional options to append to elasticsearch's config.yml (YAML format)."
+    default: ~
+  elasticsearch.logging_options:
+    description: "Additional options to append to elasticsearch's logging.yml (YAML format)."
+    default: ~
+  elasticsearch.plugins:
+    description: "Plugins to run elasticsearch with (array[] = { plugin-name: install-source }; e.g. [ { kopf: 'lmenezes/elasticsearch-kopf' } ])"
+    default: []
+  elasticsearch.http_host:
+    description: "The host address to bind the elasticsearch HTTP service to and to publish for HTTP clients to connect to"
+    default: 0.0.0.0

jobs/elasticsearch/templates/bin/drain.erb

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#
+# This script keeps the elasticsearch cluster stable when BOSH needs to manage
+# nodes during deploys. This is simplistic and only accounts for serial
+# changes to the elasticsearch nodes, so be sure elasticsearch jobs are
+# configured to deploy serially.
+#
+# When you get into fancier cluster scenarios, you'll want to disable this with
+# the `elasticsearch.drain: false` property and manage your own cluster
+# settings.
+#
+# This is invoked somewhere around here:
+# https://github.com/cloudfoundry/bosh/blob/7a7e42312a6a1dce50b578be579f17d10797e556/bosh_agent/lib/bosh_agent/message/drain.rb#L172
+#
+
+set -e
+set -u
+
+# disable allocations before bringing down data nodes
+<% if p('elasticsearch.node.allow_data') %>
+curl -s \
+    -X PUT \
+    -d '{"transient":{"cluster.routing.allocation.enable":"none"}}' \
+    'localhost:9200/_cluster/settings' \
+    > /dev/null
+<% end %>
+
+echo 0
+exit 0

jobs/elasticsearch/templates/bin/elasticsearch_ctl

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/elasticsearch/helpers/ctl_setup.sh 'elasticsearch'
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+
+<% p("elasticsearch.exec.environment", {}).each do | k, v | %>
+export <%= k %>=<%= v %>
+<% end %>
+
+<% if not p('elasticsearch.exec.environment', {}).has_key? 'ES_HEAP_SIZE' then %>
+export ES_HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * 46 ) / 100 ))K
+<% end %>
+
+
+PLUGIN_SCRIPTS_DIR=/var/vcap/data/elasticsearch/plugin-scripts
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+    pid_guard $PIDFILE-init $JOB_NAME-init
+
+    # lock while we update plugins and run drain
+    echo "$$" > "$PIDFILE-init"
+
+    mkdir -p /var/vcap/packages/elasticsearch/plugins
+
+    # install plugins
+    rm -rf /var/vcap/packages/elasticsearch/plugins/*
+
+    # install kopf as it is always packaged with the elasticsearch job
+    # /var/vcap/packages/elasticsearch/bin/plugin install file:///var/vcap/packages/elasticsearch-plugins-kopf/elasticsearch-kopf.zip
+
+    # install user defined plugins
+    <% p("elasticsearch.plugins").each do |plugin| _, path = plugin.first %>
+      <% if path.start_with? '/var/vcap' %>
+        /var/vcap/packages/elasticsearch/bin/plugin install "file://<%= path %>"
+      <% else %>
+        /var/vcap/packages/elasticsearch/bin/plugin install "<%= path %>"
+      <% end %>
+    <% end %>
+
+
+    ulimit -n <%= p("elasticsearch.limits.fd") %>
+    ulimit -l unlimited  # required to enable elasticsearch's mlockall setting
+
+    mkdir -p $PLUGIN_SCRIPTS_DIR
+
+    # v21 switched to running as vcap; remove after a couple versions
+    chown -R vcap:vcap $STORE_DIR $LOG_DIR $RUN_DIR $PLUGIN_SCRIPTS_DIR
+
+    # ES2.0 deprcated -Des.config and insists on having the config file under ES_HOME/config
+    cp --remove-destination $JOB_DIR/config/elasticsearch.yml /var/vcap/packages/elasticsearch/config
+
+    chpst -u vcap:vcap /var/vcap/packages/elasticsearch/bin/elasticsearch \
+         -p ${PIDFILE} \
+         --XX:HeapDumpPath=${TMPDIR}/heap-dump/ \
+         <%= p("elasticsearch.exec.options", []).join(' ') %> \
+         >>$LOG_DIR/$JOB_NAME.stdout.log \
+         2>>$LOG_DIR/$JOB_NAME.stderr.log
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: elasticsearch_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0

jobs/elasticsearch/templates/bin/monit_debugger

@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1

jobs/elasticsearch/templates/bin/post-deploy.erb

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+curl -s \
+    -X PUT \
+    -d '{"transient":{"cluster.routing.allocation.enable":"all"}}' \
+    '<%= p('elasticsearch.master_hosts').first %>:9200/_cluster/settings' \
+    > /dev/null

jobs/elasticsearch/templates/config/config.yml.erb

@@ -0,0 +1,27 @@
+bootstrap.mlockall: true
+
+path.conf: "/var/vcap/jobs/elasticsearch/config"
+path.logs: "/var/vcap/sys/log/elasticsearch"
+path.data: "/var/vcap/store/elasticsearch"
+path.scripts: "/var/vcap/data/elasticsearch/plugin-scripts"
+
+cluster.name: "<%= p("elasticsearch.cluster_name") %>"
+
+node.max_local_storage_nodes: 1
+node.name: "<%= name %>/<%= index %>"
+node.master: <%= p("elasticsearch.node.allow_master") %>
+node.data: <%= p("elasticsearch.node.allow_data") %>
+node.job_name: "<%= name %>"
+node.job_index: "<%= index %>"
+<% p("elasticsearch.node.tags", {}).each do | k, v | %>
+node.<%= k %>: "<%= v %>"
+<% end %>
+
+network.host: "0.0.0.0"
+http.host: <%= p("elasticsearch.http_host") %>
+
+discovery.zen.minimum_master_nodes: <%= p("elasticsearch.discovery.minimum_master_nodes") %>
+discovery.zen.ping.multicast.enabled: false
+discovery.zen.ping.unicast.hosts: "<%= p("elasticsearch.master_hosts").join(',') %>"
+
+<% if_p('elasticsearch.config_options') do | v | %><%= v %><% end %>

jobs/elasticsearch/templates/config/logging.yml.erb

@@ -0,0 +1,10 @@
+rootLogger: "<%= p("elasticsearch.log_level") %>, console"
+
+appender:
+  console:
+    type: "console"
+    layout:
+      type: "consolePattern"
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+<% if_p('elasticsearch.logging_options') do | v | %><%= v %><% end %>

jobs/elasticsearch/templates/data/properties.sh.erb

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# job template binding variables
+
+# job name & index of this VM within cluster
+# e.g. JOB_NAME=redis, JOB_INDEX=0
+export NAME='<%= name %>'
+export JOB_INDEX=<%= index %>
+# full job name, like redis/0 or webapp/3
+export JOB_FULL="$NAME/$JOB_INDEX"