updated BOSH release to use BOSH2 linking. no static IP addresses are…

… required for any instances. linking shares this information in a deployment.

Readme has been updated to be more useful.

README.md

@@ -1,25 +1,49 @@
 # graylog-boshrelease
 BOSH release for graylog (https://www.graylog.org/) - tool for centrally collecting logging events for your infrastructure and applications.
 
-NOTE: this is initial development release and is not recommended for production environments.  it has currently only been tested with BOSH-lite.
+## Usage
 
-### Creating a development bosh release
+### Developing and deploying locally
+Setup a local BOSH2 environment with VBOX.  Follow https://github.com/cloudfoundry/bosh-deployment
+Remember to apply the provided BOSH cloud-config https://github.com/cloudfoundry/bosh-deployment/blob/master/warden/cloud-config.yml
+
+
+Create a local BOSH dev release for graylog
 ```
-bosh create release --force
-bosh upload release
+bosh2 create-release --force --name graylog
+bosh2 -e vbox upload-release
 ```
 
-### Deploying to *bosh-lite*.
-A sample cloud-config and deployment manifest is provided in the `templates` directory.  
-There is a bunch of settings available to be tuned in the deployment manifest. In each of the jobs there is a spec file that contains a properties key. Any of these can be add to the `properties:` at the bottom of your deployment manifest.
+The base manifest `manifests/graylog.yml` should "Just Work".  It is setup with with BOSH linking so no static-ip addresses or specific settings should be required.  It uses `default` for vm_type, stemcll, persistent_disk_type, and networks as setup in the cloud-config above.
+```
+bosh2 -e vbox deploy -n -d graylog manifests/graylog.yml
+```
 
+Check the running VMs
 ```
-cp templates/bosh-lite-deployment.yml bosh-lite-deployment.yml
-bosh update cloud-config templates/bosh-lite-cloud-config.yml
-bosh deployment bosh-lite-deployment.yml
-bosh deploy
+$ bosh2 vms
+Using environment '192.168.50.6' as client 'admin'
+
+Task 56. Done
+
+Deployment 'graylog'
+
+Instance                                                   Process State  AZ  IPs         VM CID                                VM Type
+elasticsearch-data/2ad178ba-612c-4282-b22c-81915deb3fdd    running        z1  10.244.0.5  bd23c68f-737f-4e34-551a-100d0ebec3bd  default
+elasticsearch-data/763aadcb-b866-4b73-a13f-05fe7db2b79a    running        z1  10.244.0.4  11f16ae0-e8dd-48a3-7ad3-09dfa0ced59b  default
+elasticsearch-master/0a659b3d-3156-4919-b255-c618177c34c7  running        z1  10.244.0.3  961ca2a8-8c95-4b82-4519-8d94c39de345  default
+graylog/f812bbca-3057-4236-b2c8-6ba007fa7f9d               running        z1  10.244.0.6  92b31138-b4e6-4395-6e25-fb57cc4fbcf7  default
+mongodb/e5a742e9-4f59-45bb-b957-bab36cae4aa5               running        z1  10.244.0.2  7d4ae78b-8c67-4c86-7c2d-0c41bd7709fd  default
+
+5 vms
+
+Succeeded
 ```
 
+Point your browser the IP address for the graylog instance on port 9000 (above it is 10.244.0.6).
+You should be prompted with the graylog login page.  The default credentials provided in the `manifests/graylog.yml` are admin/admin.
+
+
 
 ### attribution
 This BOSH release for Graylog was heavily inspired by an existing BOSH release for the ELK stack - https://github.com/cloudfoundry-community/logsearch-boshrelease.  

jobs/elasticsearch/monit

@@ -1,7 +1,7 @@
 check process elasticsearch
   with pidfile /var/vcap/sys/run/elasticsearch/elasticsearch.pid
-  start program "/var/vcap/jobs/elasticsearch/bin/monit_debugger elasticsearch_ctl '/var/vcap/jobs/elasticsearch/bin/elasticsearch_ctl start'" with timeout 120 seconds
-  stop program "/var/vcap/jobs/elasticsearch/bin/monit_debugger elasticsearch_ctl '/var/vcap/jobs/elasticsearch/bin/elasticsearch_ctl stop'"
+  start program "/var/vcap/jobs/elasticsearch/bin/monit_debugger ctl '/var/vcap/jobs/elasticsearch/bin/ctl start'" with timeout 120 seconds
+  stop program "/var/vcap/jobs/elasticsearch/bin/monit_debugger ctl '/var/vcap/jobs/elasticsearch/bin/ctl stop'"
   group vcap
 
 check device elasticsearch-ephemeral_disk with path /var/vcap/data

jobs/elasticsearch/spec

@@ -1,23 +1,30 @@
 ---
 name: elasticsearch
+
 packages:
 - elasticsearch
 - java
 
 templates:
   bin/drain.erb: bin/drain
   bin/post-deploy.erb: bin/post-deploy
-  bin/elasticsearch_ctl: bin/elasticsearch_ctl
+  bin/ctl.erb: bin/ctl
   bin/monit_debugger: bin/monit_debugger
   config/config.yml.erb: config/elasticsearch.yml
   config/logging.yml.erb: config/logging.yml
   data/properties.sh.erb: data/properties.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
 
+provides:
+- name: elasticsearch
+  type: elasticsearch
+
+consumes:
+- name: elasticsearch
+  type: elasticsearch
+
 properties:
-  elasticsearch.master_hosts:
-    description: The list of elasticsearch master node IPs
   elasticsearch.cluster_name:
     description: The name of the elastic search cluster
   elasticsearch.log_level:

jobs/elasticsearch/templates/bin/post-deploy.erb

@@ -1,9 +1,13 @@
 #!/bin/bash
+<%
+  elasticsearch_master = link("elasticsearch")
+  elasticsearch_master_host = elasticsearch_master.instances.first.address
+-%>
 
 set -e
 
 curl -s \
     -X PUT \
     -d '{"transient":{"cluster.routing.allocation.enable":"all"}}' \
-    '<%= p('elasticsearch.master_hosts').first %>:9200/_cluster/settings' \
+    '<%= elasticsearch_master_host %>:9200/_cluster/settings' \
     > /dev/null

jobs/elasticsearch/templates/config/config.yml.erb

@@ -1,3 +1,7 @@
+<%
+  elasticsearch_master = link("elasticsearch")
+  elasticsearch_master_host = elasticsearch_master.instances.first.address
+-%>
 bootstrap.mlockall: true
 
 path.conf: "/var/vcap/jobs/elasticsearch/config"
@@ -22,6 +26,6 @@ http.host: <%= p("elasticsearch.http_host") %>
 
 discovery.zen.minimum_master_nodes: <%= p("elasticsearch.discovery.minimum_master_nodes") %>
 discovery.zen.ping.multicast.enabled: false
-discovery.zen.ping.unicast.hosts: "<%= p("elasticsearch.master_hosts").join(',') %>"
+discovery.zen.ping.unicast.hosts: "<%= elasticsearch_master_host %>"
 
 <% if_p('elasticsearch.config_options') do | v | %><%= v %><% end %>

jobs/graylog/monit

@@ -1,5 +1,5 @@
 check process graylog
   with pidfile /var/vcap/sys/run/graylog/graylog.pid
-  start program "/var/vcap/jobs/graylog/bin/monit_debugger graylog_ctl '/var/vcap/jobs/graylog/bin/graylog_ctl start'" with timeout 120 seconds
-  stop program "/var/vcap/jobs/graylog/bin/monit_debugger graylog_ctl '/var/vcap/jobs/graylog/bin/graylog_ctl stop'"
+  start program "/var/vcap/jobs/graylog/bin/monit_debugger ctl '/var/vcap/jobs/graylog/bin/ctl start'" with timeout 120 seconds
+  stop program "/var/vcap/jobs/graylog/bin/monit_debugger ctl '/var/vcap/jobs/graylog/bin/ctl stop'"
   group vcap

jobs/graylog/spec

@@ -1,16 +1,24 @@
 ---
 name: graylog
+
+packages:
+- graylog
+- java
+
 templates:
-  bin/graylog_ctl.erb: bin/graylog_ctl
+  bin/ctl.erb: bin/ctl
   bin/monit_debugger: bin/monit_debugger
   config/graylog.conf.erb: config/graylog.conf
   data/properties.sh.erb: data/properties.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
 
-packages:
-- graylog
-- java
+consumes:
+- name: elasticsearch
+  type: elasticsearch
+- name: mongodb
+  type: mongodb
+
 
 properties:
   graylog.is_master:
@@ -24,10 +32,8 @@ properties:
     default: replacethiswithyourownsecret!
   graylog.root_username:
     description: The default root user
-    default: admin
   graylog.root_password_sha2:
     description: hash password for the root user
-    default: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
   graylog.root_email:
     description: The email address of the root user
   graylog.plugin_dir:
@@ -38,12 +44,6 @@ properties:
   graylog.message_journal_dir:
     description: directory which will be used to store the message journal
     default: /var/vcap/store/graylog
-  graylog.elasticsearch_discovery_zen_ping_unicast_hosts:
-    description: master elasticsearch host zen ping
-    default: elasticsearch_master:9300
-  graylog.mongodb_uri:
-    description: MongoDB connection string
-    default: mongodb://mongo/graylog
   graylog.content_packs_dir:
     description: directory which contains content packs which should be loaded on the first start of Graylog
     default: /var/vcap/packages/graylog/data/contentpacks

jobs/graylog/templates/config/graylog.conf.erb

@@ -12,7 +12,6 @@ password_secret = <%= p("graylog.password_secret") %>
 
 # The default root user is named 'admin'
 root_username = <%= p("graylog.root_username") %>
-admin
 
 # You MUST specify a hash password for the root user (which you only need to initially set up the
 # system and in case you lose connectivity to your authentication backend)
@@ -210,7 +209,12 @@ elasticsearch_transport_tcp_port = 9350
 elasticsearch_http_enabled = false
 
 elasticsearch_discovery_zen_ping_multicast_enabled = false
-elasticsearch_discovery_zen_ping_unicast_hosts = <%= p("graylog.elasticsearch_discovery_zen_ping_unicast_hosts") %>
+<%
+  elasticsearch_master = link("elasticsearch")
+  host = elasticsearch_master.instances.first.address
+  elasticsearch_master_hosts = "#{host}:9300"
+-%>
+elasticsearch_discovery_zen_ping_unicast_hosts = <%= elasticsearch_master_hosts %>
 
 # Change the following setting if you are running into problems with timeouts during Elasticsearch cluster discovery.
 # The setting is specified in milliseconds, the default is 5000ms (5 seconds).
@@ -351,7 +355,13 @@ lb_recognition_period_seconds = 3
 
 # MongoDB connection string
 # See http://docs.mongodb.org/manual/reference/connection-string/ for details
-mongodb_uri = <%= p("graylog.mongodb_uri") %>
+<%
+  mongodb = link("mongodb")
+  host = mongodb.instances.first.address
+  port = mongodb.p("mongodb.net.port")
+  mongodb_uri = "mongodb://#{host}:#{port}/graylog"
+-%>
+mongodb_uri = <%= mongodb_uri %>
 
 # Authenticate against the MongoDB server
 #mongodb_uri = mongodb://grayloguser:secret@localhost:27017/graylog

jobs/mongodb/monit

@@ -1,5 +1,5 @@
 check process mongodb
   with pidfile /var/vcap/sys/run/mongodb/mongodb.pid
-  start program "/var/vcap/jobs/mongodb/bin/monit_debugger mongodb_ctl '/var/vcap/jobs/mongodb/bin/mongodb_ctl start'" with timeout 120 seconds
-  stop program "/var/vcap/jobs/mongodb/bin/monit_debugger mongodb_ctl '/var/vcap/jobs/mongodb/bin/mongodb_ctl stop'"
+  start program "/var/vcap/jobs/mongodb/bin/monit_debugger ctl '/var/vcap/jobs/mongodb/bin/ctl start'" with timeout 120 seconds
+  stop program "/var/vcap/jobs/mongodb/bin/monit_debugger ctl '/var/vcap/jobs/mongodb/bin/ctl stop'"
   group vcap

jobs/mongodb/spec

@@ -1,16 +1,23 @@
+
 ---
 name: mongodb
 
+packages:
+- mongodb
+
 templates:
-  bin/mongodb_ctl.erb: bin/mongodb_ctl
+  bin/ctl.erb: bin/ctl
   bin/monit_debugger: bin/monit_debugger
   config/mongodb.conf.erb: config/mongodb.conf
   data/properties.sh.erb: data/properties.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
 
-packages:
-- mongodb
+provides:
+- name: mongodb
+  type: mongodb
+  properties:
+  - mongodb.net.port
 
 properties:
   mongodb.systemLog.path:
@@ -24,7 +31,6 @@ properties:
     default: /var/vcap/sys/run/mongodb/mongodb.pid
   mongodb.net.port:
     description:
-    default: 27017
   mongodb.net.bindIp:
     description:
     default: 0.0.0.0

manifests/graylog.yml

@@ -0,0 +1,104 @@
+---
+name: graylog
+
+
+instance_groups:
+- name: mongodb
+  azs: [z1]
+  instances: 1
+  vm_type: default
+  stemcell: default
+  persistent_disk_type: default
+  networks: [{name: default}]
+  jobs:
+  - name: mongodb
+    release: graylog
+    provides:
+      mongodb: {as: mongodb}
+    properties:
+      mongodb:
+        net:
+          port: 27017
+- name: elasticsearch-master
+  azs: [z1]
+  instances: 1
+  vm_type: default
+  stemcell: default
+  persistent_disk_type: default
+  networks: [{name: default}]
+  jobs:
+  - name: elasticsearch
+    release: graylog
+    provides:
+      elasticsearch: {as: elasticsearch-master}
+    consumes:
+      elasticsearch: {from: elasticsearch-master}
+    properties:
+      elasticsearch:
+        cluster_name: graylog
+        node:
+          allow_data: false
+          allow_master: true
+        # exec: null
+        exec:
+          environment:
+            ES_HEAP_SIZE: 500000K
+- name: elasticsearch-data
+  azs: [z1]
+  instances: 2
+  vm_type: default
+  stemcell: default
+  persistent_disk_type: default
+  networks: [{name: default}]
+  jobs:
+  - name: elasticsearch
+    release: graylog
+    provides:
+      elasticsearch: {as: elasticsearch-data}
+    consumes:
+      elasticsearch: {from: elasticsearch-master}
+    properties:
+      elasticsearch:
+        cluster_name: graylog
+        node:
+          allow_data: false
+          allow_master: true
+        # exec: null
+        exec:
+          environment:
+            ES_HEAP_SIZE: 500000K
+- name: graylog
+  azs: [z1]
+  instances: 1
+  vm_type: default
+  stemcell: default
+  persistent_disk_type: default
+  networks: [{name: default}]
+  jobs:
+  - name: graylog
+    release: graylog
+    consumes:
+      elasticsearch: {from: elasticsearch-master}
+      mongodb: {from: mongodb}
+    properties:
+      graylog:
+        root_email: [email protected]
+        root_username: admin
+        # default root_password = admin ; below is sha2 hash of it.
+        root_password_sha2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  canary_watch_time: 1000-600000
+  update_watch_time: 1000-600000
+
+stemcells:
+- alias: default
+  os: ubuntu-trusty
+  version: 3421.latest
+
+releases:
+- name: graylog
+  version: latest