inital commit

README.md

@@ -0,0 +1,30 @@
+dockerとfigを使って、norikra、elasticsearch, kibana4, Basic認証用のnginx, ログ収集用fluentdをまとめて起動します。
+最初に`fig up`を実行した時にDockerfileのビルドを行います。
+以降Dockerfileに変更を加えた場合`fig build`でdocker imageを更新します。
+
+## 必要な物
+- docker
+- fig
+
+## コンテナ構成
+- Elasticsearch  
+- Kibana4
+- fluentd  
+- Nginx  
+- Norikra
+
+## コマンド
+```bash
+# 起動
+fig up
+
+# バックグラウンド起動
+fig up -d
+
+# Dockerfileの再ビルド
+fig build
+```
+
+## Basic認証
+HTTPのアクセスにはBasic認証(nginx)が必要です。
+ID，PASSは`nginx_conf/htpasswd`に追加してください。

containers/elasticsearch/Dockerfile

@@ -0,0 +1,27 @@
+#
+# ElasticSearch Dockerfile
+#
+# https://github.com/dockerfile/elasticsearch
+
+FROM dockerfile/java:oracle-java7
+
+ENV ES_PKG_NAME elasticsearch-1.4.1
+
+RUN \
+  cd / && \
+  wget https://download.elasticsearch.org/elasticsearch/elasticsearch/$ES_PKG_NAME.tar.gz && \
+  tar xvzf $ES_PKG_NAME.tar.gz && \
+  rm -f $ES_PKG_NAME.tar.gz && \
+  mv /$ES_PKG_NAME /elasticsearch
+
+RUN apt-get update && apt-get install -y supervisor cron
+
+RUN /elasticsearch/bin/plugin --install mobz/elasticsearch-head
+
+ADD ./run.sh /run.sh
+RUN chmod +x /run.sh
+
+EXPOSE 9200
+EXPOSE 9300
+
+CMD ["/run.sh"]

containers/elasticsearch/run.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+/usr/bin/crontab /setting/crond/crontab
+/usr/bin/supervisord -n

containers/fluentd/Dockerfile

@@ -0,0 +1,12 @@
+FROM ruby:2.1.4
+
+RUN gem install fluentd fluent-plugin-elasticsearch fluent-plugin-forest fluent-plugin-norikra fluent-plugin-flatten-hash fluent-plugin-twitter
+RUN gem install fluent-plugin-docker-metrics
+RUN gem install specific_install
+RUN gem specific_install -l 'https://github.com/lburgazzoli/lb-fluent-plugin-jolokia.git'
+
+# timezone
+RUN cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
+
+CMD ["/usr/bin/supervisord -n"]
+

containers/kibana/Dockerfile

@@ -0,0 +1,12 @@
+# from https://github.com/arcus-io/docker-kibana.git
+FROM java:8
+
+RUN apt-get update -y
+RUN apt-get install -y nginx-full wget
+
+RUN mkdir /kibana \
+ && curl -sL "https://download.elasticsearch.org/kibana/kibana/kibana-4.0.0-beta3.tar.gz" | tar xz -C /kibana --strip=1
+
+EXPOSE 5601
+
+CMD ["/kibana/bin/kibana"]

containers/nginx/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx
+

containers/norikra/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:14.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get -y upgrade
+RUN apt-get -y install build-essential curl git openjdk-7-jdk
+
+RUN curl -L http://jruby.org.s3.amazonaws.com/downloads/1.7.16/jruby-bin-1.7.16.tar.gz | tar zxf -
+ENV PATH /jruby-1.7.16/bin:$PATH
+
+RUN gem install specific_install
+RUN gem specific_install -l 'https://github.com/ixixi/norikra.git'
+RUN gem install norikra-udf-percentile norikra-udf-uri_parser --no-ri --no-rdoc
+
+RUN curl http://labs.consol.de/maven/repository/org/jolokia/jolokia-jvm/1.1.5/jolokia-jvm-1.1.5-agent.jar > /usr/local/lib/jolokia-jvm-agent.jar
+
+# set up locale
+RUN locale-gen ja_JP.UTF-8
+RUN update-locale LANG=ja_JP.UTF-8
+ENV LC_ALL ja_JP.UTF-8
+ENV LANG ja_JP.UTF-8
+RUN cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
+
+EXPOSE 8778
+EXPOSE 26578
+EXPOSE 26571

fig/log_collector/fig.yml

@@ -0,0 +1,57 @@
+elasticsearch:
+  build: ../../containers/elasticsearch
+  ports:
+    - '9300:9300'
+  volumes:
+    - setting/elasticsearch/elasticsearch.yml:/data/elasticsearch.yml:ro
+    - data/elasticsearch/data:/data/data
+    - data/elasticsearch/work:/data/work
+    - log/elasticsearch:/data/log
+    - setting/elasticsearch-supervisord:/etc/supervisor/conf.d:ro
+    - log/elasticsearch-supervisord:/var/log/supervisor
+    - setting/elasticsearch/crontab:/setting/crond/crontab:ro
+
+kibana:
+  build: ../../containers/kibana
+  links:
+    - elasticsearch:elasticsearch
+  volumes:
+    - setting/kibana/kibana.yml:/kibana/config/kibana.yml:ro
+
+fluentd:
+  build: ../../containers/fluentd
+  ports:
+    - '24224:24224'
+  links:
+    - norikra:norikra
+    - elasticsearch:elasticsearch
+  volumes:
+    - setting/fluentd:/fluentd/conf
+    - log/fluentd:/fluentd/log
+    - setting/fluentd-supervisord:/etc/supervisor/conf.d:ro
+    - log/fluentd-supervisord:/var/log/supervisor
+    - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+  command: fluentd --use-v1-config -c /fluentd/conf/fluentd.conf
+
+nginx:
+  build: ../../containers/nginx
+  ports:
+    - '5601:5601'
+    - '9200:9200'
+    - '26578:26578'
+  volumes:
+    - setting/nginx/htpasswd:/data/htpasswd:ro
+    - setting/nginx/nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
+    - log/nginx:/var/log/nginx
+  links:
+    - elasticsearch
+    - kibana
+    - norikra
+
+norikra:
+  build: ../../containers/norikra
+  volumes:
+    - data/norikra:/norikra/data
+    - log/norikra:/norikra/log
+  command: norikra start --stats=/norikra/data/norikra.stats.json --dump-stat-interval=60 --gc-log=/norikra/log/gc.log -javaagent:/usr/local/lib/jolokia-jvm-agent.jar=port=8778,host=0.0.0.0

fig/log_collector/setting/elasticsearch-supervisord/crond.conf

@@ -0,0 +1,10 @@
+[program:cron]
+command = cron -f -L 15
+process_name = %(program_name)s
+startsecs = 5
+stopwaitsecs = 3600
+stopasgroup = false
+killasgroup = true
+
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-stderr.log

fig/log_collector/setting/elasticsearch-supervisord/elasticsearch.conf

@@ -0,0 +1,17 @@
+[program:elasticsearch]
+command=/elasticsearch/bin/elasticsearch -Des.config=/data/elasticsearch.yml
+autorestart=true
+
+process_name = %(program_name)s
+user=root
+directory=/
+
+autostart=true
+autorestart=true
+startsecs = 5
+stopwaitsecs = 3600
+stopasgroup = false
+killasgroup = true
+
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-stderr.log

fig/log_collector/setting/elasticsearch-supervisord/supervisord.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

fig/log_collector/setting/elasticsearch/crontab

@@ -0,0 +1,6 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+* * * * * PREFIX="docker-"; DAY=9; INDEX=`date +$PREFIX\%Y.\%m.\%d --date "${DAY} days ago"`; curl -XDELETE -w'\n' "http://localhost:9200/${INDEX}"
+* * * * * PREFIX="norikra-"; DAY=9; INDEX=`date +$PREFIX\%Y.\%m.\%d --date "${DAY} days ago"`; curl -XDELETE -w'\n' "http://localhost:9200/${INDEX}"
+* * * * * PREFIX="twitter-"; DAY=9; INDEX=`date +$PREFIX\%Y.\%m.\%d --date "${DAY} days ago"`; curl -XDELETE -w'\n' "http://localhost:9200/${INDEX}"

fig/log_collector/setting/elasticsearch/elasticsearch.yml

@@ -0,0 +1,4 @@
+path:
+  data: /data/data
+  logs: /data/log
+  work: /data/work

fig/log_collector/setting/fluentd/fluentd.conf

@@ -0,0 +1,141 @@
+<source>
+  type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<source>
+  type monitor_agent
+  port 24220
+</source>
+
+<source>
+  type debug_agent
+  port 24230
+</source>
+
+<source>
+  type docker_metrics
+  stats_interval 10s
+</source>
+
+#<source>
+#  consumer_key        ${YOUR_API_key}
+#  consumer_secret     ${YOUR_API_Secret}
+#  oauth_token         ${YOUR_access_token}
+#  oauth_token_secret  ${YOUR_access_token_secret}
+#  tag                 nested.twitter
+#  timeline            sampling
+#  output_format       nest
+#</source>
+
+<source>
+
+  type jolokia
+
+  tag nested.norikra.memory
+
+  jolokia_url "#{'http://norikra:'+ENV['NORIKRA_PORT_8778_TCP_PORT']}/jolokia/"
+
+  jmx_bean java.lang:type=Memory
+  jmx_attribute NonHeapMemoryUsage
+
+  run_interval 5s
+
+</source>
+
+<source>
+
+  type jolokia
+
+  tag nested.norikra.memory
+
+  jolokia_url "#{'http://norikra:'+ENV['NORIKRA_PORT_8778_TCP_PORT']}/jolokia/"
+
+  jmx_bean java.lang:type=Memory
+  jmx_attribute HeapMemoryUsage
+
+  run_interval 5s
+
+</source>
+
+<source>
+
+  type jolokia
+
+  tag nested.norikra.threads
+
+  jolokia_url "#{'http://norikra:'+ENV['NORIKRA_PORT_8778_TCP_PORT']}/jolokia/"
+
+  jmx_bean java.lang:type=Threading
+  jmx_attribute ThreadCount
+
+  run_interval 5s
+
+</source>
+
+<match nested.**>
+  type flatten_hash
+  remove_tag_prefix nested.
+  separator _
+</match>
+
+<match **>
+
+  type copy
+
+  <store>
+    type stdout
+  </store>
+
+  <store>
+    type norikra
+    norikra "#{'norikra:'+ENV['NORIKRA_PORT_26571_TCP_PORT']}"
+    target_map_tag true
+    buffer_chunk_limit 32m
+    buffer_queue_limit 96
+    flush_interval 1s
+    retry_limit 100
+    retry_wait 6s
+    <default>
+      auto_field true
+    </default>
+  </store>
+
+  <store>
+
+    type forest
+    subtype elasticsearch
+
+    <template>
+
+      host elasticsearch
+      port 9200
+      type_name  ${tag_parts[1]}
+
+      buffer_type memory
+      flush_interval 3
+      retry_limit 60
+      retry_wait 1.0
+      num_threads 1
+
+      logstash_prefix ${tag_parts[0]}
+      logstash_format true
+
+    </template>
+  </store>
+</match>
+
+
+<source>
+  type norikra
+  norikra "#{ENV['NORIKRA_PORT_26571_TCP_ADDR']+':'+ENV['NORIKRA_PORT_26571_TCP_PORT']}"
+  connect_timeout 600
+  <fetch>
+    method sweep
+    target out
+    interval 1s
+    tag field tag_name
+  </fetch>
+</source>
+