Update

fluentd/docker-compose-test-aws/docker-compose.yaml

@@ -1,21 +1,40 @@
 version: '2'
+
+### Use for all application except fluentd
+x-logging-application: &logging-application
+  logging:
+    driver: "fluentd"
+    options:
+      fluentd-address: localhost:24224
+      fluentd-async-connect: "true"
+      tag: application
+
+### Only use for fluentd
+x-logging-defaults: &logging-defaults
+  logging:
+    driver: ${LOGGING_DRIVER}
+    options:
+      max-size: ${LOGGING_OPTIONS_MAX_SIZE}
+      max-file: ${LOGGING_OPTIONS_MAX_FILE}
+
 services:
   web:
+    <<: *logging-application
     image: httpd
     ports:
-      - "80:80"
-    links:
-      - fluentd
-    logging:
-      driver: "fluentd"
-      options:
-        fluentd-address: localhost:24224
-        tag: httpd.access
+      - "8099:80"
 
   fluentd:
+    <<: *logging-defaults
     build: ./fluentd
     volumes:
       - ./fluentd/conf:/fluentd/etc
     ports:
       - "24224:24224"
       - "24224:24224/udp"
+    environment:
+      - ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST}
+      - ELASTICSEARCH_SCHEME=${ELASTICSEARCH_SCHEME}
+      - ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME}
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD}

fluentd/docker-compose-test-aws/fluentd/.env

@@ -0,0 +1,8 @@
+LOGGING_OPTIONS_MAX_SIZE=500m
+LOGGING_OPTIONS_MAX_FILE=10
+
+ELASTICSEARCH_HOST=search-xxxxx-yyyyyy.ap-southeast-2.es.amazonaws.com
+ELASTICSEARCH_SCHEME=https
+ELASTICSEARCH_PORT=443
+ELASTICSEARCH_USERNAME=this_is_username
+ELASTICSEARCH_PASSWORD=this_is_password

fluentd/docker-compose-test-aws/fluentd/conf/fluent.conf

@@ -4,23 +4,23 @@
   bind 0.0.0.0
 </source>
 
-<match *.**>
+<match application.**>
   @type copy
 
   <store>
     @type elasticsearch
-    scheme https
-    host search-personalisationhubes-xxxxxxxxxxxxx.ap-southeast-2.es.amazonaws.com
-    porelasticsearcht 9200
+    scheme "#{ENV['ELASTICSEARCH_SCHEME']}"
+    host "#{ENV['ELASTICSEARCH_HOST']}"
+    port "#{ENV['ELASTICSEARCH_PORT']}"
     logstash_format true
-    logstash_prefix fluentd
-    logstash_dateformat %Y%m%d
+    logstash_prefix application-data
+    logstash_dateformat %Y.%m.%d
     include_tag_key true
-    type_name access_log
+    type_name application-data
     tag_key @log_name
     flush_interval 1s
-    user xxxxxxxxx
-    password xxxxxxxxxxx
+    user "#{ENV['ELASTICSEARCH_USERNAME']}"
+    password "#{ENV['ELASTICSEARCH_PASSWORD']}"
   </store>
 
   <store>