Merge branch 'master' of https://github.com/Cobrijani/Elusive

.vscode/settings.json

@@ -1,4 +1,5 @@
 // Place your settings in this file to overwrite default and user settings.
 {
-    "files.eol": "\n"
+    "files.eol": "\n",
+    "python.linting.pylintEnabled": false
 }

README.md

@@ -28,4 +28,6 @@ Student project that consists of creating SIEM center using ELK stack.
 
 - [Docker tips and cheatsheet](https://blog.jez.io/2015/07/12/docker-tips-and-cheatsheet/)
 
+- [Github repo for Docker cheatsheets](https://github.com/wsargent/docker-cheat-sheet)
+
 

cryptography/README.md

@@ -0,0 +1,6 @@
+This folder contains all configurations for security aspects
+
+
+- instance.yml configuration for generating certificates for elasticsearch cluster
+
+- [Guide for generating certificates for elasticsearch cluster](https://www.elastic.co/guide/en/x-pack/current/ssl-tls.html)

deployment/elk/cryptography/instances.yml → cryptography/instances.yml

@@ -4,4 +4,7 @@ instances:
       - "localhost"
       - "es"
     ip:
-      - "127.0.0.1"
+      - "127.0.0.1"
+      - "0.0.0.0"
+      - "172.20.0.4"
+      - "172.20.0.5"

deployment/docker-compose.yml

@@ -27,28 +27,30 @@ services:
         - ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
         - ./logstash/pipeline/:/usr/share/logstash/pipeline/
         - ./logstash/patterns/:/usr/share/logstash/patterns/
+    apachebeat:
+      build: "filebeat"
+      networks:
+        - elk
+      volumes:
+        - ./../test_logs/apache/:/var/log/apache
+      environment:
+        - BEAT_NAME=apache-beat
+    firebeat:
+      build: "filebeat"
+      networks:
+       - elk
+      volumes:
+       - ./../test_logs/firewall/:/var/log/firewall
+      environment:
+       - BEAT_NAME=fire-beat
+
 
-    # elk:
-    #   build: "elk"
-    #   #image: 'cobrijani/elusive:latest_elk'
-    #   ports:
-    #     -  "5601:5601"
-    #     -  "9200:9200" 
-    #     -  "5044:5044"
-    #   networks: 
-    #     - elk
     # redis:
     #   image: "redis:alpine"
     #   ports:
     #     - "6379:6379"
     #   networks: 
     #     - elk
-    # filebeat:
-    #   build: "filebeat"
-    #   networks: 
-    #     - elk
-    #   depends_on: 
-    #     - elk
 
 networks: 
   elk:

deployment/elasticsearch/elasticsearch.yml

@@ -34,7 +34,7 @@ node.name: elusive-1
 #
 # Path to log files:
 #
-#path.logs: /path/to/logs
+#path.logs: /var/log/elasticsearch
 #
 # ----------------------------------- Memory -----------------------------------
 #

deployment/elk[DEPRECATED]/README.md

@@ -0,0 +1,5 @@
+# Elk docker image [DEPRECATED]
+
+This image is not used anymore it is split into 3 seprate docker images
+
+This folder contains tweaked version of docker image located [here](https://hub.docker.com/r/sebp/elk/)

deployment/logstash/pipeline/02-firewall.conf → deployment/elk[DEPRECATED]/pipeline/02-firewall-input.conf

@@ -1,9 +1,10 @@
 #This configuration file represents parsing stategy for firewall logs
 filter {
-    if("firewall-logs-agent" in [tags]){
+    if("fire-beat" in [beat][name]){
+        # change index for firebeat
+
         grok { #match message to regular expression
-            patterns_dir => ["./patterns"]
-            match => { "message" => "%{MY_TIMESTAMP:logdate} %{ACTION:action} %{PROTOCOL:protocol} %{IP:src-ip} %{IP:dst-ip} %{NUMBER:src-port} %{NUMBER:dst-port} %{NUMBER:size} %{FLAG:tcpflags} %{FLAG:tcpsyn} %{FLAG:tcpack} %{FLAG:tcpwin} %{FLAG:icmptype} %{FLAG:icmpcode} %{FLAG:info} %{PATH:path}" }
+            match => { "message" => "%{MY_TIMESTAMP:logdate} %{ACTION:action} %{PROTOCOL:protocol} %{IP:src-ip} %{IP:dst-ip} %{NUMBER:src-port} %{NUMBER:dst-port} %{NUMBER:size} %{TCP_FLAG:tcpflags} %{FLAG:tcpsyn} %{FLAG:tcpack} %{FLAG:tcpwin} %{FLAG:icmptype} %{FLAG:icmpcode} %{FLAG:info} %{PATH:path}" }
         }
         date { #set time from log file to @timestamp field in elasticsearch
             match => [ "logdate", "yyyy-MM-dd HH:mm:ss" ]

deployment/elk[DEPRECATED]/pipeline/03-apache-input.conf

@@ -0,0 +1,15 @@
+#This configuration file represents parsing stategy for apache web server logs
+filter {
+
+    if ("apache-beat" in [beat][name]){
+        # change index for apachebeat
+
+
+        grok { #match message to regular expression
+        match => { "message" => "%{COMBINEDAPACHELOG}"}
+        }
+        geoip {
+            source => "clientip"
+        }
+    }
+}

deployment/filebeat/Dockerfile

@@ -1,13 +1,16 @@
 FROM phusion/baseimage
-MAINTAINER Stefan Bratic
+LABEL maintainer "Stefan Bratic <[email protected]>"
+
 ENV REFRESHED_AT 2017-05-23
 
 
 ###############################################################################
 #                                INSTALLATION
 ###############################################################################
 
-ENV FILEBEAT_VERSION 5.4.0
+ENV FILEBEAT_VERSION 5.4.1
+ENV BEAT_NAME GENERIC_BEAT
+
 
 RUN apt-get update -qq \
  && apt-get install -qqy curl \
@@ -17,7 +20,6 @@ RUN curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${
  && dpkg -i filebeat-${FILEBEAT_VERSION}-amd64.deb \
  && rm filebeat-${FILEBEAT_VERSION}-amd64.deb
 
-
 ###############################################################################
 #                                CONFIGURATION
 ###############################################################################
@@ -36,7 +38,6 @@ ADD logstash-beats.crt /etc/pki/tls/certs/logstash-beats.crt
 
 ADD wait.sh /wait.sh
 RUN chmod +x /wait.sh
-
 ADD start.sh /usr/local/bin/start.sh
 RUN chmod +x /usr/local/bin/start.sh
 CMD [ "/usr/local/bin/start.sh" ]

deployment/filebeat/filebeat.yml

@@ -112,7 +112,9 @@ filebeat.prospectors:
 
   # Paths that should be crawled and fetched. Glob based paths.
   paths:
-    - /var/log/*.log
+    - /var/log/firewall/*.log
+    - /var/log/apache/*.log
+
     #- c:\programdata\elasticsearch\logs\*
 
   # Exclude lines. A list of regular expressions to match. It drops the lines that are
@@ -154,11 +156,11 @@ filebeat.prospectors:
 
 # The name of the shipper that publishes the network data. It can be used to group
 # all the transactions sent by a single shipper in the web interface.
-name: "linux-logs-agent"
+name: ${BEAT_NAME:test}
 
 # The tags of the shipper are included in their own field with each
 # transaction published.
-tags: ["linux-logs-agent"]
+#tags: []
 
 # Optional fields that you can specify to add additional information to the
 # output.
@@ -183,7 +185,7 @@ tags: ["linux-logs-agent"]
 #----------------------------- Logstash output --------------------------------
 output.logstash:
   # The Logstash hosts
-  hosts: ["elk:5044"]
+  hosts: ["logstash:5044"]
 
   # Optional SSL. By default is off.
   # List of root certificates for HTTPS server verifications

deployment/filebeat/start.sh

@@ -1,14 +1,32 @@
 #!/bin/sh
 
-./wait.sh -t 30 -h elk -p 9200 #wait for elasticsearch
-./wait.sh -t 15 -h elk -p 5044 # wait for logstash
+./wait.sh -t 300 -h es -p 9200 #wait for elasticsearch
+
+
+#Add role that is configured for logstash indexer named logstash_writer
+curl -k \
+   --user elastic:changeme \
+   -X POST \
+   -H "Content-Type: application/json"\
+   -d "{ \"cluster\": [\"manage_index_templates\", \"monitor\"], \"indices\": [{ \"names\": [\"logstash-*\", \"filebeat-*\", \"winlogbeat-*\"], \"privileges\": [\"write\", \"delete\", \"create_index\"]}]}" \
+   https://es:9200/_xpack/security/role/logstash_writer
+
+# Add user with role logstash writer
+curl -k \
+   --user elastic:changeme \
+   -X POST \
+   -H "Content-Type: application/json"\
+   -d "{ \"password\" : \"changeme\",\"roles\" : [ \"logstash_writer\"],\"full_name\" : \"Internal Logstash User\"}"\
+   https://es:9200/_xpack/security/user/logstash_internal
+
 curl -k \
  --user elastic:changeme \
  -X PUT \
- 'https://elk:9200/_template/filebeat?pretty' \
+ 'https://es:9200/_template/filebeat?pretty' \
  -d@/etc/filebeat/filebeat.template.json
-service filebeat start
+./wait.sh -t 15 -h logstash -p 5044 # wait for logstash
+/etc/init.d/filebeat start -e
+
 mkdir /var/log/filebeat
 touch /var/log/filebeat/filebeat.log
-#tail -f /var/log/filebeat/filebeat.log
 tail -f /var/log/filebeat/filebeat.log & wait

deployment/filebeat_firewall/README.md → deployment/filebeat_firewall[DEPRECATED]/README.md

@@ -1,4 +1,6 @@
-# Firewall filebeat scripts and configuration folder
+# Firewall filebeat scripts and configuration folder [DEPRECATED]
+
+Folder filebeat contains configuration that is needed to process firewall logs
 
 - "__install.ps1__" is script that downloads latest filebeat and installs it on local windows machine with preffered install location from [filebeat reference documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation.html)
 
@@ -33,3 +35,4 @@
 19. After this Kibana will show all successfull and unsuccessfull firewall logs
 20. We specify rule to block outbound on ports 5678-5680
 21. If you want to fire this event go to: https://www.google.rs:5678/ 
+