handling rotation of jetty logs as well

geneontology · Dec 7, 2021 · 6b4219c · 6b4219c
1 parent 426ed81
commit 6b4219c
Show file tree

Hide file tree

Showing 10 changed files with 91 additions and 9 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -45,7 +45,10 @@ RUN apt-get -qq update && apt-get -qq -y install \
 	libjson-xs-perl liburi-perl libwww-mechanize-perl \
 	liburi-encode-perl libxml-libxml-perl libxml-xpath-perl \
 	dh-make-perl apache2 openjdk-8-jdk openjdk-8-jre jetty9 \
-        monit psmisc
+        monit psmisc logrotate
+
+RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d \
+    && rm -rf /var/log && mkdir -p /var/log
 
 ## AmiGO weirdness.
 ADD docker/libsql-tokenizer-perl_0.24-2_all.deb /tmp/libsql-tokenizer-perl_0.24-2_all.deb
@@ -133,6 +136,8 @@ ENV GOLR_SOLR_MEMORY=4G
 COPY ./docker/entrypoint.sh /
 COPY ./docker/run-golr.sh /
 RUN chmod +x /entrypoint.sh /run-golr.sh && md5sum ./conf/amigo.yaml > amigo-hash
-COPY ./docker/java /etc/monit/conf-enabled/
+COPY ./docker/java-golr-monit /etc/monit/conf-enabled/java
+COPY ./docker/java-golr-monit /etc/monit/conf-enabled/java
+COPY ./docker/console-capture-golr.xml /etc/jetty9/console-capture.xml
 ENTRYPOINT [ "/entrypoint.sh" ]
 CMD ["tail", "-f", "/dev/null" ]
diff --git a/docker/console-capture-golr.xml b/docker/console-capture-golr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
+
+<Configure id="logging" class="org.eclipse.jetty.util.log.Log">
+    <New id="ServerLog" class="java.io.PrintStream">
+      <Arg>
+        <New class="org.eclipse.jetty.util.RolloverFileOutputStream">
+                <Arg><Property name="jetty.console-capture.dir" deprecated="jetty.logging.dir" default="/var/log/jetty9"/>/yyyy_mm_dd.jetty.log</Arg>
+          <Arg type="boolean"><Property name="jetty.console-capture.append" deprecated="jetty.logging.append" default="false"/></Arg>
+          <Arg type="int"><Property name="jetty.console-capture.retainDays" deprecated="jetty.logging.retainDays" default="30"/></Arg>
+          <Arg>
+            <Call class="java.util.TimeZone" name="getTimeZone">
+              <Arg><Property name="jetty.console-capture.timezone" deprecated="jetty.logging.timezone" default="GMT"/></Arg>
+            </Call>
+          </Arg>
+          <Get id="ServerLogName" name="datedFilename"/>
+        </New>
+      </Arg>
+    </New>
+
+    <Get name="rootLogger">
+      <Call name="info"><Arg>Console stderr/stdout captured to <Ref refid="ServerLogName"/></Arg></Call>
+    </Get>
+    <Call class="java.lang.System" name="setErr"><Arg><Ref refid="ServerLog"/></Arg></Call>
+    <Call class="java.lang.System" name="setOut"><Arg><Ref refid="ServerLog"/></Arg></Call>
+</Configure>
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -37,7 +37,9 @@ if [ $AMIGO -ne 0 ]; then
 fi
 
 if [ $GOLR -ne 0 ]; then
-   echo "Starting the jetty server with Solr installed"
+   echo "Starting crond"
+   /etc/init.d/cron start
+   echo "Starting monit which will start the jetty server with Solr installed"
    /etc/init.d/monit start
 fi
 

diff --git a/docker/java → docker/java-golr-monit b/docker/java → docker/java-golr-monit
@@ -2,5 +2,4 @@ check process java
         matching "java"
         start program = "/run-golr.sh"
         stop program = "/usr/bin/killall java"
-        if loadavg (5min) > 3 then restart
         if failed port 8080 then restart
diff --git a/docker/run-golr.sh b/docker/run-golr.sh
@@ -2,6 +2,8 @@
 
 SOLR_MEM=${GOLR_SOLR_MEMORY:="4G"}
 
+killall java
 echo "Starting the jetty server with Solr installed ($SOLR_MEM)"
+mkdir -p /var/log/jetty9
 cd /usr/share/jetty9
-java -Xms$SOLR_MEM -Xmx$SOLR_MEM -DentityExpansionLimit=8172000 -Djava.awt.headless=true -Dsolr.solr.home=/srv/solr -Djava.io.tmpdir=/tmp/jetty9 -Djava.library.path=/usr/lib -Djetty.home=/usr/share/jetty9 -Djetty.logs=/var/log/jetty9 -Djetty.state=/tmp/jetty.state -Djetty.host=0.0.0.0 -Djetty.port=8080 -jar /usr/share/jetty9/start.jar --daemon /etc/jetty9/jetty-started.xml &
+java -Xms$SOLR_MEM -Xmx$SOLR_MEM -DentityExpansionLimit=8172000 -Djava.awt.headless=true -Dsolr.solr.home=/srv/solr -Djava.io.tmpdir=/tmp/jetty9 -Djava.library.path=/usr/lib -Djetty.home=/usr/share/jetty9 -Djetty.state=/tmp/jetty.state -Djetty.http.host=0.0.0.0 -Djetty.http.port=8080 -jar /usr/share/jetty9/start.jar --daemon /etc/jetty9/jetty-started.xml /etc/jetty9/console-capture.xml &
diff --git a/provision/stage.yaml b/provision/stage.yaml
@@ -96,10 +96,13 @@
       - not CREATE_INDEX
       - not index_result.stat.exists
 
-  - name: Create proxy-configs directory
+  - name: Create config directories
     file:
-      path: '{{ stage_dir }}/proxy-configs'
+      path: '{{ stage_dir }}/{{ item }}'
       state: directory
+    with_items:
+      - proxy-configs 
+      - golr-configs
 
   - name: install docker-compose.yaml and http configs ...
     template:
@@ -113,3 +116,5 @@
       - { file: 'httpd-vhosts-amigo.conf', dir: '/proxy-configs' }
       - { file: 'httpd-vhosts-golr.conf', dir: '/proxy-configs' }
       - { file: 's3cfg', dir: '/proxy-configs' }
+      - { file: 'java-golr-monit', dir: '/golr-configs' }
+      - { file: 'console-capture-golr.xml', dir: '/golr-configs' }
diff --git a/provision/templates/console-capture-golr.xml b/provision/templates/console-capture-golr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
+
+<Configure id="logging" class="org.eclipse.jetty.util.log.Log">
+    <New id="ServerLog" class="java.io.PrintStream">
+      <Arg>
+        <New class="org.eclipse.jetty.util.RolloverFileOutputStream">
+                <Arg><Property name="jetty.console-capture.dir" deprecated="jetty.logging.dir" default="/var/log/jetty9"/>/yyyy_mm_dd.jetty.log</Arg>
+          <Arg type="boolean"><Property name="jetty.console-capture.append" deprecated="jetty.logging.append" default="false"/></Arg>
+          <Arg type="int"><Property name="jetty.console-capture.retainDays" deprecated="jetty.logging.retainDays" default="{{ JETTY_RETAIN_LOGS_IN_DAYS }}"/></Arg>
+          <Arg>
+            <Call class="java.util.TimeZone" name="getTimeZone">
+              <Arg><Property name="jetty.console-capture.timezone" deprecated="jetty.logging.timezone" default="GMT"/></Arg>
+            </Call>
+          </Arg>
+          <Get id="ServerLogName" name="datedFilename"/>
+        </New>
+      </Arg>
+    </New>
+
+    <Get name="rootLogger">
+      <Call name="info"><Arg>Console stderr/stdout captured to <Ref refid="ServerLogName"/></Arg></Call>
+    </Get>
+    <Call class="java.lang.System" name="setErr"><Arg><Ref refid="ServerLog"/></Arg></Call>
+    <Call class="java.lang.System" name="setOut"><Arg><Ref refid="ServerLog"/></Arg></Call>
+</Configure>
diff --git a/provision/templates/docker-compose.yaml b/provision/templates/docker-compose.yaml
@@ -9,9 +9,10 @@ services:
       - GOLR=1
       - AMIGO=0
     volumes:
-      - {{ repo_dir }}:/srv/amigo
+      - {{ stage_dir }}/golr-configs/java-golr-monit:/etc/monit/conf-enabled/java
+      - {{ stage_dir }}/golr-configs/console-capture-golr.xml:/etc/jetty9/console-capture.xml
       - {{ stage_dir }}/srv-solr-data:/srv/solr/data
-      - {{ stage_dir }}/amigo.yaml:/srv/amigo/conf/amigo.yaml
+      - {{ stage_dir }}/golr_logs:/var/log
     restart: unless-stopped
 
   amigo:

diff --git a/provision/templates/java-golr-monit b/provision/templates/java-golr-monit
@@ -0,0 +1,10 @@
+check system golr 
+        if loadavg (1min) > {{ LOAD_AVG }} then alert
+        if loadavg (5min) > {{ LOAD_AVG }} then exec "/run-golr.sh" 
+
+check process java 
+        matching "java"
+        start program = "/run-golr.sh"
+        stop program = "/usr/bin/killall java"
+        if failed port 8080 then restart
+        depend golr
diff --git a/provision/vars.yaml b/provision/vars.yaml
@@ -36,6 +36,12 @@ ACCESS_KEY: REPLACE_ME
 SECRET_KEY: REPLACE_ME
 S3_BUCKET: REPLACE_ME
 
+########
+# For golr monit and jetty.  
+#######`
+LOAD_AVG: 3
+JETTY_RETAIN_LOGS_IN_DAYS: 30
+
 #####
 AMIGO_DYNAMIC: amigo.example.com
 AMIGO_DYNAMIC_ALIAS: amigo.example.com