W-15612002: Fix cron job scripts that report metrics (#877)

carbonj.service/src/main/docker/files/entrypoint.sh

@@ -125,7 +125,7 @@ echo $JAVA_OPTS $JAVA_OPTS_OVERRIDE
 env >> /etc/environment
 
 # no logs on disk
-crontab -l | { cat; echo "* */6 * * * /app/bin/logCleanup.sh ${SERVICEDIR}/log 7 >/dev/null 2>&1"; } | crontab -
+crontab -l | { cat; echo "* */6 * * * /app/bin/logCleanup.sh ${SERVICEDIR}/log/ 7 >/dev/null 2>&1"; } | crontab -
 crontab -l | { cat; echo "*/1 * * * * /app/bin/fdlog.sh"; } | crontab -
 crontab -l | { cat; echo "*/1 * * * * /app/bin/iolog.sh"; } | crontab -
 crontab -l | { cat; echo "*/1 * * * * /app/bin/reportGcMetrics.sh"; } | crontab -

carbonj.service/src/main/docker/files/reportGcMetrics.sh

@@ -32,7 +32,6 @@ grafPort=2003
 # gc log file
 # java 9+ has new GC logging...
 log=$( echo $( ls -t /app/log/gc.log ) | awk '{print $1}' )
-#log=$( echo $( ls -t /app/log/gc*current ) | awk '{print $1}' )
 
 #
 # do work
@@ -42,8 +41,7 @@ log=$( echo $( ls -t /app/log/gc.log ) | awk '{print $1}' )
 time=$( date +%s )
 
 # grep and sum all times in file for last one minute
-# TODO remove hashes
-gcTime=$( tail -n 10000 $log | grep "Total time" | sed 's/\[//g'|sed 's/\]/ /g'| awk -v LAST_MINUTE="$(date -ud "-1 minutes" +"%Y-%m-%dT%H:%M:%S")" '($1 " " $2) >= LAST_MINUTE' | grep -Eo "Total time for which application threads were stopped: [0-9]+\.[0-9]+\ " |awk '{ print $9}' > /tmp/gc && awk '{s+=$1} END {print s}' /tmp/gc )
+gcTime=$(tail -n 10000 $log | grep "Pause" | sed 's/\[//g' | sed 's/\]/ /g' | awk -v LAST_MINUTE="$(date -ud "-1 minutes" +"%Y-%m-%dT%H:%M:%S")" '($1 " " $2) >= LAST_MINUTE' | awk '{print $NF}' | sed 's/ms//g' > /tmp/gc && awk '{s+=$1} END {print s}' /tmp/gc)
 
 # print to carbon
 echo "$namespace $gcTime $time"

carbonj.service/src/main/docker/files/reportRocksDbMetrics.sh

@@ -16,11 +16,14 @@ serviceName=${DW_SVC_VERSION}
 graphite_host=${DW_GRAPHITE_HOST}
 graphite_port=2003
 
-all=("30m2y" "5m7d"  "60s24h"  "index-id"  "index-name")
+all=("30m2y" "5m7d" "60s24h" "60s30d" "index-id" "index-name")
 
 for db in "${all[@]}"
 do
     log="/data/carbonj-data/${db}/LOG"
+    if [ ! -f $log ]; then
+      continue
+    fi
     metric_name="$pod.$realm.carbonj.$instance.${serviceName//./_}.rocksdb.${db}.compaction.bytes"
     time=$( date +%s )
     # Format date: 2017/11/24-12:22