From e7b5c1fe76c45a1d1a3db568bd376ab503b9f0e0 Mon Sep 17 00:00:00 2001 From: arunan Date: Tue, 5 Mar 2024 17:26:17 +0530 Subject: [PATCH] Include Runtime diagnostics tool into the server --- modules/distribution/product/pom.xml | 13 ++ .../product/src/main/assembly/bin.xml | 35 +++++ .../product/src/main/conf/log4j2.properties | 2 +- .../src/main/resources/conf/default.json | 61 +++++++- .../diagnostics-tool/conf/config.toml.j2 | 143 ++++++++++++++++++ .../src/main/startup-scripts/api-manager.sh | 10 ++ .../src/main/startup-scripts/diagnostics.sh | 139 +++++++++++++++++ pom.xml | 10 ++ 8 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 create mode 100644 modules/distribution/product/src/main/startup-scripts/diagnostics.sh diff --git a/modules/distribution/product/pom.xml b/modules/distribution/product/pom.xml index 3a29b1f6e4..9168a225ef 100644 --- a/modules/distribution/product/pom.xml +++ b/modules/distribution/product/pom.xml @@ -68,6 +68,10 @@ org.wso2.orbit.com.lmax disruptor + + org.wso2.diagnostics + runtime-diagnostics + @@ -92,6 +96,15 @@ true target + + org.wso2.diagnostics + runtime-diagnostics + ${diagnostics.tool.version} + zip + true + target + diagnostics-tool + diff --git a/modules/distribution/product/src/main/assembly/bin.xml b/modules/distribution/product/src/main/assembly/bin.xml index 2df75d517f..f9f71e14b3 100644 --- a/modules/distribution/product/src/main/assembly/bin.xml +++ b/modules/distribution/product/src/main/assembly/bin.xml @@ -175,6 +175,26 @@ **/lib/xalan*.jar + + target/diagnostics-tool + wso2am-${pom.version}/diagnostics-tool + + bin/diagnostics + lib/** + + + + target/diagnostics-tool/lib + wso2am-${pom.version}/lib + + antlr-runtime-*.jar + gson*.jar + commons-lang3-*.jar + log4j-api-*.jar + log4j-core-*.jar + cava-toml-*.jar + + target/wso2carbon-core-${carbon.kernel.version} wso2am-${pom.version} @@ -1291,6 +1311,21 @@ 755 + + src/main/startup-scripts/diagnostics.sh + wso2am-${pom.version}/diagnostics-tool/bin/ + true + 755 + + + + src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 + wso2am-${pom.version}/repository/resources/conf/templates/diagnostics-tool/conf + config.toml.j2 + true + 644 + + ../../../config/bam.xml wso2am-${pom.version}/repository/conf/etc diff --git a/modules/distribution/product/src/main/conf/log4j2.properties b/modules/distribution/product/src/main/conf/log4j2.properties index 6225df7eab..61d6414e6c 100644 --- a/modules/distribution/product/src/main/conf/log4j2.properties +++ b/modules/distribution/product/src/main/conf/log4j2.properties @@ -118,7 +118,7 @@ appender.ERROR_LOGFILE.name = ERROR_LOGFILE appender.ERROR_LOGFILE.fileName = ${sys:carbon.home}/repository/logs/wso2-apigw-errors.log appender.ERROR_LOGFILE.filePattern = ${sys:carbon.home}/repository/logs/wso2-apigw-errors-%d{MM-dd-yyyy}-%i.log.gz appender.ERROR_LOGFILE.layout.type = PatternLayout -appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p %c{1} %m%n +appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p {%c} %m%n appender.ERROR_LOGFILE.policies.type = Policies appender.ERROR_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy appender.ERROR_LOGFILE.policies.time.interval = 1 diff --git a/modules/distribution/product/src/main/resources/conf/default.json b/modules/distribution/product/src/main/resources/conf/default.json index 987c1a15d4..2b6b52f6a3 100644 --- a/modules/distribution/product/src/main/resources/conf/default.json +++ b/modules/distribution/product/src/main/resources/conf/default.json @@ -471,5 +471,64 @@ "apim.analytics.properties.truststore_location": "${carbon.home}/repository/resources/security/$ref{truststore.file_name}", "apim.analytics.properties.truststore_password": "$ref{truststore.password}", "tenant_mgt.disable_email_domain_validation": true, - "apim.jwt.use_kid_property": true + "apim.jwt.use_kid_property": true, + "server_configuration": { + "deployment_toml_path": "../conf/deployment.toml", + "logs_directory": "../repository/logs", + "updates_config_path": "../updates/config.json", + "diagnostic_log_file_path": "logs/diagnostics.log", + "carbon_log_file_path": "../repository/logs/wso2-apigw-errors.log", + "process_id_path": "../wso2carbon.pid", + "server_name": "WSO2 API Manager", + "server_version": "4.3.0" + }, + "cpu_watcher": { + "enabled": "true", + "threshold": "80", + "retry_count": "2", + "interval": "5", + "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo" + }, + "memory_watcher": { + "enabled": "true", + "threshold": "80", + "retry_count": "2", + "interval": "5", + "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo" + }, + "log_watcher": { + "enabled": "true", + "interval": 0.1 + }, + "traffic_analyzer": { + "last_second_requests_enabled": "false", + "last_second_requests_windows_size": "300", + "last_second_requests_delay": "60", + "last_second_requests_interval": "1", + "last_fifteen_seconds_requests_enabled": "true", + "last_fifteen_seconds_requests_window_size": "100", + "last_fifteen_seconds_requests_delay": "4", + "last_fifteen_seconds_requests_interval": "15", + "last_minutes_requests_enabled": "true", + "last_minutes_requests_window_size": "100", + "last_minutes_requests_delay": "1", + "last_minutes_requests_interval": "60", + "notify_interval": "60" + }, + "zip_file_configuration": { + "output_directory": "data", + "max_count": "20" + }, + "log_pattern.patterns": [ + { + "pattern.regex": "(.*)org.apache.synapse.transport.passthru(.*)", + "pattern.executors": "MetricsSnapshot,Netstat,OpenFileFinder,ThreadDumper,ServerInfo", + "pattern.reload_time": "30" + }, + { + "pattern.regex": "(.*)org.apache.synapse(.*)", + "pattern.executors": "ServerInfo", + "pattern.reload_time": "10" + } + ] } diff --git a/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 b/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 new file mode 100644 index 0000000000..65994e110d --- /dev/null +++ b/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 @@ -0,0 +1,143 @@ +# +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +############################################## +# +# WARNING: Don't edit the file manually unless you are not using the deployment.toml file. +# +############################################## + +## This file contains the configuration parameters used by the Diagnostic tool + +# Server Configurations +[server_configuration] +deployment_toml_path = "{{server_configuration.deployment_toml_path}}" +logs_directory = "{{server_configuration.logs_directory}}" +updates_config_path = "{{server_configuration.updates_config_path}}" +diagnostic_log_file_path = "{{server_configuration.diagnostic_log_file_path}}" +carbon_log_file_path = "{{server_configuration.carbon_log_file_path}}" +process_id_path = "{{server_configuration.process_id_path}}" +server_name = "@product.name@" +server_version = "@product.version@" + +## Action Executor Configurations + +# Example +#[[action_executor_configuration]] +#executor = "ActionExecutor" +#reload_time = "180" # in seconds + +[[action_executor_configuration]] +executor = "MemoryDumper" +reload_time = "180" + +[[action_executor_configuration]] +executor = "ThreadDumper" +count = "5" +delay = "2000" + +[[action_executor_configuration]] +executor = "OpenFileFinder" + +[[action_executor_configuration]] +executor = "Netstat" +command = "netstat -alt" + +[[action_executor_configuration]] +executor = "ServerInfo" + +[[action_executor_configuration]] +executor = "MetricsSnapshot" + +{%for action in action_executor_configuration %} +[[action_executor_configuration]] +executor = "{{action.executor}}" +{% endfor %} + +# Watcher Configurations +[cpu_watcher] +enabled = "{{cpu_watcher.enabled}}" +threshold = "{{cpu_watcher.threshold}}" +retry_count = "{{cpu_watcher.retry_count}}" +interval = "{{cpu_watcher.interval}}" +action_executors = "{{cpu_watcher.action_executors}}" + +[memory_watcher] +enabled = "{{memory_watcher.enabled}}" +threshold = "{{memory_watcher.threshold}}" +retry_count = "{{memory_watcher.retry_count}}" +interval = "{{memory_watcher.interval}}" +action_executors = "{{memory_watcher.action_executors}}" + +[log_watcher] +enabled = "{{log_watcher.enabled}}" +interval = {{log_watcher.interval}} + +# Traffic Analyzer Configurations +[traffic_analyzer] +last_second_requests_enabled = "{{traffic_analyzer.last_second_requests_enabled}}" +last_second_requests_windows_size = "{{traffic_analyzer.last_second_requests_windows_size}}" +last_second_requests_delay = "{{traffic_analyzer.last_second_requests_delay}}" +last_second_requests_interval = "{{traffic_analyzer.last_second_requests_interval}}" +last_fifteen_seconds_requests_enabled = "{{traffic_analyzer.last_fifteen_seconds_requests_enabled}}" +last_fifteen_seconds_requests_window_size = "{{traffic_analyzer.last_fifteen_seconds_requests_window_size}}" +last_fifteen_seconds_requests_delay = "{{traffic_analyzer.last_fifteen_seconds_requests_delay}}" +last_fifteen_seconds_requests_interval = "{{traffic_analyzer.last_fifteen_seconds_requests_interval}}" +last_minutes_requests_enabled = "{{traffic_analyzer.last_minutes_requests_enabled}}" +last_minutes_requests_window_size = "{{traffic_analyzer.last_minutes_requests_window_size}}" +last_minutes_requests_delay = "{{traffic_analyzer.last_minutes_requests_delay}}" +last_minutes_requests_interval = "{{traffic_analyzer.last_minutes_requests_interval}}" +notify_interval = "{{traffic_analyzer.notify_interval}}" + +# Output data zip configurations +[zip_file_configuration] +output_directory = "{{zip_file_configuration.output_directory}}" +max_count = "{{zip_file_configuration.max_count}}" + +# Error regex patterns and diagnosis +{%for pattern in log_pattern.patterns %} +[[log_pattern]] +regex = "{{pattern.pattern.regex}}" +executors = "{{pattern.pattern.executors}}" +reload_time = "{{pattern.pattern.reload_time}}" + +{% endfor %} + +{% if ftp_uploader is defined %} +## FTP Uploader configurations +[ftp_uploader] +enabled = "{{ftp_uploader.enabled}}" +host = "{{ftp_uploader.host}}" +port = "{{ftp_uploader.port}}" +username = "{{ftp_uploader.username}}" +password = "{{ftp_uploader.password}}" +directory = "{{ftp_uploader.directory}}" +{% endif %} + +{% if sftp_uploader is defined %} +## SFTP Uploader configurations +[sftp_uploader] +enabled = "{{sftp_uploader.enabled}}" +host = "{{sftp_uploader.host}}" +port = "{{sftp_uploader.port}}" +username = "{{sftp_uploader.username}}" +password = "{{sftp_uploader.password}}" +directory = "{{sftp_uploader.directory}}" +known_hosts_path = "{{sftp_uploader.known_hosts_path}}" +strict_host_key_checking = "{{sftp_uploader.strict_host_key_checking}}" +{% endif %} diff --git a/modules/distribution/product/src/main/startup-scripts/api-manager.sh b/modules/distribution/product/src/main/startup-scripts/api-manager.sh index a5d7a17ef3..45fe5f6a42 100755 --- a/modules/distribution/product/src/main/startup-scripts/api-manager.sh +++ b/modules/distribution/product/src/main/startup-scripts/api-manager.sh @@ -313,6 +313,16 @@ if [ $java_version_formatted -ge 1700 ]; then JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED" fi +# start diagnostic tool in background in diagnostic-tool/bin/diagnostic +"$CARBON_HOME"/diagnostics-tool/bin/diagnostics.sh & +diagnostic_tool_pid=$! + +# trap signals so we can shutdown the diagnostic tool +cleanup() { + kill "$diagnostic_tool_pid" +} +trap 'cleanup' EXIT + while [ "$status" = "$START_EXIT_STATUS" ] do $JAVACMD \ diff --git a/modules/distribution/product/src/main/startup-scripts/diagnostics.sh b/modules/distribution/product/src/main/startup-scripts/diagnostics.sh new file mode 100644 index 0000000000..ffb8d96ce8 --- /dev/null +++ b/modules/distribution/product/src/main/startup-scripts/diagnostics.sh @@ -0,0 +1,139 @@ +#!/bin/sh +# diagnostics.sh +# ---------------------------------------------------------------------------- +# Copyright 2024 WSO2, LLC. http://www.wso2.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# resolve links - $0 may be a softlink +PRG="$0" + +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done + +PRGDIR=`dirname "$PRG"` +BASEDIR=`cd "$PRGDIR/.." >/dev/null; pwd` + +# Reset the REPO variable. If you need to influence this use the environment setup file. +REPO= + + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +case "`uname`" in + CYGWIN*) cygwin=true ;; + Darwin*) darwin=true + if [ -z "$JAVA_VERSION" ] ; then + JAVA_VERSION="CurrentJDK" + else + echo "Using Java version: $JAVA_VERSION" + fi + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME=`/usr/libexec/java_home` + else + JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# If a specific java binary isn't specified search for the standard 'java' binary +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD=`which java` + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." 1>&2 + echo " We cannot execute $JAVACMD" 1>&2 + exit 1 +fi + +TMP_DIR="$BASEDIR"/temp +if [ -d "$TMP_DIR" ]; then + rm -rf "$TMP_DIR"/* +fi + +if [ -z "$REPO" ] +then + REPO="$BASEDIR"/../lib +fi + +CLASSPATH="$BASEDIR"/conf:"$REPO"/* + +ENDORSED_DIR= +if [ -n "$ENDORSED_DIR" ] ; then + CLASSPATH=$BASEDIR/$ENDORSED_DIR/*:$CLASSPATH +fi + +if [ -n "$CLASSPATH_PREFIX" ] ; then + CLASSPATH=$CLASSPATH_PREFIX:$CLASSPATH +fi + +if [ -z "$JVM_MEM_OPTS" ]; then + java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') + JVM_MEM_OPTS="-Xms32m -Xmx128m" + if [ "$java_version" \< "1.8" ]; then + JVM_MEM_OPTS="$JVM_MEM_OPTS" + fi +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$HOME" ] && HOME=`cygpath --path --windows "$HOME"` + [ -n "$BASEDIR" ] && BASEDIR=`cygpath --path --windows "$BASEDIR"` + [ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"` +fi + +exec "$JAVACMD" $JAVA_OPTS \ +$JVM_MEM_OPTS \ + -classpath "$CLASSPATH" \ + -Dapp.name="runtime-diagnostics" \ + -Dapp.pid="$$" \ + -Dapp.repo="$REPO" \ + -Dapp.home="$BASEDIR" \ + -Dbasedir="$BASEDIR" \ + org.wso2.diagnostics.DiagnosticsApp \ + "$@" diff --git a/pom.xml b/pom.xml index 6cab144c82..9d03a50747 100644 --- a/pom.xml +++ b/pom.xml @@ -1191,6 +1191,13 @@ ${javax.jms.version} + + + org.wso2.diagnostics + runtime-diagnostics + ${diagnostics.tool.version} + + com.fasterxml.jackson.dataformat @@ -1470,6 +1477,9 @@ 2.1.0 1.0.2 + + 1.0.3 + 5.11.33 2.0.14