diff --git a/modules/distribution/product/pom.xml b/modules/distribution/product/pom.xml
index 3a29b1f6e4..9168a225ef 100644
--- a/modules/distribution/product/pom.xml
+++ b/modules/distribution/product/pom.xml
@@ -68,6 +68,10 @@
org.wso2.orbit.com.lmax
disruptor
+
+ org.wso2.diagnostics
+ runtime-diagnostics
+
@@ -92,6 +96,15 @@
true
target
+
+ org.wso2.diagnostics
+ runtime-diagnostics
+ ${diagnostics.tool.version}
+ zip
+ true
+ target
+ diagnostics-tool
+
diff --git a/modules/distribution/product/src/main/assembly/bin.xml b/modules/distribution/product/src/main/assembly/bin.xml
index 2df75d517f..f9f71e14b3 100644
--- a/modules/distribution/product/src/main/assembly/bin.xml
+++ b/modules/distribution/product/src/main/assembly/bin.xml
@@ -175,6 +175,26 @@
**/lib/xalan*.jar
+
+ target/diagnostics-tool
+ wso2am-${pom.version}/diagnostics-tool
+
+ bin/diagnostics
+ lib/**
+
+
+
+ target/diagnostics-tool/lib
+ wso2am-${pom.version}/lib
+
+ antlr-runtime-*.jar
+ gson*.jar
+ commons-lang3-*.jar
+ log4j-api-*.jar
+ log4j-core-*.jar
+ cava-toml-*.jar
+
+
target/wso2carbon-core-${carbon.kernel.version}
wso2am-${pom.version}
@@ -1291,6 +1311,21 @@
755
+
+
+ wso2am-${pom.version}/diagnostics-tool/bin/
+ true
+ 755
+
+
+
+
+ wso2am-${pom.version}/repository/resources/conf/templates/diagnostics-tool/conf
+ config.toml.j2
+ true
+ 644
+
+
wso2am-${pom.version}/repository/conf/etc
diff --git a/modules/distribution/product/src/main/conf/log4j2.properties b/modules/distribution/product/src/main/conf/log4j2.properties
index 6225df7eab..61d6414e6c 100644
--- a/modules/distribution/product/src/main/conf/log4j2.properties
+++ b/modules/distribution/product/src/main/conf/log4j2.properties
@@ -118,7 +118,7 @@ appender.ERROR_LOGFILE.name = ERROR_LOGFILE
appender.ERROR_LOGFILE.fileName = ${sys:carbon.home}/repository/logs/wso2-apigw-errors.log
appender.ERROR_LOGFILE.filePattern = ${sys:carbon.home}/repository/logs/wso2-apigw-errors-%d{MM-dd-yyyy}-%i.log.gz
appender.ERROR_LOGFILE.layout.type = PatternLayout
-appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p %c{1} %m%n
+appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p {%c} %m%n
appender.ERROR_LOGFILE.policies.type = Policies
appender.ERROR_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy
appender.ERROR_LOGFILE.policies.time.interval = 1
diff --git a/modules/distribution/product/src/main/resources/conf/default.json b/modules/distribution/product/src/main/resources/conf/default.json
index 987c1a15d4..2b6b52f6a3 100644
--- a/modules/distribution/product/src/main/resources/conf/default.json
+++ b/modules/distribution/product/src/main/resources/conf/default.json
@@ -471,5 +471,64 @@
"apim.analytics.properties.truststore_location": "${carbon.home}/repository/resources/security/$ref{truststore.file_name}",
"apim.analytics.properties.truststore_password": "$ref{truststore.password}",
"tenant_mgt.disable_email_domain_validation": true,
- "apim.jwt.use_kid_property": true
+ "apim.jwt.use_kid_property": true,
+ "server_configuration": {
+ "deployment_toml_path": "../conf/deployment.toml",
+ "logs_directory": "../repository/logs",
+ "updates_config_path": "../updates/config.json",
+ "diagnostic_log_file_path": "logs/diagnostics.log",
+ "carbon_log_file_path": "../repository/logs/wso2-apigw-errors.log",
+ "process_id_path": "../wso2carbon.pid",
+ "server_name": "WSO2 API Manager",
+ "server_version": "4.3.0"
+ },
+ "cpu_watcher": {
+ "enabled": "true",
+ "threshold": "80",
+ "retry_count": "2",
+ "interval": "5",
+ "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+ },
+ "memory_watcher": {
+ "enabled": "true",
+ "threshold": "80",
+ "retry_count": "2",
+ "interval": "5",
+ "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+ },
+ "log_watcher": {
+ "enabled": "true",
+ "interval": 0.1
+ },
+ "traffic_analyzer": {
+ "last_second_requests_enabled": "false",
+ "last_second_requests_windows_size": "300",
+ "last_second_requests_delay": "60",
+ "last_second_requests_interval": "1",
+ "last_fifteen_seconds_requests_enabled": "true",
+ "last_fifteen_seconds_requests_window_size": "100",
+ "last_fifteen_seconds_requests_delay": "4",
+ "last_fifteen_seconds_requests_interval": "15",
+ "last_minutes_requests_enabled": "true",
+ "last_minutes_requests_window_size": "100",
+ "last_minutes_requests_delay": "1",
+ "last_minutes_requests_interval": "60",
+ "notify_interval": "60"
+ },
+ "zip_file_configuration": {
+ "output_directory": "data",
+ "max_count": "20"
+ },
+ "log_pattern.patterns": [
+ {
+ "pattern.regex": "(.*)org.apache.synapse.transport.passthru(.*)",
+ "pattern.executors": "MetricsSnapshot,Netstat,OpenFileFinder,ThreadDumper,ServerInfo",
+ "pattern.reload_time": "30"
+ },
+ {
+ "pattern.regex": "(.*)org.apache.synapse(.*)",
+ "pattern.executors": "ServerInfo",
+ "pattern.reload_time": "10"
+ }
+ ]
}
diff --git a/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2 b/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2
new file mode 100644
index 0000000000..65994e110d
--- /dev/null
+++ b/modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2
@@ -0,0 +1,143 @@
+#
+# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+##############################################
+#
+# WARNING: Don't edit the file manually unless you are not using the deployment.toml file.
+#
+##############################################
+
+## This file contains the configuration parameters used by the Diagnostic tool
+
+# Server Configurations
+[server_configuration]
+deployment_toml_path = "{{server_configuration.deployment_toml_path}}"
+logs_directory = "{{server_configuration.logs_directory}}"
+updates_config_path = "{{server_configuration.updates_config_path}}"
+diagnostic_log_file_path = "{{server_configuration.diagnostic_log_file_path}}"
+carbon_log_file_path = "{{server_configuration.carbon_log_file_path}}"
+process_id_path = "{{server_configuration.process_id_path}}"
+server_name = "@product.name@"
+server_version = "@product.version@"
+
+## Action Executor Configurations
+
+# Example
+#[[action_executor_configuration]]
+#executor = "ActionExecutor"
+#reload_time = "180" # in seconds
+
+[[action_executor_configuration]]
+executor = "MemoryDumper"
+reload_time = "180"
+
+[[action_executor_configuration]]
+executor = "ThreadDumper"
+count = "5"
+delay = "2000"
+
+[[action_executor_configuration]]
+executor = "OpenFileFinder"
+
+[[action_executor_configuration]]
+executor = "Netstat"
+command = "netstat -alt"
+
+[[action_executor_configuration]]
+executor = "ServerInfo"
+
+[[action_executor_configuration]]
+executor = "MetricsSnapshot"
+
+{%for action in action_executor_configuration %}
+[[action_executor_configuration]]
+executor = "{{action.executor}}"
+{% endfor %}
+
+# Watcher Configurations
+[cpu_watcher]
+enabled = "{{cpu_watcher.enabled}}"
+threshold = "{{cpu_watcher.threshold}}"
+retry_count = "{{cpu_watcher.retry_count}}"
+interval = "{{cpu_watcher.interval}}"
+action_executors = "{{cpu_watcher.action_executors}}"
+
+[memory_watcher]
+enabled = "{{memory_watcher.enabled}}"
+threshold = "{{memory_watcher.threshold}}"
+retry_count = "{{memory_watcher.retry_count}}"
+interval = "{{memory_watcher.interval}}"
+action_executors = "{{memory_watcher.action_executors}}"
+
+[log_watcher]
+enabled = "{{log_watcher.enabled}}"
+interval = {{log_watcher.interval}}
+
+# Traffic Analyzer Configurations
+[traffic_analyzer]
+last_second_requests_enabled = "{{traffic_analyzer.last_second_requests_enabled}}"
+last_second_requests_windows_size = "{{traffic_analyzer.last_second_requests_windows_size}}"
+last_second_requests_delay = "{{traffic_analyzer.last_second_requests_delay}}"
+last_second_requests_interval = "{{traffic_analyzer.last_second_requests_interval}}"
+last_fifteen_seconds_requests_enabled = "{{traffic_analyzer.last_fifteen_seconds_requests_enabled}}"
+last_fifteen_seconds_requests_window_size = "{{traffic_analyzer.last_fifteen_seconds_requests_window_size}}"
+last_fifteen_seconds_requests_delay = "{{traffic_analyzer.last_fifteen_seconds_requests_delay}}"
+last_fifteen_seconds_requests_interval = "{{traffic_analyzer.last_fifteen_seconds_requests_interval}}"
+last_minutes_requests_enabled = "{{traffic_analyzer.last_minutes_requests_enabled}}"
+last_minutes_requests_window_size = "{{traffic_analyzer.last_minutes_requests_window_size}}"
+last_minutes_requests_delay = "{{traffic_analyzer.last_minutes_requests_delay}}"
+last_minutes_requests_interval = "{{traffic_analyzer.last_minutes_requests_interval}}"
+notify_interval = "{{traffic_analyzer.notify_interval}}"
+
+# Output data zip configurations
+[zip_file_configuration]
+output_directory = "{{zip_file_configuration.output_directory}}"
+max_count = "{{zip_file_configuration.max_count}}"
+
+# Error regex patterns and diagnosis
+{%for pattern in log_pattern.patterns %}
+[[log_pattern]]
+regex = "{{pattern.pattern.regex}}"
+executors = "{{pattern.pattern.executors}}"
+reload_time = "{{pattern.pattern.reload_time}}"
+
+{% endfor %}
+
+{% if ftp_uploader is defined %}
+## FTP Uploader configurations
+[ftp_uploader]
+enabled = "{{ftp_uploader.enabled}}"
+host = "{{ftp_uploader.host}}"
+port = "{{ftp_uploader.port}}"
+username = "{{ftp_uploader.username}}"
+password = "{{ftp_uploader.password}}"
+directory = "{{ftp_uploader.directory}}"
+{% endif %}
+
+{% if sftp_uploader is defined %}
+## SFTP Uploader configurations
+[sftp_uploader]
+enabled = "{{sftp_uploader.enabled}}"
+host = "{{sftp_uploader.host}}"
+port = "{{sftp_uploader.port}}"
+username = "{{sftp_uploader.username}}"
+password = "{{sftp_uploader.password}}"
+directory = "{{sftp_uploader.directory}}"
+known_hosts_path = "{{sftp_uploader.known_hosts_path}}"
+strict_host_key_checking = "{{sftp_uploader.strict_host_key_checking}}"
+{% endif %}
diff --git a/modules/distribution/product/src/main/startup-scripts/api-manager.sh b/modules/distribution/product/src/main/startup-scripts/api-manager.sh
index a5d7a17ef3..45fe5f6a42 100755
--- a/modules/distribution/product/src/main/startup-scripts/api-manager.sh
+++ b/modules/distribution/product/src/main/startup-scripts/api-manager.sh
@@ -313,6 +313,16 @@ if [ $java_version_formatted -ge 1700 ]; then
JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED"
fi
+# start diagnostic tool in background in diagnostic-tool/bin/diagnostic
+"$CARBON_HOME"/diagnostics-tool/bin/diagnostics.sh &
+diagnostic_tool_pid=$!
+
+# trap signals so we can shutdown the diagnostic tool
+cleanup() {
+ kill "$diagnostic_tool_pid"
+}
+trap 'cleanup' EXIT
+
while [ "$status" = "$START_EXIT_STATUS" ]
do
$JAVACMD \
diff --git a/modules/distribution/product/src/main/startup-scripts/diagnostics.sh b/modules/distribution/product/src/main/startup-scripts/diagnostics.sh
new file mode 100644
index 0000000000..ffb8d96ce8
--- /dev/null
+++ b/modules/distribution/product/src/main/startup-scripts/diagnostics.sh
@@ -0,0 +1,139 @@
+#!/bin/sh
+# diagnostics.sh
+# ----------------------------------------------------------------------------
+# Copyright 2024 WSO2, LLC. http://www.wso2.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+BASEDIR=`cd "$PRGDIR/.." >/dev/null; pwd`
+
+# Reset the REPO variable. If you need to influence this use the environment setup file.
+REPO=
+
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ Darwin*) darwin=true
+ if [ -z "$JAVA_VERSION" ] ; then
+ JAVA_VERSION="CurrentJDK"
+ else
+ echo "Using Java version: $JAVA_VERSION"
+ fi
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ JAVA_HOME=`/usr/libexec/java_home`
+ else
+ JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=`java-config --jre-home`
+ fi
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# If a specific java binary isn't specified search for the standard 'java' binary
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD=`which java`
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." 1>&2
+ echo " We cannot execute $JAVACMD" 1>&2
+ exit 1
+fi
+
+TMP_DIR="$BASEDIR"/temp
+if [ -d "$TMP_DIR" ]; then
+ rm -rf "$TMP_DIR"/*
+fi
+
+if [ -z "$REPO" ]
+then
+ REPO="$BASEDIR"/../lib
+fi
+
+CLASSPATH="$BASEDIR"/conf:"$REPO"/*
+
+ENDORSED_DIR=
+if [ -n "$ENDORSED_DIR" ] ; then
+ CLASSPATH=$BASEDIR/$ENDORSED_DIR/*:$CLASSPATH
+fi
+
+if [ -n "$CLASSPATH_PREFIX" ] ; then
+ CLASSPATH=$CLASSPATH_PREFIX:$CLASSPATH
+fi
+
+if [ -z "$JVM_MEM_OPTS" ]; then
+ java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}')
+ JVM_MEM_OPTS="-Xms32m -Xmx128m"
+ if [ "$java_version" \< "1.8" ]; then
+ JVM_MEM_OPTS="$JVM_MEM_OPTS"
+ fi
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+ [ -n "$HOME" ] && HOME=`cygpath --path --windows "$HOME"`
+ [ -n "$BASEDIR" ] && BASEDIR=`cygpath --path --windows "$BASEDIR"`
+ [ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"`
+fi
+
+exec "$JAVACMD" $JAVA_OPTS \
+$JVM_MEM_OPTS \
+ -classpath "$CLASSPATH" \
+ -Dapp.name="runtime-diagnostics" \
+ -Dapp.pid="$$" \
+ -Dapp.repo="$REPO" \
+ -Dapp.home="$BASEDIR" \
+ -Dbasedir="$BASEDIR" \
+ org.wso2.diagnostics.DiagnosticsApp \
+ "$@"
diff --git a/pom.xml b/pom.xml
index 6cab144c82..9d03a50747 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1191,6 +1191,13 @@
${javax.jms.version}
+
+
+ org.wso2.diagnostics
+ runtime-diagnostics
+ ${diagnostics.tool.version}
+
+
com.fasterxml.jackson.dataformat
@@ -1470,6 +1477,9 @@
2.1.0
1.0.2
+
+ 1.0.3
+
5.11.33
2.0.14