Include Runtime diagnostics tool into the server

modules/distribution/product/pom.xml

@@ -68,6 +68,10 @@
  <groupId>org.wso2.orbit.com.lmax</groupId>
  <artifactId>disruptor</artifactId>
  </dependency>
+ <dependency>
+ <groupId>org.wso2.diagnostics</groupId>
+ <artifactId>runtime-diagnostics</artifactId>
+ </dependency>
  </dependencies>
  <build>
  <plugins>
@@ -92,6 +96,15 @@
  <overWrite>true</overWrite>
  <outputDirectory>target</outputDirectory>
  </artifactItem>
+ <artifactItem>
+ <groupId>org.wso2.diagnostics</groupId>
+ <artifactId>runtime-diagnostics</artifactId>
+ <version>${diagnostics.tool.version}</version>
+ <type>zip</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>target</outputDirectory>
+ <destFileName>diagnostics-tool</destFileName>
+ </artifactItem>
  </artifactItems>
  </configuration>
  </execution>

modules/distribution/product/src/main/assembly/bin.xml

@@ -175,6 +175,26 @@
  <exclude>**/lib/xalan*.jar</exclude>
  </excludes>
  </fileSet>
+ <fileSet>
+ <directory>target/diagnostics-tool</directory>
+ <outputDirectory>wso2am-${pom.version}/diagnostics-tool</outputDirectory>
+ <excludes>
+ <exclude>bin/diagnostics</exclude>
+ <exclude>lib/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target/diagnostics-tool/lib</directory>
+ <outputDirectory>wso2am-${pom.version}/lib</outputDirectory>
+ <excludes>
+ <exclude>antlr-runtime-*.jar</exclude>
+ <exclude>gson*.jar</exclude>
+ <exclude>commons-lang3-*.jar</exclude>
+ <exclude>log4j-api-*.jar</exclude>
+ <exclude>log4j-core-*.jar</exclude>
+ <exclude>cava-toml-*.jar</exclude>
+ </excludes>
+ </fileSet>
  <fileSet>
  <directory>target/wso2carbon-core-${carbon.kernel.version}</directory>
  <outputDirectory>wso2am-${pom.version}</outputDirectory>
@@ -1291,6 +1311,21 @@
  <fileMode>755</fileMode>
  </file>
 
+ <file>
+ <source>src/main/startup-scripts/diagnostics.sh</source>
+ <outputDirectory>wso2am-${pom.version}/diagnostics-tool/bin/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>755</fileMode>
+ </file>
+
+ <file>
+ <source>src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2</source>
+ <outputDirectory>wso2am-${pom.version}/repository/resources/conf/templates/diagnostics-tool/conf</outputDirectory>
+ <destName>config.toml.j2</destName>
+ <filtered>true</filtered>
+ <fileMode>644</fileMode>
+ </file>
+
  <file>
  <source>../../../config/bam.xml</source>
  <outputDirectory>wso2am-${pom.version}/repository/conf/etc</outputDirectory>

modules/distribution/product/src/main/conf/log4j2.properties

@@ -118,7 +118,7 @@ appender.ERROR_LOGFILE.name = ERROR_LOGFILE
 appender.ERROR_LOGFILE.fileName = ${sys:carbon.home}/repository/logs/wso2-apigw-errors.log
 appender.ERROR_LOGFILE.filePattern = ${sys:carbon.home}/repository/logs/wso2-apigw-errors-%d{MM-dd-yyyy}-%i.log.gz
 appender.ERROR_LOGFILE.layout.type = PatternLayout
-appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p %c{1} %m%n
+appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p {%c} %m%n
 appender.ERROR_LOGFILE.policies.type = Policies
 appender.ERROR_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy
 appender.ERROR_LOGFILE.policies.time.interval = 1

modules/distribution/product/src/main/resources/conf/default.json

@@ -471,5 +471,64 @@
  "apim.analytics.properties.truststore_location": "${carbon.home}/repository/resources/security/$ref{truststore.file_name}",
  "apim.analytics.properties.truststore_password": "$ref{truststore.password}",
  "tenant_mgt.disable_email_domain_validation": true,
- "apim.jwt.use_kid_property": true
+ "apim.jwt.use_kid_property": true,
+ "server_configuration": {
+ "deployment_toml_path": "../conf/deployment.toml",
+ "logs_directory": "../repository/logs",
+ "updates_config_path": "../updates/config.json",
+ "diagnostic_log_file_path": "logs/diagnostics.log",
+ "carbon_log_file_path": "../repository/logs/wso2-apigw-errors.log",
+ "process_id_path": "../wso2carbon.pid",
+ "server_name": "WSO2 API Manager",
+ "server_version": "4.3.0"
+ },
+ "cpu_watcher": {
+ "enabled": "true",
+ "threshold": "80",
+ "retry_count": "2",
+ "interval": "5",
+ "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+ },
+ "memory_watcher": {
+ "enabled": "true",
+ "threshold": "80",
+ "retry_count": "2",
+ "interval": "5",
+ "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+ },
+ "log_watcher": {
+ "enabled": "true",
+ "interval": 0.1
+ },
+ "traffic_analyzer": {
+ "last_second_requests_enabled": "false",
+ "last_second_requests_windows_size": "300",
+ "last_second_requests_delay": "60",
+ "last_second_requests_interval": "1",
+ "last_fifteen_seconds_requests_enabled": "true",
+ "last_fifteen_seconds_requests_window_size": "100",
+ "last_fifteen_seconds_requests_delay": "4",
+ "last_fifteen_seconds_requests_interval": "15",
+ "last_minutes_requests_enabled": "true",
+ "last_minutes_requests_window_size": "100",
+ "last_minutes_requests_delay": "1",
+ "last_minutes_requests_interval": "60",
+ "notify_interval": "60"
+ },
+ "zip_file_configuration": {
+ "output_directory": "data",
+ "max_count": "20"
+ },
+ "log_pattern.patterns": [
+ {
+ "pattern.regex": "(.*)org.apache.synapse.transport.passthru(.*)",
+ "pattern.executors": "MetricsSnapshot,Netstat,OpenFileFinder,ThreadDumper,ServerInfo",
+ "pattern.reload_time": "30"
+ },
+ {
+ "pattern.regex": "(.*)org.apache.synapse(.*)",
+ "pattern.executors": "ServerInfo",
+ "pattern.reload_time": "10"
+ }
+ ]
 }

modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2

@@ -0,0 +1,143 @@
+#
+# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+##############################################
+#
+# WARNING: Don't edit the file manually unless you are not using the deployment.toml file.
+#
+##############################################
+
+## This file contains the configuration parameters used by the Diagnostic tool
+
+# Server Configurations
+[server_configuration]
+deployment_toml_path = "{{server_configuration.deployment_toml_path}}"
+logs_directory = "{{server_configuration.logs_directory}}"
+updates_config_path = "{{server_configuration.updates_config_path}}"
+diagnostic_log_file_path = "{{server_configuration.diagnostic_log_file_path}}"
+carbon_log_file_path = "{{server_configuration.carbon_log_file_path}}"
+process_id_path = "{{server_configuration.process_id_path}}"
+server_name = "@product.name@"
+server_version = "@product.version@"
+
+## Action Executor Configurations
+
+# Example
+#[[action_executor_configuration]]
+#executor = "ActionExecutor"
+#reload_time = "180" # in seconds
+
+[[action_executor_configuration]]
+executor = "MemoryDumper"
+reload_time = "180"
+
+[[action_executor_configuration]]
+executor = "ThreadDumper"
+count = "5"
+delay = "2000"
+
+[[action_executor_configuration]]
+executor = "OpenFileFinder"
+
+[[action_executor_configuration]]
+executor = "Netstat"
+command = "netstat -alt"
+
+[[action_executor_configuration]]
+executor = "ServerInfo"
+
+[[action_executor_configuration]]
+executor = "MetricsSnapshot"
+
+{%for action in action_executor_configuration %}
+[[action_executor_configuration]]
+executor = "{{action.executor}}"
+{% endfor %}
+
+# Watcher Configurations
+[cpu_watcher]
+enabled = "{{cpu_watcher.enabled}}"
+threshold = "{{cpu_watcher.threshold}}"
+retry_count = "{{cpu_watcher.retry_count}}"
+interval = "{{cpu_watcher.interval}}"
+action_executors = "{{cpu_watcher.action_executors}}"
+
+[memory_watcher]
+enabled = "{{memory_watcher.enabled}}"
+threshold = "{{memory_watcher.threshold}}"
+retry_count = "{{memory_watcher.retry_count}}"
+interval = "{{memory_watcher.interval}}"
+action_executors = "{{memory_watcher.action_executors}}"
+
+[log_watcher]
+enabled = "{{log_watcher.enabled}}"
+interval = {{log_watcher.interval}}
+
+# Traffic Analyzer Configurations
+[traffic_analyzer]
+last_second_requests_enabled = "{{traffic_analyzer.last_second_requests_enabled}}"
+last_second_requests_windows_size = "{{traffic_analyzer.last_second_requests_windows_size}}"
+last_second_requests_delay = "{{traffic_analyzer.last_second_requests_delay}}"
+last_second_requests_interval = "{{traffic_analyzer.last_second_requests_interval}}"
+last_fifteen_seconds_requests_enabled = "{{traffic_analyzer.last_fifteen_seconds_requests_enabled}}"
+last_fifteen_seconds_requests_window_size = "{{traffic_analyzer.last_fifteen_seconds_requests_window_size}}"
+last_fifteen_seconds_requests_delay = "{{traffic_analyzer.last_fifteen_seconds_requests_delay}}"
+last_fifteen_seconds_requests_interval = "{{traffic_analyzer.last_fifteen_seconds_requests_interval}}"
+last_minutes_requests_enabled = "{{traffic_analyzer.last_minutes_requests_enabled}}"
+last_minutes_requests_window_size = "{{traffic_analyzer.last_minutes_requests_window_size}}"
+last_minutes_requests_delay = "{{traffic_analyzer.last_minutes_requests_delay}}"
+last_minutes_requests_interval = "{{traffic_analyzer.last_minutes_requests_interval}}"
+notify_interval = "{{traffic_analyzer.notify_interval}}"
+
+# Output data zip configurations
+[zip_file_configuration]
+output_directory = "{{zip_file_configuration.output_directory}}"
+max_count = "{{zip_file_configuration.max_count}}"
+
+# Error regex patterns and diagnosis
+{%for pattern in log_pattern.patterns %}
+[[log_pattern]]
+regex = "{{pattern.pattern.regex}}"
+executors = "{{pattern.pattern.executors}}"
+reload_time = "{{pattern.pattern.reload_time}}"
+
+{% endfor %}
+
+{% if ftp_uploader is defined %}
+## FTP Uploader configurations
+[ftp_uploader]
+enabled = "{{ftp_uploader.enabled}}"
+host = "{{ftp_uploader.host}}"
+port = "{{ftp_uploader.port}}"
+username = "{{ftp_uploader.username}}"
+password = "{{ftp_uploader.password}}"
+directory = "{{ftp_uploader.directory}}"
+{% endif %}
+
+{% if sftp_uploader is defined %}
+## SFTP Uploader configurations
+[sftp_uploader]
+enabled = "{{sftp_uploader.enabled}}"
+host = "{{sftp_uploader.host}}"
+port = "{{sftp_uploader.port}}"
+username = "{{sftp_uploader.username}}"
+password = "{{sftp_uploader.password}}"
+directory = "{{sftp_uploader.directory}}"
+known_hosts_path = "{{sftp_uploader.known_hosts_path}}"
+strict_host_key_checking = "{{sftp_uploader.strict_host_key_checking}}"
+{% endif %}

modules/distribution/product/src/main/startup-scripts/api-manager.sh

@@ -313,6 +313,16 @@ if [ $java_version_formatted -ge 1700 ]; then
  JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED"
 fi
 
+# start diagnostic tool in background in diagnostic-tool/bin/diagnostic
+"$CARBON_HOME"/diagnostics-tool/bin/diagnostics.sh &
+diagnostic_tool_pid=$!
+
+# trap signals so we can shutdown the diagnostic tool
+cleanup() {
+ kill "$diagnostic_tool_pid"
+}
+trap 'cleanup' EXIT
+
 while [ "$status" = "$START_EXIT_STATUS" ]
 do
  $JAVACMD \