Include Runtime diagnostics tool into the server

modules/distribution/product/pom.xml

@@ -68,6 +68,10 @@
             <groupId>org.wso2.orbit.com.lmax</groupId>
             <artifactId>disruptor</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.wso2.diagnostics</groupId>
+            <artifactId>runtime-diagnostics</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -92,6 +96,15 @@
                                     <overWrite>true</overWrite>
                                     <outputDirectory>target</outputDirectory>
                                 </artifactItem>
+                                <artifactItem>
+                                    <groupId>org.wso2.diagnostics</groupId>
+                                    <artifactId>runtime-diagnostics</artifactId>
+                                    <version>${diagnostics.tool.version}</version>
+                                    <type>zip</type>
+                                    <overWrite>true</overWrite>
+                                    <outputDirectory>target</outputDirectory>
+                                    <destFileName>diagnostics-tool</destFileName>
+                                </artifactItem>
                             </artifactItems>
                         </configuration>
                     </execution>

modules/distribution/product/src/main/assembly/bin.xml

@@ -175,6 +175,26 @@
                 <exclude>**/lib/xalan*.jar</exclude>
             </excludes>
         </fileSet>
+        <fileSet>
+            <directory>target/diagnostics-tool</directory>
+            <outputDirectory>wso2am-${pom.version}/diagnostics-tool</outputDirectory>
+            <excludes>
+                <exclude>bin/diagnostics</exclude>
+                <exclude>lib/**</exclude>
+            </excludes>
+        </fileSet>
+        <fileSet>
+            <directory>target/diagnostics-tool/lib</directory>
+            <outputDirectory>wso2am-${pom.version}/lib</outputDirectory>
+            <excludes>
+                <exclude>antlr-runtime-*.jar</exclude>
+                <exclude>gson*.jar</exclude>
+                <exclude>commons-lang3-*.jar</exclude>
+                <exclude>log4j-api-*.jar</exclude>
+                <exclude>log4j-core-*.jar</exclude>
+                <exclude>cava-toml-*.jar</exclude>
+            </excludes>
+        </fileSet>
         <fileSet>
             <directory>target/wso2carbon-core-${carbon.kernel.version}</directory>
             <outputDirectory>wso2am-${pom.version}</outputDirectory>
@@ -1291,6 +1311,21 @@
             <fileMode>755</fileMode>
         </file>
 
+        <file>
+            <source>src/main/startup-scripts/diagnostics.sh</source>
+            <outputDirectory>wso2am-${pom.version}/diagnostics-tool/bin/</outputDirectory>
+            <filtered>true</filtered>
+            <fileMode>755</fileMode>
+        </file>
+
+        <file>
+            <source>src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2</source>
+            <outputDirectory>wso2am-${pom.version}/repository/resources/conf/templates/diagnostics-tool/conf</outputDirectory>
+            <destName>config.toml.j2</destName>
+            <filtered>true</filtered>
+            <fileMode>644</fileMode>
+        </file>
+
         <file>
             <source>../../../config/bam.xml</source>
             <outputDirectory>wso2am-${pom.version}/repository/conf/etc</outputDirectory>

modules/distribution/product/src/main/conf/log4j2.properties

@@ -118,7 +118,7 @@ appender.ERROR_LOGFILE.name = ERROR_LOGFILE
 appender.ERROR_LOGFILE.fileName = ${sys:carbon.home}/repository/logs/wso2-apigw-errors.log
 appender.ERROR_LOGFILE.filePattern = ${sys:carbon.home}/repository/logs/wso2-apigw-errors-%d{MM-dd-yyyy}-%i.log.gz
 appender.ERROR_LOGFILE.layout.type = PatternLayout
-appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p %c{1} %m%n
+appender.ERROR_LOGFILE.layout.pattern = %d{ISO8601} [%X{ip}-%X{host}] [%t] %5p {%c} %m%n
 appender.ERROR_LOGFILE.policies.type = Policies
 appender.ERROR_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy
 appender.ERROR_LOGFILE.policies.time.interval = 1

modules/distribution/product/src/main/resources/conf/default.json

@@ -471,5 +471,64 @@
   "apim.analytics.properties.truststore_location": "${carbon.home}/repository/resources/security/$ref{truststore.file_name}",
   "apim.analytics.properties.truststore_password": "$ref{truststore.password}",
   "tenant_mgt.disable_email_domain_validation": true,
-  "apim.jwt.use_kid_property": true
+  "apim.jwt.use_kid_property": true,
+  "server_configuration": {
+    "deployment_toml_path": "../conf/deployment.toml",
+    "logs_directory": "../repository/logs",
+    "updates_config_path": "../updates/config.json",
+    "diagnostic_log_file_path": "logs/diagnostics.log",
+    "carbon_log_file_path": "../repository/logs/wso2-apigw-errors.log",
+    "process_id_path": "../wso2carbon.pid",
+    "server_name": "WSO2 API Manager",
+    "server_version": "4.3.0"
+  },
+  "cpu_watcher": {
+    "enabled": "true",
+    "threshold": "80",
+    "retry_count": "2",
+    "interval": "5",
+    "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+  },
+  "memory_watcher": {
+    "enabled": "true",
+    "threshold": "80",
+    "retry_count": "2",
+    "interval": "5",
+    "action_executors": "ThreadDumper,MetricsSnapshot,ServerInfo"
+  },
+  "log_watcher": {
+    "enabled": "true",
+    "interval": 0.1
+  },
+  "traffic_analyzer": {
+    "last_second_requests_enabled": "false",
+    "last_second_requests_windows_size": "300",
+    "last_second_requests_delay": "60",
+    "last_second_requests_interval": "1",
+    "last_fifteen_seconds_requests_enabled": "true",
+    "last_fifteen_seconds_requests_window_size": "100",
+    "last_fifteen_seconds_requests_delay": "4",
+    "last_fifteen_seconds_requests_interval": "15",
+    "last_minutes_requests_enabled": "true",
+    "last_minutes_requests_window_size": "100",
+    "last_minutes_requests_delay": "1",
+    "last_minutes_requests_interval": "60",
+    "notify_interval": "60"
+  },
+  "zip_file_configuration": {
+    "output_directory": "data",
+    "max_count": "20"
+  },
+  "log_pattern.patterns": [
+    {
+      "pattern.regex": "(.*)org.apache.synapse.transport.passthru(.*)",
+      "pattern.executors": "MetricsSnapshot,Netstat,OpenFileFinder,ThreadDumper,ServerInfo",
+      "pattern.reload_time": "30"
+    },
+    {
+      "pattern.regex": "(.*)org.apache.synapse(.*)",
+      "pattern.executors": "ServerInfo",
+      "pattern.reload_time": "10"
+    }
+  ]
 }

modules/distribution/product/src/main/resources/conf/templates/diagnostics-tool/conf/config.toml.j2

@@ -0,0 +1,143 @@
+#
+# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+##############################################
+#
+# WARNING: Don't edit the file manually unless you are not using the deployment.toml file.
+#
+##############################################
+
+## This file contains the configuration parameters used by the Diagnostic tool
+
+# Server Configurations
+[server_configuration]
+deployment_toml_path = "{{server_configuration.deployment_toml_path}}"
+logs_directory = "{{server_configuration.logs_directory}}"
+updates_config_path = "{{server_configuration.updates_config_path}}"
+diagnostic_log_file_path = "{{server_configuration.diagnostic_log_file_path}}"
+carbon_log_file_path = "{{server_configuration.carbon_log_file_path}}"
+process_id_path = "{{server_configuration.process_id_path}}"
+server_name = "@product.name@"
+server_version = "@product.version@"
+
+## Action Executor Configurations
+
+# Example
+#[[action_executor_configuration]]
+#executor = "ActionExecutor"
+#reload_time = "180" # in seconds
+
+[[action_executor_configuration]]
+executor = "MemoryDumper"
+reload_time = "180"
+
+[[action_executor_configuration]]
+executor = "ThreadDumper"
+count = "5"
+delay = "2000"
+
+[[action_executor_configuration]]
+executor = "OpenFileFinder"
+
+[[action_executor_configuration]]
+executor = "Netstat"
+command = "netstat -alt"
+
+[[action_executor_configuration]]
+executor = "ServerInfo"
+
+[[action_executor_configuration]]
+executor = "MetricsSnapshot"
+
+{%for action in action_executor_configuration %}
+[[action_executor_configuration]]
+executor = "{{action.executor}}"
+{% endfor %}
+
+# Watcher Configurations
+[cpu_watcher]
+enabled = "{{cpu_watcher.enabled}}"
+threshold = "{{cpu_watcher.threshold}}"
+retry_count = "{{cpu_watcher.retry_count}}"
+interval = "{{cpu_watcher.interval}}"
+action_executors = "{{cpu_watcher.action_executors}}"
+
+[memory_watcher]
+enabled = "{{memory_watcher.enabled}}"
+threshold = "{{memory_watcher.threshold}}"
+retry_count = "{{memory_watcher.retry_count}}"
+interval = "{{memory_watcher.interval}}"
+action_executors = "{{memory_watcher.action_executors}}"
+
+[log_watcher]
+enabled = "{{log_watcher.enabled}}"
+interval = {{log_watcher.interval}}
+
+# Traffic Analyzer Configurations
+[traffic_analyzer]
+last_second_requests_enabled = "{{traffic_analyzer.last_second_requests_enabled}}"
+last_second_requests_windows_size = "{{traffic_analyzer.last_second_requests_windows_size}}"
+last_second_requests_delay = "{{traffic_analyzer.last_second_requests_delay}}"
+last_second_requests_interval = "{{traffic_analyzer.last_second_requests_interval}}"
+last_fifteen_seconds_requests_enabled = "{{traffic_analyzer.last_fifteen_seconds_requests_enabled}}"
+last_fifteen_seconds_requests_window_size = "{{traffic_analyzer.last_fifteen_seconds_requests_window_size}}"
+last_fifteen_seconds_requests_delay = "{{traffic_analyzer.last_fifteen_seconds_requests_delay}}"
+last_fifteen_seconds_requests_interval = "{{traffic_analyzer.last_fifteen_seconds_requests_interval}}"
+last_minutes_requests_enabled = "{{traffic_analyzer.last_minutes_requests_enabled}}"
+last_minutes_requests_window_size = "{{traffic_analyzer.last_minutes_requests_window_size}}"
+last_minutes_requests_delay = "{{traffic_analyzer.last_minutes_requests_delay}}"
+last_minutes_requests_interval = "{{traffic_analyzer.last_minutes_requests_interval}}"
+notify_interval = "{{traffic_analyzer.notify_interval}}"
+
+# Output data zip configurations
+[zip_file_configuration]
+output_directory = "{{zip_file_configuration.output_directory}}"
+max_count = "{{zip_file_configuration.max_count}}"
+
+# Error regex patterns and diagnosis
+{%for pattern in log_pattern.patterns %}
+[[log_pattern]]
+regex = "{{pattern.pattern.regex}}"
+executors = "{{pattern.pattern.executors}}"
+reload_time = "{{pattern.pattern.reload_time}}"
+
+{% endfor %}
+
+{% if ftp_uploader is defined %}
+## FTP Uploader configurations
+[ftp_uploader]
+enabled = "{{ftp_uploader.enabled}}"
+host = "{{ftp_uploader.host}}"
+port = "{{ftp_uploader.port}}"
+username = "{{ftp_uploader.username}}"
+password = "{{ftp_uploader.password}}"
+directory = "{{ftp_uploader.directory}}"
+{% endif %}
+
+{% if sftp_uploader is defined %}
+## SFTP Uploader configurations
+[sftp_uploader]
+enabled = "{{sftp_uploader.enabled}}"
+host = "{{sftp_uploader.host}}"
+port = "{{sftp_uploader.port}}"
+username = "{{sftp_uploader.username}}"
+password = "{{sftp_uploader.password}}"
+directory = "{{sftp_uploader.directory}}"
+known_hosts_path = "{{sftp_uploader.known_hosts_path}}"
+strict_host_key_checking = "{{sftp_uploader.strict_host_key_checking}}"
+{% endif %}

modules/distribution/product/src/main/startup-scripts/api-manager.sh

@@ -313,6 +313,16 @@ if [ $java_version_formatted -ge 1700 ]; then
     JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED"
 fi
 
+# start diagnostic tool in background in diagnostic-tool/bin/diagnostic
+"$CARBON_HOME"/diagnostics-tool/bin/diagnostics.sh &
+diagnostic_tool_pid=$!
+
+# trap signals so we can shutdown the diagnostic tool
+cleanup() {
+    kill "$diagnostic_tool_pid"
+}
+trap 'cleanup' EXIT
+
 while [ "$status" = "$START_EXIT_STATUS" ]
 do
     $JAVACMD \