Skip to content

Commit

Permalink
Merge pull request #1969 from AzeemMuzammil/fb-access-log-improvement
Browse files Browse the repository at this point in the history
Implement AccessLog support for Ballerina
  • Loading branch information
TharmiganK authored Jul 15, 2024
2 parents 99b2877 + 4d8e9b6 commit bab94cd
Show file tree
Hide file tree
Showing 39 changed files with 1,102 additions and 107 deletions.
3 changes: 3 additions & 0 deletions ballerina-tests/http-dispatching-tests/tests/Config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[ballerina.http.accessLogConfig]
console = true
format = "json"
6 changes: 3 additions & 3 deletions ballerina/Ballerina.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
org = "ballerina"
name = "http"
version = "2.11.2"
version = "2.11.3"
authors = ["Ballerina"]
keywords = ["http", "network", "service", "listener", "client"]
repository = "https://github.com/ballerina-platform/module-ballerina-http"
Expand All @@ -16,8 +16,8 @@ graalvmCompatible = true
[[platform.java17.dependency]]
groupId = "io.ballerina.stdlib"
artifactId = "http-native"
version = "2.11.2"
path = "../native/build/libs/http-native-2.11.2.jar"
version = "2.11.3"
path = "../native/build/libs/http-native-2.11.3-SNAPSHOT.jar"

[[platform.java17.dependency]]
groupId = "io.ballerina.stdlib"
Expand Down
2 changes: 1 addition & 1 deletion ballerina/CompilerPlugin.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ id = "http-compiler-plugin"
class = "io.ballerina.stdlib.http.compiler.HttpCompilerPlugin"

[[dependency]]
path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.2.jar"
path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.3-SNAPSHOT.jar"
4 changes: 2 additions & 2 deletions ballerina/Dependencies.toml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ modules = [
[[package]]
org = "ballerina"
name = "crypto"
version = "2.7.1"
version = "2.7.2"
dependencies = [
{org = "ballerina", name = "jballerina.java"},
{org = "ballerina", name = "time"}
Expand All @@ -76,7 +76,7 @@ modules = [
[[package]]
org = "ballerina"
name = "http"
version = "2.11.2"
version = "2.11.3"
dependencies = [
{org = "ballerina", name = "auth"},
{org = "ballerina", name = "cache"},
Expand Down
4 changes: 4 additions & 0 deletions ballerina/http_log_manager.bal
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ public type TraceLogAdvancedConfiguration record {|
# Represents HTTP access log configuration.
#
# + console - Boolean value to enable or disable console access logs
# + format - The format of access logs to be printed (either `flat` or `json`)
# + attributes - The list of attributes of access logs to be printed
# + path - Optional file path to store access logs
public type AccessLogConfiguration record {|
boolean console = false;
string format = "flat";
string[] attributes?;
string path?;
|};

Expand Down
1 change: 1 addition & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added

- [Introduce default status code response record](https://github.com/ballerina-platform/ballerina-library/issues/6491)
- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options.](https://github.com/ballerina-platform/ballerina-library/issues/6111)

## [2.11.2] - 2024-06-14

Expand Down
42 changes: 35 additions & 7 deletions docs/spec/spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -2498,20 +2498,48 @@ path = "testTraceLog.txt" # Optional
host = "localhost" # Optional
port = 8080 # Optional
```

#### 8.2.4 Access log
Ballerina supports HTTP access logs for HTTP services, providing insights into web traffic and request handling.
The access log feature is **disabled by default** to allow users to opt-in as per their requirements.

Ballerina supports HTTP access logs for HTTP services. The access log format used is the combined log format.
The HTTP access logs are **disabled as default**.
To enable access logs, set console=true under the ballerina.http.accessLogConfig in the Config.toml file. Also,
the path field can be used to specify the file path to save the access logs.
To enable access logs, configuration settings are provided under `ballerina.http.accessLogConfig` in the
`Config.toml` file. Users can specify whether logs should be output to the console, a file, or both,
and can select the format and specific attributes to log.

```toml
[ballerina.http.accessLogConfig]
# Enable printing access logs in console
console = true # Default is false
# Specify the file path to save the access logs
path = "testAccessLog.txt" # Optional
```
# Specify the file path to save the access logs
path = "testAccessLog.txt" # Optional, omit to disable file logging
# Select the format of the access logs
format = "json" # Options: "flat", "json"; Default is "flat". Omit to stick to the default.
# Specify which attributes to log. Omit to stick to the default set.
attributes = ["ip", "date_time", "request", "status", "response_body_size", "http_referrer", "http_user_agent"]
# Default attributes: ip, date_time, request, status, response_body_size, http_referrer, http_user_agent
```

##### Configurable Attributes
Users can customize which parts of the access data are logged by specifying attributes in the configuration.
This allows for tailored logging that can focus on particular details relevant to the users' needs.

| Attribute | Description |
|:----------------------:|:---------------------------------------------------:|
| ip | Client's IP address |
| date_time | HTTP request received time |
| request | Full HTTP request line (method, URI, protocol) |
| request_method | HTTP method of the request |
| request_uri | URI of the request, including parameters |
| scheme | Scheme of the request and HTTP version |
| status | HTTP status code returned to the client |
| request_body_size | Size of the request body in bytes |
| response_body_size | Size of the HTTP response body in bytes |
| request_time | Total time taken to process the request |
| http_referrer | HTTP Referer header, indicating the previous page |
| http_user_agent | User-Agent header, identifying the client software |
| http_x_forwarded_for | Originating IP address if using a proxy |
| http_(X-Custom-Header) | Header fields. Referring to them with `http` followed by the header name. (`x-request-id` ->; `http_x-request-id`) |

#### 8.2.5 Panic inside resource

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public boolean isHTTPTraceLoggerEnabled() {
return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_TRACE_LOG_ENABLED));
}

private boolean isHTTPAccessLoggerEnabled() {
public boolean isHTTPAccessLoggerEnabled() {
return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_ACCESS_LOG_ENABLED));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,33 @@ public final class HttpConstants {
public static final String HTTP_TRACE_LOG_ENABLED = "http.tracelog.enabled";
public static final String HTTP_ACCESS_LOG = "http.accesslog";
public static final String HTTP_ACCESS_LOG_ENABLED = "http.accesslog.enabled";
public static final String HTTP_LOG_FORMAT_JSON = "json";
public static final String HTTP_LOG_FORMAT_FLAT = "flat";

// TraceLog and AccessLog configs
public static final BString HTTP_LOG_CONSOLE = StringUtils.fromString("console");
public static final BString HTTP_LOG_FORMAT = StringUtils.fromString("format");
public static final BString HTTP_LOG_ATTRIBUTES = StringUtils.fromString("attributes");
public static final BString HTTP_LOG_FILE_PATH = StringUtils.fromString("path");
public static final BString HTTP_TRACE_LOG_HOST = StringUtils.fromString("host");
public static final BString HTTP_TRACE_LOG_PORT = StringUtils.fromString("port");
public static final BString HTTP_LOGGING_PROTOCOL = StringUtils.fromString("HTTP");

// AccessLog fiend names
public static final String ATTRIBUTE_IP = "ip";
public static final String ATTRIBUTE_DATE_TIME = "date_time";
public static final String ATTRIBUTE_REQUEST_METHOD = "request_method";
public static final String ATTRIBUTE_REQUEST_URI = "request_uri";
public static final String ATTRIBUTE_SCHEME = "scheme";
public static final String ATTRIBUTE_REQUEST = "request";
public static final String ATTRIBUTE_STATUS = "status";
public static final String ATTRIBUTE_REQUEST_BODY_SIZE = "request_body_size";
public static final String ATTRIBUTE_RESPONSE_BODY_SIZE = "response_body_size";
public static final String ATTRIBUTE_REQUEST_TIME = "request_time";
public static final String ATTRIBUTE_HTTP_REFERRER = "http_referrer";
public static final String ATTRIBUTE_HTTP_USER_AGENT = "http_user_agent";
public static final String ATTRIBUTE_HTTP_X_FORWARDED_FOR = "http_x_forwarded_for";

// ResponseCacheControl struct field names
public static final BString RES_CACHE_CONTROL_MUST_REVALIDATE_FIELD = StringUtils.fromString("mustRevalidate");
public static final BString RES_CACHE_CONTROL_NO_CACHE_FIELD = StringUtils.fromString("noCache");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,8 @@ protected static void executeNonBlockingAction(DataContext dataContext, boolean
}
outboundRequestMsg.setProperty(HttpConstants.ORIGIN_HOST,
dataContext.getEnvironment().getStrandLocal(HttpConstants.ORIGIN_HOST));
outboundRequestMsg.setProperty(HttpConstants.INBOUND_MESSAGE,
dataContext.getEnvironment().getStrandLocal(HttpConstants.INBOUND_MESSAGE));
sendOutboundRequest(dataContext, outboundRequestMsg, async);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import static io.ballerina.stdlib.http.api.HttpConstants.CURRENT_TRANSACTION_CONTEXT_PROPERTY;
import static io.ballerina.stdlib.http.api.HttpConstants.EMPTY;
import static io.ballerina.stdlib.http.api.HttpConstants.EQUAL_SIGN;
import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE;
import static io.ballerina.stdlib.http.api.HttpConstants.MAIN_STRAND;
import static io.ballerina.stdlib.http.api.HttpConstants.ORIGIN_HOST;
import static io.ballerina.stdlib.http.api.HttpConstants.POOLED_BYTE_BUFFER_FACTORY;
Expand Down Expand Up @@ -242,7 +243,7 @@ public void notifyFailure(BError bError) {

private static Map<String, Object> getPropertiesToPropagate(Environment env) {
String[] keys = {CURRENT_TRANSACTION_CONTEXT_PROPERTY, KEY_OBSERVER_CONTEXT, SRC_HANDLER, MAIN_STRAND,
POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST};
POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST, INBOUND_MESSAGE};
Map<String, Object> subMap = new HashMap<>();
for (String key : keys) {
Object value = env.getStrandLocal(key);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ public static Object createSimpleHttpClient(BObject httpClient, BMap globalPoolC
if (connectionManager.isHTTPTraceLoggerEnabled()) {
senderConfiguration.setHttpTraceLogEnabled(true);
}
if (connectionManager.isHTTPAccessLoggerEnabled()) {
senderConfiguration.setHttpAccessLogEnabled(true);
}
senderConfiguration.setTLSStoreType(HttpConstants.PKCS_STORE_TYPE);

String httpVersion = clientEndpointConfig.getStringValue(HttpConstants.CLIENT_EP_HTTP_VERSION).getValue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@

package io.ballerina.stdlib.http.api.logging;

import io.ballerina.runtime.api.values.BArray;
import io.ballerina.runtime.api.values.BMap;
import io.ballerina.runtime.api.values.BString;
import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig;
import io.ballerina.stdlib.http.api.logging.formatters.HttpAccessLogFormatter;
import io.ballerina.stdlib.http.api.logging.formatters.HttpTraceLogFormatter;
import io.ballerina.stdlib.http.api.logging.formatters.JsonLogFormatter;
Expand All @@ -36,8 +38,12 @@

import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG_ENABLED;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_CONSOLE;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FILE_PATH;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_FLAT;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_ENABLED;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_HOST;
Expand Down Expand Up @@ -70,6 +76,7 @@ public HttpLogManager(boolean traceLogConsole, BMap traceLogAdvancedConfig, BMap
this.protocol = protocol.getValue();
this.setHttpTraceLogHandler(traceLogConsole, traceLogAdvancedConfig);
this.setHttpAccessLogHandler(accessLogConfig);
HttpAccessLogConfig.getInstance().initializeHttpAccessLogConfig(accessLogConfig);
}

/**
Expand Down Expand Up @@ -161,10 +168,22 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) {
}
}

BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT);
if (logFormat != null &&
!(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) ||
logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) {
stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() +
"'. Defaulting to 'flat' format.");
}

BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES);
if (logAttributes != null && logAttributes.getLength() == 0) {
accessLogsEnabled = false;
}

if (accessLogsEnabled) {
System.setProperty(HTTP_ACCESS_LOG_ENABLED, "true");
stdErr.println("ballerina: " + protocol + " access log enabled");
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org).
*
* WSO2 LLC. licenses this file to you under the Apache License,
* Version 2.0 (the "License"); you may not use this file except
* in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package io.ballerina.stdlib.http.api.logging.accesslog;

import io.ballerina.runtime.api.values.BArray;
import io.ballerina.runtime.api.values.BMap;
import io.ballerina.runtime.api.values.BString;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER;
import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT;
import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT;
import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON;

/**
* Provides a singleton configuration manager for HTTP access logging within the application.
* This class manages settings related to log formats, attributes, and custom header.
*
* @since 2.12.0
*/
public class HttpAccessLogConfig {

private static final HttpAccessLogConfig instance = new HttpAccessLogConfig();

private final Set<String> excludedAttributes = new HashSet<>(List.of(
ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR
));
private BMap accessLogConfig;

private HttpAccessLogConfig() {}

public static HttpAccessLogConfig getInstance() {
return instance;
}

public void initializeHttpAccessLogConfig(BMap accessLogConfig) {
this.accessLogConfig = accessLogConfig;
}

public List<String> getCustomHeaders() {
List<String> attributes = getAccessLogAttributes();

return attributes.stream()
.filter(attr -> attr.startsWith("http_") && !excludedAttributes.contains(attr))
.map(attr -> attr.substring(5))
.collect(Collectors.toList());
}

public HttpAccessLogFormat getAccessLogFormat() {
if (accessLogConfig != null) {
BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT);
if (logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON)) {
return HttpAccessLogFormat.JSON;
}
}
return HttpAccessLogFormat.FLAT;
}

public List<String> getAccessLogAttributes() {
if (accessLogConfig != null) {
BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES);
if (logAttributes != null) {
return Arrays.stream(logAttributes.getStringArray())
.collect(Collectors.toList());
}
}
return Collections.emptyList();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org).
*
* WSO2 LLC. licenses this file to you under the Apache License,
* Version 2.0 (the "License"); you may not use this file except
* in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package io.ballerina.stdlib.http.api.logging.accesslog;

/**
* Represents the format types available for HTTP access logging.
* This enum is used to specify the preferred logging format for HTTP access logs.
*
* @since 2.12.0
*/
public enum HttpAccessLogFormat {
FLAT, JSON
}
Loading

0 comments on commit bab94cd

Please sign in to comment.