Implement AccessLog support for Ballerina

ballerina/Ballerina.toml

@@ -1,7 +1,7 @@
 [package]
 org = "ballerina"
 name = "http"
-version = "2.11.2"
+version = "2.11.3"
 authors = ["Ballerina"]
 keywords = ["http", "network", "service", "listener", "client"]
 repository = "https://github.com/ballerina-platform/module-ballerina-http"
@@ -16,8 +16,8 @@ graalvmCompatible = true
 [[platform.java17.dependency]]
 groupId = "io.ballerina.stdlib"
 artifactId = "http-native"
-version = "2.11.2"
-path = "../native/build/libs/http-native-2.11.2.jar"
+version = "2.11.3"
+path = "../native/build/libs/http-native-2.11.3-SNAPSHOT.jar"
 
 [[platform.java17.dependency]]
 groupId = "io.ballerina.stdlib"

ballerina/CompilerPlugin.toml

@@ -3,4 +3,4 @@ id = "http-compiler-plugin"
 class = "io.ballerina.stdlib.http.compiler.HttpCompilerPlugin"
 
 [[dependency]]
-path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.2.jar"
+path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.3-SNAPSHOT.jar"

ballerina/Dependencies.toml

@@ -50,7 +50,7 @@ modules = [
 [[package]]
 org = "ballerina"
 name = "crypto"
-version = "2.7.1"
+version = "2.7.0"
 dependencies = [
  {org = "ballerina", name = "jballerina.java"},
  {org = "ballerina", name = "time"}
@@ -76,7 +76,7 @@ modules = [
 [[package]]
 org = "ballerina"
 name = "http"
-version = "2.11.2"
+version = "2.11.3"
 dependencies = [
  {org = "ballerina", name = "auth"},
  {org = "ballerina", name = "cache"},

ballerina/http_log_manager.bal

@@ -35,9 +35,13 @@ public type TraceLogAdvancedConfiguration record {|
 # Represents HTTP access log configuration.
 #
 # + console - Boolean value to enable or disable console access logs
+# + format - The format of access logs to be printed (either `flat` or `json`)
+# + attributes - The list of attributes of access logs to be printed
 # + path - Optional file path to store access logs
 public type AccessLogConfiguration record {|
  boolean console = false;
+ string format = "flat";
+ string[] attributes?;
  string path?;
 |};
 

changelog.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 - [Introduce default status code response record](https://github.com/ballerina-platform/ballerina-library/issues/6491)
+- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options. The update support both JSON and flat logging formats](https://github.com/ballerina-platform/ballerina-library/issues/6111)
 
 ## [2.11.2] - 2024-06-14
 

docs/spec/spec.md

@@ -2498,20 +2498,47 @@ path = "testTraceLog.txt" # Optional
 host = "localhost" # Optional
 port = 8080 # Optional
 ```
+
 #### 8.2.4 Access log
+Ballerina supports HTTP access logs for HTTP services, providing insights into web traffic and request handling.
+The access log feature is **disabled by default** to allow users to opt-in as per their requirements.
 
-Ballerina supports HTTP access logs for HTTP services. The access log format used is the combined log format.
-The HTTP access logs are **disabled as default**.
-To enable access logs, set console=true under the ballerina.http.accessLogConfig in the Config.toml file. Also, 
-the path field can be used to specify the file path to save the access logs.
+To enable access logs, configuration settings are provided under `ballerina.http.accessLogConfig` in the
+`Config.toml` file. Users can specify whether logs should be output to the console, a file, or both, 
+and can select the format and specific attributes to log.
 
 ```toml
 [ballerina.http.accessLogConfig]
 # Enable printing access logs in console
 console = true # Default is false
-# Specify the file path to save the access logs 
-path = "testAccessLog.txt" # Optional
-```
+# Specify the file path to save the access logs
+path = "testAccessLog.txt" # Optional, omit to disable file logging
+# Select the format of the access logs
+format = "json" # Options: "flat", "json"; Default is "flat". Omit to stick to the default.
+# Specify which attributes to log. Omit to stick to the default set.
+attributes = ["ip", "date_time", "request", "status", "response_body_size", "http_referrer", "http_user_agent"]
+# Default attributes: ip, date_time, request, status, response_body_size, http_referrer, http_user_agent
+```
+##### Configurable Attributes
+Users can customize which parts of the access data are logged by specifying attributes in the configuration.
+This allows for tailored logging that can focus on particular details relevant to the users' needs.
+
+| Attribute | Description |
+|:----------------------:|:---------------------------------------------------:|
+| ip | Client's IP address |
+| date_time | HTTP request received time |
+| request | Full HTTP request line (method, URI, protocol) |
+| request_method | HTTP method of the request |
+| request_uri | URI of the request, including parameters |
+| scheme | Scheme of the request and HTTP version |
+| status | HTTP status code returned to the client |
+| request_body_size | Size of the request body in bytes |
+| response_body_size | Size of the HTTP response body in bytes |
+| request_time | Total time taken to process the request |
+| http_referrer | HTTP Referer header, indicating the previous page |
+| http_user_agent | User-Agent header, identifying the client software |
+| http_x_forwarded_for | Originating IP address if using a proxy |
+| http_(X-Custom-Header) | Header fields. Referring to them with `http` followed by the header name. (`x-request-id` ->; `http_x-request-id`) |
 
 #### 8.2.5 Panic inside resource
 

native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java

@@ -143,7 +143,7 @@ public boolean isHTTPTraceLoggerEnabled() {
  return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_TRACE_LOG_ENABLED));
  }
 
- private boolean isHTTPAccessLoggerEnabled() {
+ public boolean isHTTPAccessLoggerEnabled() {
  return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_ACCESS_LOG_ENABLED));
  }
 

native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java

@@ -325,14 +325,32 @@ public final class HttpConstants {
  public static final String HTTP_TRACE_LOG_ENABLED = "http.tracelog.enabled";
  public static final String HTTP_ACCESS_LOG = "http.accesslog";
  public static final String HTTP_ACCESS_LOG_ENABLED = "http.accesslog.enabled";
+ public static final String HTTP_LOG_FORMAT_JSON = "json";
 
  // TraceLog and AccessLog configs
  public static final BString HTTP_LOG_CONSOLE = StringUtils.fromString("console");
+ public static final BString HTTP_LOG_FORMAT = StringUtils.fromString("format");
+ public static final BString HTTP_LOG_ATTRIBUTES = StringUtils.fromString("attributes");
  public static final BString HTTP_LOG_FILE_PATH = StringUtils.fromString("path");
  public static final BString HTTP_TRACE_LOG_HOST = StringUtils.fromString("host");
  public static final BString HTTP_TRACE_LOG_PORT = StringUtils.fromString("port");
  public static final BString HTTP_LOGGING_PROTOCOL = StringUtils.fromString("HTTP");
 
+ // AccessLog fiend names
+ public static final String ATTRIBUTE_IP = "ip";
+ public static final String ATTRIBUTE_DATE_TIME = "date_time";
+ public static final String ATTRIBUTE_REQUEST_METHOD = "request_method";
+ public static final String ATTRIBUTE_REQUEST_URI = "request_uri";
+ public static final String ATTRIBUTE_SCHEME = "scheme";
+ public static final String ATTRIBUTE_REQUEST = "request";
+ public static final String ATTRIBUTE_STATUS = "status";
+ public static final String ATTRIBUTE_REQUEST_BODY_SIZE = "request_body_size";
+ public static final String ATTRIBUTE_RESPONSE_BODY_SIZE = "response_body_size";
+ public static final String ATTRIBUTE_REQUEST_TIME = "request_time";
+ public static final String ATTRIBUTE_HTTP_REFERRER = "http_referrer";
+ public static final String ATTRIBUTE_HTTP_USER_AGENT = "http_user_agent";
+ public static final String ATTRIBUTE_HTTP_X_FORWARDED_FOR = "http_x_forwarded_for";
+
  // ResponseCacheControl struct field names
  public static final BString RES_CACHE_CONTROL_MUST_REVALIDATE_FIELD = StringUtils.fromString("mustRevalidate");
  public static final BString RES_CACHE_CONTROL_NO_CACHE_FIELD = StringUtils.fromString("noCache");

native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java

@@ -290,6 +290,8 @@ protected static void executeNonBlockingAction(DataContext dataContext, boolean
  }
  outboundRequestMsg.setProperty(HttpConstants.ORIGIN_HOST,
  dataContext.getEnvironment().getStrandLocal(HttpConstants.ORIGIN_HOST));
+ outboundRequestMsg.setProperty(HttpConstants.INBOUND_MESSAGE,
+ dataContext.getEnvironment().getStrandLocal(HttpConstants.INBOUND_MESSAGE));
  sendOutboundRequest(dataContext, outboundRequestMsg, async);
  }
 

native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java

@@ -52,6 +52,7 @@
 import static io.ballerina.stdlib.http.api.HttpConstants.CURRENT_TRANSACTION_CONTEXT_PROPERTY;
 import static io.ballerina.stdlib.http.api.HttpConstants.EMPTY;
 import static io.ballerina.stdlib.http.api.HttpConstants.EQUAL_SIGN;
+import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE;
 import static io.ballerina.stdlib.http.api.HttpConstants.MAIN_STRAND;
 import static io.ballerina.stdlib.http.api.HttpConstants.ORIGIN_HOST;
 import static io.ballerina.stdlib.http.api.HttpConstants.POOLED_BYTE_BUFFER_FACTORY;
@@ -242,7 +243,7 @@ public void notifyFailure(BError bError) {
 
  private static Map<String, Object> getPropertiesToPropagate(Environment env) {
  String[] keys = {CURRENT_TRANSACTION_CONTEXT_PROPERTY, KEY_OBSERVER_CONTEXT, SRC_HANDLER, MAIN_STRAND,
- POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST};
+ POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST, INBOUND_MESSAGE};
  Map<String, Object> subMap = new HashMap<>();
  for (String key : keys) {
  Object value = env.getStrandLocal(key);

native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java

@@ -93,6 +93,9 @@ public static Object createSimpleHttpClient(BObject httpClient, BMap globalPoolC
  if (connectionManager.isHTTPTraceLoggerEnabled()) {
  senderConfiguration.setHttpTraceLogEnabled(true);
  }
+ if (connectionManager.isHTTPAccessLoggerEnabled()) {
+ senderConfiguration.setHttpAccessLogEnabled(true);
+ }
  senderConfiguration.setTLSStoreType(HttpConstants.PKCS_STORE_TYPE);
 
  String httpVersion = clientEndpointConfig.getStringValue(HttpConstants.CLIENT_EP_HTTP_VERSION).getValue();

native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java

@@ -20,6 +20,7 @@
 
 import io.ballerina.runtime.api.values.BMap;
 import io.ballerina.runtime.api.values.BString;
+import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig;
 import io.ballerina.stdlib.http.api.logging.formatters.HttpAccessLogFormatter;
 import io.ballerina.stdlib.http.api.logging.formatters.HttpTraceLogFormatter;
 import io.ballerina.stdlib.http.api.logging.formatters.JsonLogFormatter;
@@ -70,6 +71,7 @@ public HttpLogManager(boolean traceLogConsole, BMap traceLogAdvancedConfig, BMap
  this.protocol = protocol.getValue();
  this.setHttpTraceLogHandler(traceLogConsole, traceLogAdvancedConfig);
  this.setHttpAccessLogHandler(accessLogConfig);
+ HttpAccessLogConfig.getInstance().initializeHttpAccessLogConfig(accessLogConfig);
  }
 
  /**
@@ -166,5 +168,4 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) {
  stdErr.println("ballerina: " + protocol + " access log enabled");
  }
  }
-
 }

native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java

@@ -0,0 +1,71 @@
+package io.ballerina.stdlib.http.api.logging.accesslog;
+
+import io.ballerina.runtime.api.values.BArray;
+import io.ballerina.runtime.api.values.BMap;
+import io.ballerina.runtime.api.values.BString;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER;
+import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT;
+import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR;
+import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES;
+import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT;
+import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON;
+
+public class HttpAccessLogConfig {
+
+ private static final HttpAccessLogConfig instance = new HttpAccessLogConfig();
+ private static final Set<String> EXCLUDED_ATTRIBUTES = new HashSet<>(List.of(
+ ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR
+ ));
+ private BMap accessLogConfig;
+
+ private HttpAccessLogConfig() {}
+
+ public static HttpAccessLogConfig getInstance() {
+ return instance;
+ }
+
+ public void initializeHttpAccessLogConfig(BMap accessLogConfig) {
+ this.accessLogConfig = accessLogConfig;
+ }
+
+ public List<String> getCustomHeaders() {
+ List<String> attributes = getAccessLogAttributes();
+ if (attributes == null) {
+ return Collections.emptyList();
+ }
+
+ return attributes.stream()
+ .filter(attr -> attr.startsWith("http_") && !EXCLUDED_ATTRIBUTES.contains(attr))
+ .map(attr -> attr.substring(5))
+ .collect(Collectors.toList());
+ }
+
+ public HttpAccessLogFormat getAccessLogFormat() {
+ if (accessLogConfig != null) {
+ BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT);
+ if (logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON)) {
+ return HttpAccessLogFormat.JSON;
+ }
+ }
+ return HttpAccessLogFormat.FLAT;
+ }
+
+ public List<String> getAccessLogAttributes() {
+ if (accessLogConfig != null) {
+ BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES);
+ if (logAttributes != null) {
+ return Arrays.stream(logAttributes.getStringArray())
+ .collect(Collectors.toList());
+ }
+ }
+ return null;
+ }
+}

native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+ *
+ * WSO2 LLC. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.ballerina.stdlib.http.api.logging.accesslog;
+
+public enum HttpAccessLogFormat {
+ FLAT, JSON
+}