diff --git a/ballerina-tests/http-dispatching-tests/tests/Config.toml b/ballerina-tests/http-dispatching-tests/tests/Config.toml new file mode 100644 index 0000000000..77330aad47 --- /dev/null +++ b/ballerina-tests/http-dispatching-tests/tests/Config.toml @@ -0,0 +1,3 @@ +[ballerina.http.accessLogConfig] +console = true +format = "json" diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 72331b2559..7570a49bbf 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "http" -version = "2.11.2" +version = "2.11.3" authors = ["Ballerina"] keywords = ["http", "network", "service", "listener", "client"] repository = "https://github.com/ballerina-platform/module-ballerina-http" @@ -16,8 +16,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.stdlib" artifactId = "http-native" -version = "2.11.2" -path = "../native/build/libs/http-native-2.11.2.jar" +version = "2.11.3" +path = "../native/build/libs/http-native-2.11.3-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "io.ballerina.stdlib" diff --git a/ballerina/CompilerPlugin.toml b/ballerina/CompilerPlugin.toml index d766246cdc..1f6001a559 100644 --- a/ballerina/CompilerPlugin.toml +++ b/ballerina/CompilerPlugin.toml @@ -3,4 +3,4 @@ id = "http-compiler-plugin" class = "io.ballerina.stdlib.http.compiler.HttpCompilerPlugin" [[dependency]] -path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.2.jar" +path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.3-SNAPSHOT.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 4def31d650..ee2ab8a1f1 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -50,7 +50,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.7.1" +version = "2.7.2" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} @@ -76,7 +76,7 @@ modules = [ [[package]] org = "ballerina" name = "http" -version = "2.11.2" +version = "2.11.3" dependencies = [ {org = "ballerina", name = "auth"}, {org = "ballerina", name = "cache"}, diff --git a/ballerina/http_log_manager.bal b/ballerina/http_log_manager.bal index 67928b082b..28b9a88c97 100644 --- a/ballerina/http_log_manager.bal +++ b/ballerina/http_log_manager.bal @@ -35,9 +35,13 @@ public type TraceLogAdvancedConfiguration record {| # Represents HTTP access log configuration. # # + console - Boolean value to enable or disable console access logs +# + format - The format of access logs to be printed (either `flat` or `json`) +# + attributes - The list of attributes of access logs to be printed # + path - Optional file path to store access logs public type AccessLogConfiguration record {| boolean console = false; + string format = "flat"; + string[] attributes?; string path?; |}; diff --git a/changelog.md b/changelog.md index 748357226c..2929fb5a25 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - [Introduce default status code response record](https://github.com/ballerina-platform/ballerina-library/issues/6491) +- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options.](https://github.com/ballerina-platform/ballerina-library/issues/6111) ## [2.11.2] - 2024-06-14 diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 63cc4c3637..29339bcc0c 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -2498,20 +2498,48 @@ path = "testTraceLog.txt" # Optional host = "localhost" # Optional port = 8080 # Optional ``` + #### 8.2.4 Access log +Ballerina supports HTTP access logs for HTTP services, providing insights into web traffic and request handling. +The access log feature is **disabled by default** to allow users to opt-in as per their requirements. -Ballerina supports HTTP access logs for HTTP services. The access log format used is the combined log format. -The HTTP access logs are **disabled as default**. -To enable access logs, set console=true under the ballerina.http.accessLogConfig in the Config.toml file. Also, -the path field can be used to specify the file path to save the access logs. +To enable access logs, configuration settings are provided under `ballerina.http.accessLogConfig` in the +`Config.toml` file. Users can specify whether logs should be output to the console, a file, or both, +and can select the format and specific attributes to log. ```toml [ballerina.http.accessLogConfig] # Enable printing access logs in console console = true # Default is false -# Specify the file path to save the access logs -path = "testAccessLog.txt" # Optional -``` +# Specify the file path to save the access logs +path = "testAccessLog.txt" # Optional, omit to disable file logging +# Select the format of the access logs +format = "json" # Options: "flat", "json"; Default is "flat". Omit to stick to the default. +# Specify which attributes to log. Omit to stick to the default set. +attributes = ["ip", "date_time", "request", "status", "response_body_size", "http_referrer", "http_user_agent"] +# Default attributes: ip, date_time, request, status, response_body_size, http_referrer, http_user_agent +``` + +##### Configurable Attributes +Users can customize which parts of the access data are logged by specifying attributes in the configuration. +This allows for tailored logging that can focus on particular details relevant to the users' needs. + +| Attribute | Description | +|:----------------------:|:---------------------------------------------------:| +| ip | Client's IP address | +| date_time | HTTP request received time | +| request | Full HTTP request line (method, URI, protocol) | +| request_method | HTTP method of the request | +| request_uri | URI of the request, including parameters | +| scheme | Scheme of the request and HTTP version | +| status | HTTP status code returned to the client | +| request_body_size | Size of the request body in bytes | +| response_body_size | Size of the HTTP response body in bytes | +| request_time | Total time taken to process the request | +| http_referrer | HTTP Referer header, indicating the previous page | +| http_user_agent | User-Agent header, identifying the client software | +| http_x_forwarded_for | Originating IP address if using a proxy | +| http_(X-Custom-Header) | Header fields. Referring to them with `http` followed by the header name. (`x-request-id` ->; `http_x-request-id`) | #### 8.2.5 Panic inside resource diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java index 05a39b23ee..ceec5b68f3 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java @@ -143,7 +143,7 @@ public boolean isHTTPTraceLoggerEnabled() { return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_TRACE_LOG_ENABLED)); } - private boolean isHTTPAccessLoggerEnabled() { + public boolean isHTTPAccessLoggerEnabled() { return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_ACCESS_LOG_ENABLED)); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java index 84af585f01..cf094d30b4 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java @@ -325,14 +325,33 @@ public final class HttpConstants { public static final String HTTP_TRACE_LOG_ENABLED = "http.tracelog.enabled"; public static final String HTTP_ACCESS_LOG = "http.accesslog"; public static final String HTTP_ACCESS_LOG_ENABLED = "http.accesslog.enabled"; + public static final String HTTP_LOG_FORMAT_JSON = "json"; + public static final String HTTP_LOG_FORMAT_FLAT = "flat"; // TraceLog and AccessLog configs public static final BString HTTP_LOG_CONSOLE = StringUtils.fromString("console"); + public static final BString HTTP_LOG_FORMAT = StringUtils.fromString("format"); + public static final BString HTTP_LOG_ATTRIBUTES = StringUtils.fromString("attributes"); public static final BString HTTP_LOG_FILE_PATH = StringUtils.fromString("path"); public static final BString HTTP_TRACE_LOG_HOST = StringUtils.fromString("host"); public static final BString HTTP_TRACE_LOG_PORT = StringUtils.fromString("port"); public static final BString HTTP_LOGGING_PROTOCOL = StringUtils.fromString("HTTP"); + // AccessLog fiend names + public static final String ATTRIBUTE_IP = "ip"; + public static final String ATTRIBUTE_DATE_TIME = "date_time"; + public static final String ATTRIBUTE_REQUEST_METHOD = "request_method"; + public static final String ATTRIBUTE_REQUEST_URI = "request_uri"; + public static final String ATTRIBUTE_SCHEME = "scheme"; + public static final String ATTRIBUTE_REQUEST = "request"; + public static final String ATTRIBUTE_STATUS = "status"; + public static final String ATTRIBUTE_REQUEST_BODY_SIZE = "request_body_size"; + public static final String ATTRIBUTE_RESPONSE_BODY_SIZE = "response_body_size"; + public static final String ATTRIBUTE_REQUEST_TIME = "request_time"; + public static final String ATTRIBUTE_HTTP_REFERRER = "http_referrer"; + public static final String ATTRIBUTE_HTTP_USER_AGENT = "http_user_agent"; + public static final String ATTRIBUTE_HTTP_X_FORWARDED_FOR = "http_x_forwarded_for"; + // ResponseCacheControl struct field names public static final BString RES_CACHE_CONTROL_MUST_REVALIDATE_FIELD = StringUtils.fromString("mustRevalidate"); public static final BString RES_CACHE_CONTROL_NO_CACHE_FIELD = StringUtils.fromString("noCache"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java index b4b718cf24..f553aefa9e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java @@ -290,6 +290,8 @@ protected static void executeNonBlockingAction(DataContext dataContext, boolean } outboundRequestMsg.setProperty(HttpConstants.ORIGIN_HOST, dataContext.getEnvironment().getStrandLocal(HttpConstants.ORIGIN_HOST)); + outboundRequestMsg.setProperty(HttpConstants.INBOUND_MESSAGE, + dataContext.getEnvironment().getStrandLocal(HttpConstants.INBOUND_MESSAGE)); sendOutboundRequest(dataContext, outboundRequestMsg, async); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java index 405c062e68..819a462be4 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java @@ -52,6 +52,7 @@ import static io.ballerina.stdlib.http.api.HttpConstants.CURRENT_TRANSACTION_CONTEXT_PROPERTY; import static io.ballerina.stdlib.http.api.HttpConstants.EMPTY; import static io.ballerina.stdlib.http.api.HttpConstants.EQUAL_SIGN; +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; import static io.ballerina.stdlib.http.api.HttpConstants.MAIN_STRAND; import static io.ballerina.stdlib.http.api.HttpConstants.ORIGIN_HOST; import static io.ballerina.stdlib.http.api.HttpConstants.POOLED_BYTE_BUFFER_FACTORY; @@ -242,7 +243,7 @@ public void notifyFailure(BError bError) { private static Map getPropertiesToPropagate(Environment env) { String[] keys = {CURRENT_TRANSACTION_CONTEXT_PROPERTY, KEY_OBSERVER_CONTEXT, SRC_HANDLER, MAIN_STRAND, - POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST}; + POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST, INBOUND_MESSAGE}; Map subMap = new HashMap<>(); for (String key : keys) { Object value = env.getStrandLocal(key); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java index 63bbf76422..af38f44b8e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java @@ -93,6 +93,9 @@ public static Object createSimpleHttpClient(BObject httpClient, BMap globalPoolC if (connectionManager.isHTTPTraceLoggerEnabled()) { senderConfiguration.setHttpTraceLogEnabled(true); } + if (connectionManager.isHTTPAccessLoggerEnabled()) { + senderConfiguration.setHttpAccessLogEnabled(true); + } senderConfiguration.setTLSStoreType(HttpConstants.PKCS_STORE_TYPE); String httpVersion = clientEndpointConfig.getStringValue(HttpConstants.CLIENT_EP_HTTP_VERSION).getValue(); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java index 1cd7666f49..d3f39210ef 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java @@ -18,8 +18,10 @@ package io.ballerina.stdlib.http.api.logging; +import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; import io.ballerina.stdlib.http.api.logging.formatters.HttpAccessLogFormatter; import io.ballerina.stdlib.http.api.logging.formatters.HttpTraceLogFormatter; import io.ballerina.stdlib.http.api.logging.formatters.JsonLogFormatter; @@ -36,8 +38,12 @@ import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG_ENABLED; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_CONSOLE; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FILE_PATH; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_FLAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_ENABLED; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_HOST; @@ -70,6 +76,7 @@ public HttpLogManager(boolean traceLogConsole, BMap traceLogAdvancedConfig, BMap this.protocol = protocol.getValue(); this.setHttpTraceLogHandler(traceLogConsole, traceLogAdvancedConfig); this.setHttpAccessLogHandler(accessLogConfig); + HttpAccessLogConfig.getInstance().initializeHttpAccessLogConfig(accessLogConfig); } /** @@ -161,10 +168,22 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) { } } + BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); + if (logFormat != null && + !(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) || + logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) { + stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() + + "'. Defaulting to 'flat' format."); + } + + BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES); + if (logAttributes != null && logAttributes.getLength() == 0) { + accessLogsEnabled = false; + } + if (accessLogsEnabled) { System.setProperty(HTTP_ACCESS_LOG_ENABLED, "true"); stdErr.println("ballerina: " + protocol + " access log enabled"); } } - } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java new file mode 100644 index 0000000000..ff63969ce9 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON; + +/** + * Provides a singleton configuration manager for HTTP access logging within the application. + * This class manages settings related to log formats, attributes, and custom header. + * + * @since 2.12.0 + */ +public class HttpAccessLogConfig { + + private static final HttpAccessLogConfig instance = new HttpAccessLogConfig(); + + private final Set excludedAttributes = new HashSet<>(List.of( + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR + )); + private BMap accessLogConfig; + + private HttpAccessLogConfig() {} + + public static HttpAccessLogConfig getInstance() { + return instance; + } + + public void initializeHttpAccessLogConfig(BMap accessLogConfig) { + this.accessLogConfig = accessLogConfig; + } + + public List getCustomHeaders() { + List attributes = getAccessLogAttributes(); + + return attributes.stream() + .filter(attr -> attr.startsWith("http_") && !excludedAttributes.contains(attr)) + .map(attr -> attr.substring(5)) + .collect(Collectors.toList()); + } + + public HttpAccessLogFormat getAccessLogFormat() { + if (accessLogConfig != null) { + BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); + if (logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON)) { + return HttpAccessLogFormat.JSON; + } + } + return HttpAccessLogFormat.FLAT; + } + + public List getAccessLogAttributes() { + if (accessLogConfig != null) { + BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES); + if (logAttributes != null) { + return Arrays.stream(logAttributes.getStringArray()) + .collect(Collectors.toList()); + } + } + return Collections.emptyList(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java new file mode 100644 index 0000000000..4c670ad4e6 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +/** + * Represents the format types available for HTTP access logging. + * This enum is used to specify the preferred logging format for HTTP access logs. + * + * @since 2.12.0 + */ +public enum HttpAccessLogFormat { + FLAT, JSON +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java new file mode 100644 index 0000000000..8890888289 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; + +/** + * Handles the formatting of HTTP access log messages based on the specified log format and attributes. + * This utility class supports both FLAT and JSON formats for the rendering of access log entries, + * accommodating custom attributes and handling multiple messages for detailed logging. + * + * @since 2.12.0 + */ +public class HttpAccessLogFormatter { + + private HttpAccessLogFormatter() {} + + public static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, + List outboundMessages, HttpAccessLogFormat format, + List attributes) { + + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); + if (format == HttpAccessLogFormat.FLAT) { + String inboundFormatted = inboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" ")); + + if (!outboundMessages.isEmpty()) { + String outboundFormatted = outboundMessages.stream() + .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) + .map(outboundMap -> outboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" "))) + .collect(Collectors.joining(" ")); + + return inboundFormatted + " \"~\" " + outboundFormatted; + } else { + return inboundFormatted; + } + } else { + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + + inboundMap.forEach(jsonObject::addProperty); + + if (!outboundMessages.isEmpty()) { + JsonArray upstreamArray = new JsonArray(); + for (HttpAccessLogMessage outboundMessage : outboundMessages) { + Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); + JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); + upstreamArray.add(outboundJson); + } + jsonObject.add("upstream", upstreamArray); + } + return gson.toJson(jsonObject); + } + } + + private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, List attributes) { + List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, + ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, + ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); + List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, + ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); + + Map attributeValues = new LinkedHashMap<>(); + allAttributes.forEach(attr -> attributeValues.put(attr, null)); + + if (!attributes.isEmpty()) { + attributes.forEach(attr -> { + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); + }); + } else { + defaultAttributes.forEach(attr -> + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); + } + return attributeValues; + } + + private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, String attribute) { + return switch (attribute) { + case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); + case ATTRIBUTE_DATE_TIME -> String.format(format == HttpAccessLogFormat.FLAT ? + "[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]" : "%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz", + httpAccessLogMessage.getDateTime()); + case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); + case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); + case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); + case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), + httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); + case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); + case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); + case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); + case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); + case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); + case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); + case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor())); + default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, format, attribute); + }; + } + + private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, String attribute) { + Map customHeaders = httpAccessLogMessage.getCustomHeaders(); + if (attribute.startsWith("http_")) { + String customHeaderKey = attribute.substring(5); + for (Map.Entry entry : customHeaders.entrySet()) { + if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { + String value = entry.getValue(); + return format == HttpAccessLogFormat.FLAT ? String.format("\"%s\"", value) : value; + } + } + return format == HttpAccessLogFormat.FLAT ? "\"-\"" : "-"; + } + return null; + } + + private static String getHyphenForNull(String value) { + return value == null ? "-" : value; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java new file mode 100644 index 0000000000..c929f7570a --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a single HTTP access log message, encapsulating all relevant data for a specific request-response cycle. + * This class stores details such as IP address, date and time, request and response attributes, and custom headers. + * + * @since 2.12.0 + */ +public class HttpAccessLogMessage { + private String ip; + private Calendar dateTime; + private String requestMethod; + private String requestUri; + private String scheme; + private int status; + private long requestBodySize; + private long responseBodySize; + private long requestTime; + private String httpReferrer; + private String httpUserAgent; + private String httpXForwardedFor; + private String host; + private int port; + private Map customHeaders; + + public HttpAccessLogMessage() { + this.customHeaders = new HashMap<>(); + } + + public HttpAccessLogMessage(String ip, Calendar dateTime, String requestMethod, String requestUri, String scheme, + int status, long responseBodySize, String httpReferrer, String httpUserAgent) { + this.ip = ip; + this.dateTime = dateTime; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.scheme = scheme; + this.status = status; + this.responseBodySize = responseBodySize; + this.httpReferrer = httpReferrer; + this.httpUserAgent = httpUserAgent; + this.customHeaders = new HashMap<>(); + } + + public Calendar getDateTime() { + return dateTime; + } + + public void setDateTime(Calendar dateTime) { + this.dateTime = dateTime; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getRequestMethod() { + return requestMethod; + } + + public void setRequestMethod(String requestMethod) { + this.requestMethod = requestMethod; + } + + public String getRequestUri() { + return requestUri; + } + + public void setRequestUri(String requestUri) { + this.requestUri = requestUri; + } + + public String getScheme() { + return scheme; + } + + public void setScheme(String scheme) { + this.scheme = scheme; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public long getRequestBodySize() { + return requestBodySize; + } + + public void setRequestBodySize(Long requestBodySize) { + this.requestBodySize = requestBodySize; + } + + public long getResponseBodySize() { + return responseBodySize; + } + + public void setResponseBodySize(Long responseBodySize) { + this.responseBodySize = responseBodySize; + } + + public long getRequestTime() { + return requestTime; + } + + public void setRequestTime(Long requestTime) { + this.requestTime = requestTime; + } + + public String getHttpUserAgent() { + return httpUserAgent; + } + + public String getHttpReferrer() { + return httpReferrer; + } + + public void setHttpReferrer(String httpReferrer) { + this.httpReferrer = httpReferrer; + } + + public void setHttpUserAgent(String httpUserAgent) { + this.httpUserAgent = httpUserAgent; + } + + public String getHttpXForwardedFor() { + return httpXForwardedFor; + } + + public void setHttpXForwardedFor(String httpXForwardedFor) { + this.httpXForwardedFor = httpXForwardedFor; + } + + public String getHost() { + return this.host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return this.port; + } + + public void setPort(int port) { + this.port = port; + } + + public Map getCustomHeaders() { + return customHeaders; + } + + public void setCustomHeaders(Map customHeaders) { + this.customHeaders = customHeaders; + } + + public void putCustomHeader(String headerKey, String headerValue) { + this.customHeaders.put(headerKey, headerValue); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java new file mode 100644 index 0000000000..72f4c36f5f --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; + +import java.util.ArrayList; +import java.util.List; + +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; + +/** + * Provides utility methods for accessing and manipulating properties related to HTTP access logging + * from HttpCarbonMessage objects. This class includes type-safe property retrieval and handling specific + * logging data structures. + * + * @since 2.12.0 + */ +public class HttpAccessLogUtil { + public static T getTypedProperty(HttpCarbonMessage carbonMessage, String propertyName, Class type) { + Object property = carbonMessage.getProperty(propertyName); + if (type.isInstance(property)) { + return type.cast(property); + } + return null; + } + + public static List getHttpAccessLogMessages(HttpCarbonMessage carbonMessage) { + Object outboundAccessLogMessagesObject = carbonMessage.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return new ArrayList<>(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java new file mode 100644 index 0000000000..505c455990 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; + +/** + * Defines the contract for logging and updating HTTP access information. + * Implementations of this interface should handle the specifics of recording and maintaining HTTP access logs + * for both request and response messages. + * + * @since 2.12.0 + */ +public interface HttpAccessLogger { + + void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); + + void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java new file mode 100644 index 0000000000..8389ba68f9 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.contractimpl.common.Util; +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.util.internal.logging.InternalLogLevel; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogFormatter.formatAccessLogMessage; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; + +/** + * Implements {@link HttpAccessLogger} to log detailed HTTP access information for incoming requests + * and their corresponding responses. + * + * @since 2.12.0 + */ +public class ListenerHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(ListenerHttpAccessLogger.class); + private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); + + private final Calendar inboundRequestArrivalTime; + private Long contentLength = 0L; + private String remoteAddress; + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.remoteAddress = remoteAddress; + } + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, Long contentLength, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.contentLength = contentLength; + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { + HttpHeaders headers = inboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; + } + + // Populate request parameters + String userAgent = "-"; + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + userAgent = headers.get(HttpHeaderNames.USER_AGENT); + } + String referrer = "-"; + if (headers.contains(HttpHeaderNames.REFERER)) { + referrer = headers.get(HttpHeaderNames.REFERER); + } + String method = inboundRequestMsg.getHttpMethod(); + String uri = inboundRequestMsg.getRequestUrl(); + HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); + String protocol; + if (request != null) { + protocol = request.protocolVersion().toString(); + } else { + protocol = inboundRequestMsg.getHttpVersion(); + } + long requestBodySize; + if (headers.contains(HttpHeaderNames.CONTENT_LENGTH)) { + try { + requestBodySize = Long.parseLong(headers.get(HttpHeaderNames.CONTENT_LENGTH)); + } catch (Exception ignored) { + requestBodySize = 0L; + } + } else { + requestBodySize = (long) inboundRequestMsg.getContentSize(); + } + + // Populate response parameters + int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); + + long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); + HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, + inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); + inboundMessage.setRequestBodySize(requestBodySize); + inboundMessage.setRequestTime(requestTime); + + List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); + + String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, + HttpAccessLogConfig.getInstance().getAccessLogFormat(), + HttpAccessLogConfig.getInstance().getAccessLogAttributes()); + ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("updateAccessLogInfo is not a dependant action of this logger"); + } + + public void updateContentLength(HttpContent httpContent) { + contentLength += httpContent.content().readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java new file mode 100644 index 0000000000..fa6eede67e --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; + +/** + * Implements {@link HttpAccessLogger} for the sender side, focusing on updating and enriching + * HTTP access log information for outbound requests and corresponding inbound responses. + * + * @since 2.12.0 + */ +public class SenderHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(SenderHttpAccessLogger.class); + + private Long contentLength = 0L; + private final SocketAddress remoteAddress; + + public SenderHttpAccessLogger(SocketAddress remoteAddress) { + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("logAccessInfo is not a dependant action of this logger"); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage outboundRequestMsg, HttpCarbonMessage inboundResponseMsg) { + HttpAccessLogMessage outboundAccessLogMessage = + getTypedProperty(outboundRequestMsg, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); + if (outboundAccessLogMessage == null) { + return; + } + + if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { + InetAddress inetAddress = inetSocketAddress.getAddress(); + outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); + outboundAccessLogMessage.setHost(inetAddress.getHostName()); + outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); + } + if (outboundAccessLogMessage.getIp().startsWith("/")) { + outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); + } + + // Populate with header parameters + HttpHeaders headers = outboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? + forwardedHops.substring(0, firstCommaIndex) : forwardedHops); + } + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); + } + if (headers.contains(HttpHeaderNames.REFERER)) { + outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); + } + HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> + outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? + headers.get(customHeader) : "-")); + + outboundAccessLogMessage.setRequestMethod(outboundRequestMsg.getHttpMethod()); + outboundAccessLogMessage.setRequestUri((String) outboundRequestMsg.getProperty(TO)); + HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); + if (inboundResponse != null) { + outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); + } else { + outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); + } + long requestBodySize; + if (headers.contains(HttpHeaderNames.CONTENT_LENGTH)) { + try { + requestBodySize = Long.parseLong(headers.get(HttpHeaderNames.CONTENT_LENGTH)); + } catch (Exception ignored) { + requestBodySize = 0L; + } + } else { + requestBodySize = (long) outboundRequestMsg.getContentSize(); + } + outboundAccessLogMessage.setRequestBodySize(requestBodySize); + outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); + outboundAccessLogMessage.setResponseBodySize(contentLength); + long requestTime = Calendar.getInstance().getTimeInMillis() - + outboundAccessLogMessage.getDateTime().getTimeInMillis(); + outboundAccessLogMessage.setRequestTime(requestTime); + + HttpCarbonMessage inboundReqMsg = + getTypedProperty(outboundRequestMsg, INBOUND_MESSAGE, HttpCarbonMessage.class); + + if (inboundReqMsg != null) { + List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); + outboundAccessLogMessages.add(outboundAccessLogMessage); + } + } + + public void updateContentLength(ByteBuf content) { + contentLength += content.readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java index 5f9f282e7a..7027a3bdcf 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java @@ -158,6 +158,8 @@ public final class Constants { public static final String HTTP_REASON_PHRASE = "HTTP_REASON_PHRASE"; public static final String CHNL_HNDLR_CTX = "CHNL_HNDLR_CTX"; + public static final String OUTBOUND_ACCESS_LOG_MESSAGES = "OUTBOUND_ACCESS_LOG_MESSAGES"; + public static final String OUTBOUND_ACCESS_LOG_MESSAGE = "OUTBOUND_ACCESS_LOG_MESSAGE"; public static final String SRC_HANDLER = "SRC_HANDLER"; public static final String POOLED_BYTE_BUFFER_FACTORY = "POOLED_BYTE_BUFFER_FACTORY"; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java index 9c6b41022b..8de2834ed9 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java @@ -41,6 +41,7 @@ public static SenderConfiguration getDefault() { private String id = DEFAULT_KEY; private int socketIdleTimeout = 60000; private boolean httpTraceLogEnabled; + private boolean httpAccessLogEnabled; private ChunkConfig chunkingConfig = ChunkConfig.AUTO; private KeepAliveConfig keepAliveConfig = KeepAliveConfig.AUTO; private boolean forceHttp2 = false; @@ -93,6 +94,14 @@ public void setHttpTraceLogEnabled(boolean httpTraceLogEnabled) { this.httpTraceLogEnabled = httpTraceLogEnabled; } + public boolean isHttpAccessLogEnabled() { + return httpAccessLogEnabled; + } + + public void setHttpAccessLogEnabled(boolean httpAccessLogEnabled) { + this.httpAccessLogEnabled = httpAccessLogEnabled; + } + public ChunkConfig getChunkingConfig() { return chunkingConfig; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java index 1b6a34b4fc..ac3fd9cdff 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java @@ -19,6 +19,7 @@ package io.ballerina.stdlib.http.transport.contractimpl; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpClientConnector; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; @@ -57,8 +58,10 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Calendar; import java.util.NoSuchElementException; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_BEFORE_INITIATING_OUTBOUND_REQUEST; /** @@ -148,6 +151,12 @@ public HttpResponseFuture send(HttpCarbonMessage httpOutboundRequest) { } public HttpResponseFuture send(OutboundMsgHolder outboundMsgHolder, HttpCarbonMessage httpOutboundRequest) { + if (senderConfiguration.isHttpAccessLogEnabled()) { + HttpAccessLogMessage outboundAccessLogMessage = new HttpAccessLogMessage(); + outboundAccessLogMessage.setDateTime(Calendar.getInstance()); + httpOutboundRequest.setProperty(OUTBOUND_ACCESS_LOG_MESSAGE, outboundAccessLogMessage); + } + final HttpResponseFuture httpResponseFuture; Object sourceHandlerObject = httpOutboundRequest.getProperty(Constants.SRC_HANDLER); diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java index 9cf1499b9f..932f39e741 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Calendar; import java.util.Locale; /** @@ -53,6 +54,8 @@ public class HttpOutboundRespListener implements HttpConnectorListener { private ChunkConfig chunkConfig; private KeepAliveConfig keepAliveConfig; private String serverName; + private Calendar inboundRequestArrivalTime; + private String remoteAddress = "-"; public HttpOutboundRespListener(HttpCarbonMessage requestMsg, SourceHandler sourceHandler) { this.requestDataHolder = new RequestDataHolder(requestMsg); @@ -64,6 +67,8 @@ public HttpOutboundRespListener(HttpCarbonMessage requestMsg, SourceHandler sour this.handlerExecutor = HttpTransportContextHolder.getInstance().getHandlerExecutor(); this.serverName = sourceHandler.getServerName(); this.listenerReqRespStateManager = requestMsg.listenerReqRespStateManager; + this.remoteAddress = sourceHandler.getRemoteHost(); + this.inboundRequestArrivalTime = Calendar.getInstance(); setBackPressureObservableToHttpResponseFuture(); } @@ -146,4 +151,12 @@ public void setKeepAliveConfig(KeepAliveConfig config) { public SourceHandler getSourceHandler() { return sourceHandler; } + + public Calendar getInboundRequestArrivalTime() { + return inboundRequestArrivalTime; + } + + public String getRemoteAddress() { + return remoteAddress; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java index b5b1349d20..f85bf648f2 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java @@ -98,6 +98,7 @@ import java.security.cert.X509Certificate; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Date; import java.util.Map; import java.util.concurrent.ScheduledFuture; @@ -123,6 +124,7 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_HANDSHAKE_RESULT; import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_PASSED; import static io.ballerina.stdlib.http.transport.contract.Constants.OK_200; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.PROTOCOL; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_HEADERS; import static io.ballerina.stdlib.http.transport.contract.Constants.TO; @@ -867,6 +869,7 @@ public static HttpCarbonMessage createInboundReqCarbonMsg(HttpRequest httpReques ctx.channel().attr(Constants.MUTUAL_SSL_RESULT_ATTRIBUTE).get()); inboundRequestMsg.setProperty(BASE_64_ENCODED_CERT, ctx.channel().attr(Constants.BASE_64_ENCODED_CERT_ATTRIBUTE).get()); + inboundRequestMsg.setProperty(OUTBOUND_ACCESS_LOG_MESSAGES, new ArrayList<>()); return inboundRequestMsg; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java index e72fd6ae8a..4ed156e017 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java @@ -63,6 +63,7 @@ import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.util.ArrayList; import static io.ballerina.stdlib.http.transport.contract.Constants.BASE_64_ENCODED_CERT; import static io.ballerina.stdlib.http.transport.contract.Constants.CHNL_HNDLR_CTX; @@ -74,6 +75,7 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.LISTENER_PORT; import static io.ballerina.stdlib.http.transport.contract.Constants.LOCAL_ADDRESS; import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_HANDSHAKE_RESULT; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.POOLED_BYTE_BUFFER_FACTORY; import static io.ballerina.stdlib.http.transport.contract.Constants.PROMISED_STREAM_REJECTED_ERROR; import static io.ballerina.stdlib.http.transport.contract.Constants.PROTOCOL; @@ -158,6 +160,7 @@ public static HttpCarbonRequest setupCarbonRequest(HttpRequest httpRequest, Http String uri = httpRequest.uri(); sourceReqCMsg.setRequestUrl(uri); sourceReqCMsg.setProperty(TO, uri); + sourceReqCMsg.setProperty(OUTBOUND_ACCESS_LOG_MESSAGES, new ArrayList<>()); return sourceReqCMsg; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java index 184731c93b..9cba0c95cf 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java @@ -66,8 +66,6 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_ACCESS_LOG_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_TRACE_LOG_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.MAX_ENTITY_BODY_VALIDATION_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.SECURITY; @@ -220,9 +218,6 @@ public void configureHttpPipeline(ChannelPipeline serverPipeline, String initial if (httpTraceLogEnabled) { serverPipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } - if (httpAccessLogEnabled) { - serverPipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); - } } serverPipeline.addLast(URI_HEADER_LENGTH_VALIDATION_HANDLER, new UriAndHeaderLengthValidator(this.serverName)); if (reqSizeValidationConfig.getMaxEntityBodySize() > -1) { @@ -235,7 +230,7 @@ public void configureHttpPipeline(ChannelPipeline serverPipeline, String initial webSocketCompressionEnabled)); serverPipeline.addLast(Constants.BACK_PRESSURE_HANDLER, new BackPressureHandler()); serverPipeline.addLast(Constants.HTTP_SOURCE_HANDLER, - new SourceHandler(this.serverConnectorFuture, this.interfaceId, this.chunkConfig, + new SourceHandler(this.serverConnectorFuture, this, this.interfaceId, this.chunkConfig, keepAliveConfig, this.serverName, this.allChannels, this.listenerChannels, this.pipeliningEnabled, this.pipeliningLimit, this.pipeliningGroup)); @@ -267,9 +262,6 @@ private void configureH2cPipeline(ChannelPipeline pipeline) { pipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } - if (httpAccessLogEnabled) { - pipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); - } final HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java index 9e88ef6a97..835bfd5995 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java @@ -46,6 +46,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Map; import java.util.PriorityQueue; @@ -73,8 +74,10 @@ public class SourceHandler extends ChannelInboundHandlerAdapter { private KeepAliveConfig keepAliveConfig; private ServerConnectorFuture serverConnectorFuture; + private HttpServerChannelInitializer serverChannelInitializer; private String interfaceId; private String serverName; + private String remoteHost; private boolean idleTimeout; private ChannelGroup allChannels; private ChannelGroup listenerChannels; @@ -88,11 +91,13 @@ public class SourceHandler extends ChannelInboundHandlerAdapter { private final Queue holdingQueue = new PriorityQueue<>(NUMBER_OF_INITIAL_EVENTS_HELD); private EventExecutorGroup pipeliningGroup; - public SourceHandler(ServerConnectorFuture serverConnectorFuture, String interfaceId, ChunkConfig chunkConfig, - KeepAliveConfig keepAliveConfig, String serverName, ChannelGroup allChannels, - ChannelGroup listenerChannels, boolean pipeliningEnabled, long pipeliningLimit, - EventExecutorGroup pipeliningGroup) { + public SourceHandler(ServerConnectorFuture serverConnectorFuture, + HttpServerChannelInitializer serverChannelInitializer, String interfaceId, + ChunkConfig chunkConfig, KeepAliveConfig keepAliveConfig, String serverName, + ChannelGroup allChannels, ChannelGroup listenerChannels, boolean pipeliningEnabled, + long pipeliningLimit, EventExecutorGroup pipeliningGroup) { this.serverConnectorFuture = serverConnectorFuture; + this.serverChannelInitializer = serverChannelInitializer; this.interfaceId = interfaceId; this.chunkConfig = chunkConfig; this.keepAliveConfig = keepAliveConfig; @@ -156,6 +161,12 @@ public void channelActive(final ChannelHandlerContext ctx) { handlerExecutor.executeAtSourceConnectionInitiation(Integer.toString(ctx.hashCode())); } this.remoteAddress = ctx.channel().remoteAddress(); + if (this.remoteAddress instanceof InetSocketAddress) { + remoteHost = ((InetSocketAddress) this.remoteAddress).getAddress().toString(); + if (remoteHost.startsWith("/")) { + remoteHost = remoteHost.substring(1); + } + } } @Override @@ -317,6 +328,10 @@ public ServerConnectorFuture getServerConnectorFuture() { return serverConnectorFuture; } + public HttpServerChannelInitializer getServerChannelInitializer() { + return serverChannelInitializer; + } + public ChunkConfig getChunkConfig() { return chunkConfig; } @@ -329,6 +344,10 @@ public String getServerName() { return serverName; } + public String getRemoteHost() { + return remoteHost; + } + public void setConnectedState(boolean connectedState) { this.connectedState = connectedState; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index edbfdfd102..51bfda13b8 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -18,6 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; @@ -63,6 +64,7 @@ public class SendingEntityBody implements ListenerState { private static final Logger LOG = LoggerFactory.getLogger(SendingEntityBody.class); private final HandlerExecutor handlerExecutor; + private final HttpOutboundRespListener outboundRespListener; private final HttpResponseFuture outboundRespStatusFuture; private final ListenerReqRespStateManager listenerReqRespStateManager; private boolean headersWritten; @@ -74,12 +76,19 @@ public class SendingEntityBody implements ListenerState { private ChannelHandlerContext sourceContext; private SourceHandler sourceHandler; - SendingEntityBody(ListenerReqRespStateManager listenerReqRespStateManager, + SendingEntityBody(HttpOutboundRespListener outboundRespListener, + ListenerReqRespStateManager listenerReqRespStateManager, HttpResponseFuture outboundRespStatusFuture, boolean headersWritten) { + this.outboundRespListener = outboundRespListener; this.listenerReqRespStateManager = listenerReqRespStateManager; this.outboundRespStatusFuture = outboundRespStatusFuture; this.headersWritten = headersWritten; this.handlerExecutor = HttpTransportContextHolder.getInstance().getHandlerExecutor(); + this.headRequest = + outboundRespListener.getRequestDataHolder().getHttpMethod().equalsIgnoreCase(HTTP_HEAD_METHOD); + this.inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); + this.sourceContext = outboundRespListener.getSourceContext(); + this.sourceHandler = outboundRespListener.getSourceHandler(); } @Override @@ -100,11 +109,6 @@ public void writeOutboundResponseHeaders(HttpCarbonMessage outboundResponseMsg, @Override public void writeOutboundResponseBody(HttpOutboundRespListener outboundRespListener, HttpCarbonMessage outboundResponseMsg, HttpContent httpContent) { - - headRequest = outboundRespListener.getRequestDataHolder().getHttpMethod().equalsIgnoreCase(HTTP_HEAD_METHOD); - inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); - sourceContext = outboundRespListener.getSourceContext(); - sourceHandler = outboundRespListener.getSourceHandler(); this.outboundResponseMsg = outboundResponseMsg; ChannelFuture outboundChannelFuture; @@ -221,6 +225,12 @@ private void checkForResponseWriteStatus(HttpCarbonMessage inboundRequestMsg, } else { outboundRespStatusFuture.notifyHttpListener(inboundRequestMsg); } + if (sourceHandler.getServerChannelInitializer().isHttpAccessLogEnabled()) { + ListenerHttpAccessLogger accessLogger = new ListenerHttpAccessLogger( + outboundRespListener.getInboundRequestArrivalTime(), contentLength, + outboundRespListener.getRemoteAddress()); + accessLogger.logAccessInfo(inboundRequestMsg, outboundResponseMsg); + } resetOutboundListenerState(); }); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java index 2cf7c3dc14..047cca9212 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java @@ -123,8 +123,8 @@ public void writeOutboundResponseHeaders(HttpCarbonMessage outboundResponseMsg, } private void writeResponse(HttpCarbonMessage outboundResponseMsg, HttpContent httpContent, boolean headersWritten) { - listenerReqRespStateManager.state - = new SendingEntityBody(listenerReqRespStateManager, outboundRespStatusFuture, headersWritten); + listenerReqRespStateManager.state = new SendingEntityBody(outboundResponseListener, listenerReqRespStateManager, + outboundRespStatusFuture, headersWritten); listenerReqRespStateManager.writeOutboundResponseBody(outboundResponseListener, outboundResponseMsg, httpContent); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index 5bf39f733f..44b9c969eb 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -18,6 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states.http2; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -25,7 +26,6 @@ import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil; -import io.ballerina.stdlib.http.transport.contractimpl.listener.HttpServerChannelInitializer; import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2DataEventListener; import io.ballerina.stdlib.http.transport.message.Http2DataFrame; @@ -37,9 +37,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.codec.http2.Http2Connection; import io.netty.handler.codec.http2.Http2ConnectionEncoder; @@ -47,21 +45,13 @@ import io.netty.handler.codec.http2.Http2Exception; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.HttpConversionUtil; -import io.netty.util.internal.logging.InternalLogLevel; -import io.netty.util.internal.logging.InternalLogger; -import io.netty.util.internal.logging.InternalLoggerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Calendar; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG_FORMAT; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; /** @@ -72,36 +62,33 @@ public class SendingEntityBody implements ListenerState { private static final Logger LOG = LoggerFactory.getLogger(SendingEntityBody.class); - private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); private final Http2MessageStateContext http2MessageStateContext; private final ChannelHandlerContext ctx; - private final HttpServerChannelInitializer serverChannelInitializer; private final Http2Connection conn; private final Http2ConnectionEncoder encoder; private final HttpResponseFuture outboundRespStatusFuture; private final HttpCarbonMessage inboundRequestMsg; - private final Calendar inboundRequestArrivalTime; private final int originalStreamId; private final Http2OutboundRespListener http2OutboundRespListener; private HttpCarbonMessage outboundResponseMsg; - - private Long contentLength = 0L; - private String remoteAddress; + private ListenerHttpAccessLogger accessLogger; SendingEntityBody(Http2OutboundRespListener http2OutboundRespListener, Http2MessageStateContext http2MessageStateContext) { this.http2OutboundRespListener = http2OutboundRespListener; this.http2MessageStateContext = http2MessageStateContext; this.ctx = http2OutboundRespListener.getChannelHandlerContext(); - this.serverChannelInitializer = http2OutboundRespListener.getServerChannelInitializer(); this.conn = http2OutboundRespListener.getConnection(); this.encoder = http2OutboundRespListener.getEncoder(); this.inboundRequestMsg = http2OutboundRespListener.getInboundRequestMsg(); this.outboundRespStatusFuture = inboundRequestMsg.getHttpOutboundRespStatusFuture(); - this.inboundRequestArrivalTime = http2OutboundRespListener.getInboundRequestArrivalTime(); this.originalStreamId = http2OutboundRespListener.getOriginalStreamId(); - this.remoteAddress = http2OutboundRespListener.getRemoteAddress(); + if (http2OutboundRespListener.getServerChannelInitializer().isHttpAccessLogEnabled()) { + this.accessLogger = new ListenerHttpAccessLogger( + http2OutboundRespListener.getInboundRequestArrivalTime(), + http2OutboundRespListener.getRemoteAddress()); + } } @Override @@ -167,10 +154,6 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, HttpCarbonMessage outboundResponseMsg, HttpContent httpContent, int streamId) throws Http2Exception { if (httpContent instanceof LastHttpContent) { - if (serverChannelInitializer.isHttpAccessLogEnabled()) { - logAccessInfo(outboundResponseMsg, streamId); - } - final LastHttpContent lastContent = (httpContent == LastHttpContent.EMPTY_LAST_CONTENT) ? new DefaultLastHttpContent() : (LastHttpContent) httpContent; HttpHeaders trailers = lastContent.trailingHeaders(); @@ -185,6 +168,13 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, inboundRequestMsg); } http2OutboundRespListener.removeDefaultResponseWriter(); + if (accessLogger != null) { + if (originalStreamId != streamId) { // Skip access logs for server push messages + LOG.debug("Access logging skipped for server push response"); + return; + } + accessLogger.logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg); + } http2MessageStateContext .setListenerState(new ResponseCompleted(http2OutboundRespListener, http2MessageStateContext)); } else { @@ -193,7 +183,9 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, } private void writeData(HttpContent httpContent, int streamId, boolean endStream) throws Http2Exception { - contentLength += httpContent.content().readableBytes(); + if (accessLogger != null) { + accessLogger.updateContentLength(httpContent); + } validatePromisedStreamState(originalStreamId, streamId, conn, inboundRequestMsg); final ByteBuf content = httpContent.content(); for (Http2DataEventListener dataEventListener : http2OutboundRespListener.getHttp2ServerChannel() @@ -213,47 +205,4 @@ private void writeData(HttpContent httpContent, int streamId, boolean endStream) Util.addResponseWriteFailureListener(outboundRespStatusFuture, channelFuture, http2OutboundRespListener); } } - - private void logAccessInfo(HttpCarbonMessage outboundResponseMsg, int streamId) { - if (!ACCESS_LOGGER.isEnabled(InternalLogLevel.INFO)) { - return; - } - if (originalStreamId != streamId) { // Skip access logs for server push messages - LOG.debug("Access logging skipped for server push response"); - return; - } - HttpHeaders headers = inboundRequestMsg.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; - } - - // Populate request parameters - String userAgent = "-"; - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - userAgent = headers.get(HttpHeaderNames.USER_AGENT); - } - String referrer = "-"; - if (headers.contains(HttpHeaderNames.REFERER)) { - referrer = headers.get(HttpHeaderNames.REFERER); - } - String method = inboundRequestMsg.getHttpMethod(); - String uri = (String) inboundRequestMsg.getProperty(TO); - HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); - String protocol; - if (request != null) { - protocol = request.protocolVersion().toString(); - } else { - protocol = inboundRequestMsg.getHttpVersion(); - } - - // Populate response parameters - int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - - ACCESS_LOGGER.log(InternalLogLevel.INFO, String.format( - ACCESS_LOG_FORMAT, remoteAddress, inboundRequestArrivalTime, method, uri, protocol, - statusCode, contentLength, referrer, userAgent)); - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java index 20734fbfe5..565e624895 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java @@ -78,6 +78,7 @@ public class HttpClientChannelInitializer extends ChannelInitializer