From 7e8dee8100769aa8e7b496db7be992fa535dc1f5 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Thu, 25 Apr 2024 10:52:41 +0530 Subject: [PATCH 01/16] Implement AccessLog support for HTTP/2.0 --- ballerina/http_log_manager.bal | 4 + .../stdlib/http/api/HttpConstants.java | 18 ++ .../http/api/logging/HttpLogManager.java | 3 +- .../accesslog/HttpAccessLogConfig.java | 71 +++++++ .../accesslog/HttpAccessLogFormat.java | 23 +++ .../accesslog/HttpAccessLogMessage.java | 183 ++++++++++++++++++ .../logging/accesslog/HttpAccessLogger.java | 168 ++++++++++++++++ .../formatters/HttpAccessLogFormatter.java | 1 - .../http/transport/contract/Constants.java | 1 + .../DefaultHttpClientConnector.java | 8 + .../listener/http2/Http2SourceHandler.java | 13 ++ .../states/http2/SendingEntityBody.java | 39 ++-- .../sender/states/ReceivingEntityBody.java | 87 +++++++++ .../states/http2/ReceivingEntityBody.java | 85 ++++++++ .../transport/message/HttpCarbonMessage.java | 12 ++ 15 files changed, 699 insertions(+), 17 deletions(-) create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java diff --git a/ballerina/http_log_manager.bal b/ballerina/http_log_manager.bal index 67928b082b..28b9a88c97 100644 --- a/ballerina/http_log_manager.bal +++ b/ballerina/http_log_manager.bal @@ -35,9 +35,13 @@ public type TraceLogAdvancedConfiguration record {| # Represents HTTP access log configuration. # # + console - Boolean value to enable or disable console access logs +# + format - The format of access logs to be printed (either `flat` or `json`) +# + attributes - The list of attributes of access logs to be printed # + path - Optional file path to store access logs public type AccessLogConfiguration record {| boolean console = false; + string format = "flat"; + string[] attributes?; string path?; |}; diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java index 84af585f01..52ac12826e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java @@ -325,14 +325,32 @@ public final class HttpConstants { public static final String HTTP_TRACE_LOG_ENABLED = "http.tracelog.enabled"; public static final String HTTP_ACCESS_LOG = "http.accesslog"; public static final String HTTP_ACCESS_LOG_ENABLED = "http.accesslog.enabled"; + public static final String HTTP_LOG_FORMAT_JSON = "json"; // TraceLog and AccessLog configs public static final BString HTTP_LOG_CONSOLE = StringUtils.fromString("console"); + public static final BString HTTP_LOG_FORMAT = StringUtils.fromString("format"); + public static final BString HTTP_LOG_ATTRIBUTES = StringUtils.fromString("attributes"); public static final BString HTTP_LOG_FILE_PATH = StringUtils.fromString("path"); public static final BString HTTP_TRACE_LOG_HOST = StringUtils.fromString("host"); public static final BString HTTP_TRACE_LOG_PORT = StringUtils.fromString("port"); public static final BString HTTP_LOGGING_PROTOCOL = StringUtils.fromString("HTTP"); + // AccessLog fiend names + public static final String ATTRIBUTE_IP = "ip"; + public static final String ATTRIBUTE_DATE_TIME = "date_time"; + public static final String ATTRIBUTE_REQUEST_METHOD = "request_method"; + public static final String ATTRIBUTE_REQUEST_URI = "request_uri"; + public static final String ATTRIBUTE_SCHEME = "scheme"; + public static final String ATTRIBUTE_REQUEST = "request"; + public static final String ATTRIBUTE_STATUS = "status"; + public static final String ATTRIBUTE_REQUEST_BODY_SIZE = "request_body_size"; + public static final String ATTRIBUTE_RESPONSE_BODY_SIZE = "response_body_size"; + public static final String ATTRIBUTE_REQUEST_TIME = "request_time"; + public static final String ATTRIBUTE_HTTP_REFERRER = "http_referrer"; + public static final String ATTRIBUTE_HTTP_USER_AGENT = "http_user_agent"; + public static final String ATTRIBUTE_HTTP_X_FORWARDED_FOR = "http_x_forwarded_for"; + // ResponseCacheControl struct field names public static final BString RES_CACHE_CONTROL_MUST_REVALIDATE_FIELD = StringUtils.fromString("mustRevalidate"); public static final BString RES_CACHE_CONTROL_NO_CACHE_FIELD = StringUtils.fromString("noCache"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java index 1cd7666f49..5a12d023fa 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java @@ -20,6 +20,7 @@ import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; import io.ballerina.stdlib.http.api.logging.formatters.HttpAccessLogFormatter; import io.ballerina.stdlib.http.api.logging.formatters.HttpTraceLogFormatter; import io.ballerina.stdlib.http.api.logging.formatters.JsonLogFormatter; @@ -70,6 +71,7 @@ public HttpLogManager(boolean traceLogConsole, BMap traceLogAdvancedConfig, BMap this.protocol = protocol.getValue(); this.setHttpTraceLogHandler(traceLogConsole, traceLogAdvancedConfig); this.setHttpAccessLogHandler(accessLogConfig); + HttpAccessLogConfig.getInstance().initializeHttpAccessLogConfig(accessLogConfig); } /** @@ -166,5 +168,4 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) { stdErr.println("ballerina: " + protocol + " access log enabled"); } } - } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java new file mode 100644 index 0000000000..e0c4d6e7b9 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -0,0 +1,71 @@ +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON; + +public class HttpAccessLogConfig { + + private static final HttpAccessLogConfig instance = new HttpAccessLogConfig(); + private static final Set EXCLUDED_ATTRIBUTES = new HashSet<>(List.of( + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR + )); + private BMap accessLogConfig; + + private HttpAccessLogConfig() {} + + public static HttpAccessLogConfig getInstance() { + return instance; + } + + public void initializeHttpAccessLogConfig(BMap accessLogConfig) { + this.accessLogConfig = accessLogConfig; + } + + public List getCustomHeaders() { + List attributes = getAccessLogAttributes(); + if (attributes == null) { + return Collections.emptyList(); + } + + return attributes.stream() + .filter(attr -> attr.startsWith("http_") && !EXCLUDED_ATTRIBUTES.contains(attr)) + .map(attr -> attr.substring(5)) + .collect(Collectors.toList()); + } + + public HttpAccessLogFormat getAccessLogFormat() { + if (accessLogConfig != null) { + BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); + if (logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON)) { + return HttpAccessLogFormat.JSON; + } + } + return HttpAccessLogFormat.FLAT; + } + + public List getAccessLogAttributes() { + if (accessLogConfig != null) { + BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES); + if (logAttributes != null) { + return Arrays.stream(logAttributes.getStringArray()) + .collect(Collectors.toList()); + } + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java new file mode 100644 index 0000000000..4ac452c0f6 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +public enum HttpAccessLogFormat { + FLAT, JSON +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java new file mode 100644 index 0000000000..78947170c9 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +public class HttpAccessLogMessage { + private String ip; + private Calendar dateTime; + private String requestMethod; + private String requestUri; + private String scheme; + private int status; + private long requestBodySize; + private long responseBodySize; + private long requestTime; + private String httpReferrer; + private String httpUserAgent; + private String httpXForwardedFor; + private String host; + private int port; + private Map customHeaders; + + public HttpAccessLogMessage() { + this.customHeaders = new HashMap<>(); + } + + public HttpAccessLogMessage(String ip, Calendar dateTime, String requestMethod, String requestUri, String scheme, + int status, long responseBodySize, String httpReferrer, String httpUserAgent) { + this.ip = ip; + this.dateTime = dateTime; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.scheme = scheme; + this.status = status; + this.responseBodySize = responseBodySize; + this.httpReferrer = httpReferrer; + this.httpUserAgent = httpUserAgent; + this.customHeaders = new HashMap<>(); + } + + public Calendar getDateTime() { + return dateTime; + } + + public void setDateTime(Calendar dateTime) { + this.dateTime = dateTime; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getRequestMethod() { + return requestMethod; + } + + public void setRequestMethod(String requestMethod) { + this.requestMethod = requestMethod; + } + + public String getRequestUri() { + return requestUri; + } + + public void setRequestUri(String requestUri) { + this.requestUri = requestUri; + } + + public String getScheme() { + return scheme; + } + + public void setScheme(String scheme) { + this.scheme = scheme; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public long getRequestBodySize() { + return requestBodySize; + } + + public void setRequestBodySize(Long requestBodySize) { + this.requestBodySize = requestBodySize; + } + + public long getResponseBodySize() { + return responseBodySize; + } + + public void setResponseBodySize(Long responseBodySize) { + this.responseBodySize = responseBodySize; + } + + public long getRequestTime() { + return requestTime; + } + + public void setRequestTime(Long requestTime) { + this.requestTime = requestTime; + } + + public String getHttpUserAgent() { + return httpUserAgent; + } + + public String getHttpReferrer() { + return httpReferrer; + } + + public void setHttpReferrer(String httpReferrer) { + this.httpReferrer = httpReferrer; + } + + public void setHttpUserAgent(String httpUserAgent) { + this.httpUserAgent = httpUserAgent; + } + + public String getHttpXForwardedFor() { + return httpXForwardedFor; + } + + public void setHttpXForwardedFor(String httpXForwardedFor) { + this.httpXForwardedFor = httpXForwardedFor; + } + + public String getHost() { + return this.host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return this.port; + } + + public void setPort(int port) { + this.port = port; + } + + public Map getCustomHeaders() { + return customHeaders; + } + + public void setCustomHeaders(Map customHeaders) { + this.customHeaders = customHeaders; + } + + public void putCustomHeader(String headerKey, String headerValue) { + this.customHeaders.put(headerKey, headerValue); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java new file mode 100644 index 0000000000..f1b93bc2fb --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.netty.util.internal.logging.InternalLogLevel; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; +import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; + +public class HttpAccessLogger { + private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); + + private HttpAccessLogger() {} + + public static boolean isEnabled() { + return ACCESS_LOGGER.isEnabled(InternalLogLevel.INFO); + } + + public static void log(HttpAccessLogMessage inboundMessage, List outboundMessages) { + String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, + HttpAccessLogConfig.getInstance().getAccessLogFormat(), + HttpAccessLogConfig.getInstance().getAccessLogAttributes()); + ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); + } + + private static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, + List outboundMessages, HttpAccessLogFormat format, + List attributes) { + if (format == HttpAccessLogFormat.FLAT) { + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); + String inboundFormatted = inboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" ")); + + if (!outboundMessages.isEmpty()) { + String outboundFormatted = outboundMessages.stream() + .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) + .map(outboundMap -> outboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" "))) + .collect(Collectors.joining(" ")); + + return inboundFormatted + " \"~\" " + outboundFormatted; + } else { + return inboundFormatted; + } + } else { + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + + inboundMap.forEach(jsonObject::addProperty); + + if (!outboundMessages.isEmpty()) { + JsonArray upstreamArray = new JsonArray(); + for (HttpAccessLogMessage outboundMessage : outboundMessages) { + Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); + JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); + upstreamArray.add(outboundJson); + } + jsonObject.add("upstream", upstreamArray); + } + return gson.toJson(jsonObject); + } + } + + private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, List attributes) { + List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, + ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, + ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); + List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, + ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); + + Map attributeValues = new LinkedHashMap<>(); + allAttributes.forEach(attr -> attributeValues.put(attr, null)); + + if (attributes != null) { + attributes.forEach(attr -> { + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); + }); + } else { + defaultAttributes.forEach(attr -> + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); + } + return attributeValues; + } + + private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, String attribute) { + return switch (attribute) { + case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); + case ATTRIBUTE_DATE_TIME -> + String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); + case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); + case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); + case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); + case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), + httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); + case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); + case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); + case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); + case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); + case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); + case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); + case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); + default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); + }; + } + + private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, + String attribute) { + Map customHeaders = httpAccessLogMessage.getCustomHeaders(); + if (attribute.startsWith("http_")) { + String customHeaderKey = attribute.substring(5).toLowerCase(Locale.getDefault()); + return customHeaders.getOrDefault(customHeaderKey, "-"); + } + return null; + } + + private static String getHyphenForNull(String value) { + return value == null ? "-" : value; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java index ad676230c6..3330eaf7a1 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java @@ -29,7 +29,6 @@ * @since 0.965 */ public class HttpAccessLogFormatter extends Formatter { - private static final String format = HttpLogManager.getLogManager().getProperty( HttpAccessLogFormatter.class.getCanonicalName() + ".format"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java index 5f9f282e7a..b21025ba78 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java @@ -158,6 +158,7 @@ public final class Constants { public static final String HTTP_REASON_PHRASE = "HTTP_REASON_PHRASE"; public static final String CHNL_HNDLR_CTX = "CHNL_HNDLR_CTX"; + public static final String OUTBOUND_ACCESS_LOG_MESSAGE = "OUTBOUND_ACCESS_LOG_MESSAGE"; public static final String SRC_HANDLER = "SRC_HANDLER"; public static final String POOLED_BYTE_BUFFER_FACTORY = "POOLED_BYTE_BUFFER_FACTORY"; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java index 1b6a34b4fc..d570ef4619 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java @@ -19,6 +19,7 @@ package io.ballerina.stdlib.http.transport.contractimpl; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpClientConnector; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; @@ -57,8 +58,10 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Calendar; import java.util.NoSuchElementException; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_BEFORE_INITIATING_OUTBOUND_REQUEST; /** @@ -148,6 +151,10 @@ public HttpResponseFuture send(HttpCarbonMessage httpOutboundRequest) { } public HttpResponseFuture send(OutboundMsgHolder outboundMsgHolder, HttpCarbonMessage httpOutboundRequest) { + HttpAccessLogMessage outboundAccessLogMessage = new HttpAccessLogMessage(); + outboundAccessLogMessage.setDateTime(Calendar.getInstance()); + httpOutboundRequest.setProperty(OUTBOUND_ACCESS_LOG_MESSAGE, outboundAccessLogMessage); + final HttpResponseFuture httpResponseFuture; Object sourceHandlerObject = httpOutboundRequest.getProperty(Constants.SRC_HANDLER); @@ -191,6 +198,7 @@ public HttpResponseFuture send(OutboundMsgHolder outboundMsgHolder, HttpCarbonMe new RequestWriteStarter(outboundMsgHolder, activeHttp2ClientChannel).startWritingContent(); httpResponseFuture = outboundMsgHolder.getResponseFuture(); httpResponseFuture.notifyResponseHandle(new ResponseHandle(outboundMsgHolder)); + return httpResponseFuture; } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java index b419b326d9..4ea94255dc 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java @@ -18,6 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.http2; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -50,6 +51,8 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -81,6 +84,7 @@ public final class Http2SourceHandler extends ChannelInboundHandlerAdapter { private SocketAddress remoteAddress; private ChannelGroup allChannels; private ChannelGroup listenerChannels; + private List httpAccessLogMessages; Http2SourceHandler(HttpServerChannelInitializer serverChannelInitializer, Http2ConnectionEncoder encoder, String interfaceId, Http2Connection conn, ServerConnectorFuture serverConnectorFuture, @@ -94,6 +98,7 @@ public final class Http2SourceHandler extends ChannelInboundHandlerAdapter { this.targetChannelPool = new ConcurrentHashMap<>(); this.allChannels = allChannels; this.listenerChannels = listenerChannels; + this.httpAccessLogMessages = new ArrayList<>(); setRemoteFlowController(); setDataEventListeners(); } @@ -291,4 +296,12 @@ public Http2ServerChannel getHttp2ServerChannel() { public SocketAddress getRemoteAddress() { return remoteAddress; } + + public void addHttpAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage) { + this.httpAccessLogMessages.add(httpAccessLogMessage); + } + + public List getHttpAccessLogMessages() { + return this.httpAccessLogMessages; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index 5bf39f733f..7fe4eeae0e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -18,6 +18,8 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states.http2; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -47,20 +49,18 @@ import io.netty.handler.codec.http2.Http2Exception; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.HttpConversionUtil; -import io.netty.util.internal.logging.InternalLogLevel; -import io.netty.util.internal.logging.InternalLogger; -import io.netty.util.internal.logging.InternalLoggerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Calendar; +import java.util.List; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG_FORMAT; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.SRC_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; @@ -72,7 +72,6 @@ public class SendingEntityBody implements ListenerState { private static final Logger LOG = LoggerFactory.getLogger(SendingEntityBody.class); - private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); private final Http2MessageStateContext http2MessageStateContext; private final ChannelHandlerContext ctx; @@ -167,10 +166,6 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, HttpCarbonMessage outboundResponseMsg, HttpContent httpContent, int streamId) throws Http2Exception { if (httpContent instanceof LastHttpContent) { - if (serverChannelInitializer.isHttpAccessLogEnabled()) { - logAccessInfo(outboundResponseMsg, streamId); - } - final LastHttpContent lastContent = (httpContent == LastHttpContent.EMPTY_LAST_CONTENT) ? new DefaultLastHttpContent() : (LastHttpContent) httpContent; HttpHeaders trailers = lastContent.trailingHeaders(); @@ -185,6 +180,9 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, inboundRequestMsg); } http2OutboundRespListener.removeDefaultResponseWriter(); + if (serverChannelInitializer.isHttpAccessLogEnabled()) { + logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg, streamId); + } http2MessageStateContext .setListenerState(new ResponseCompleted(http2OutboundRespListener, http2MessageStateContext)); } else { @@ -214,8 +212,9 @@ private void writeData(HttpContent httpContent, int streamId, boolean endStream) } } - private void logAccessInfo(HttpCarbonMessage outboundResponseMsg, int streamId) { - if (!ACCESS_LOGGER.isEnabled(InternalLogLevel.INFO)) { + private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg, + int streamId) { + if (!HttpAccessLogger.isEnabled()) { return; } if (originalStreamId != streamId) { // Skip access logs for server push messages @@ -252,8 +251,18 @@ private void logAccessInfo(HttpCarbonMessage outboundResponseMsg, int streamId) // Populate response parameters int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - ACCESS_LOGGER.log(InternalLogLevel.INFO, String.format( - ACCESS_LOG_FORMAT, remoteAddress, inboundRequestArrivalTime, method, uri, protocol, - statusCode, contentLength, referrer, userAgent)); + long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); + HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, + inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); + inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); + inboundMessage.setRequestTime(requestTime); + + List outboundMessages = new ArrayList<>(); + Object sourceHandlerObject = inboundRequestMsg.getProperty(SRC_HANDLER); + + if (sourceHandlerObject instanceof Http2SourceHandler http2SourceHandler) { + outboundMessages.addAll(http2SourceHandler.getHttpAccessLogMessages()); + } + HttpAccessLogger.log(inboundMessage, outboundMessages); } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index 22aebfc1cc..21313ad909 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -18,21 +18,36 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contractimpl.common.states.SenderReqRespStateManager; import io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil; +import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.TargetHandler; import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Calendar; + +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_STATE_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.isKeepAlive; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.safelyRemoveHandlers; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.ILLEGAL_STATE_ERROR; @@ -47,6 +62,7 @@ public class ReceivingEntityBody implements SenderState { private final SenderReqRespStateManager senderReqRespStateManager; private final TargetHandler targetHandler; + private Long contentLength = 0L; ReceivingEntityBody(SenderReqRespStateManager senderReqRespStateManager, TargetHandler targetHandler) { this.senderReqRespStateManager = senderReqRespStateManager; @@ -76,9 +92,11 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent StateUtil.setInboundTrailersToNewMessage(((LastHttpContent) httpContent).trailingHeaders(), inboundResponseMsg); inboundResponseMsg.addHttpContent(httpContent); + contentLength += httpContent.content().readableBytes(); inboundResponseMsg.setLastHttpContentArrived(); targetHandler.resetInboundMsg(); safelyRemoveHandlers(targetHandler.getTargetChannel().getChannel().pipeline(), IDLE_STATE_HANDLER); + updateAccessLogInfo(targetHandler, inboundResponseMsg); senderReqRespStateManager.state = new EntityBodyReceived(senderReqRespStateManager); if (!isKeepAlive(targetHandler.getKeepAliveConfig(), @@ -88,6 +106,7 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent targetHandler.getConnectionManager().returnChannel(targetHandler.getTargetChannel()); } else { inboundResponseMsg.addHttpContent(httpContent); + contentLength += httpContent.content().readableBytes(); } } @@ -105,4 +124,72 @@ public void handleIdleTimeoutConnectionClosure(TargetHandler targetHandler, handleIncompleteInboundMessage(targetHandler.getInboundResponseMsg(), IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY); } + + private void updateAccessLogInfo(TargetHandler targetHandler, + HttpCarbonMessage inboundResponseMsg) { + HttpCarbonMessage httpOutboundRequest = targetHandler.getOutboundRequestMsg(); + HttpAccessLogMessage outboundAccessLogMessage = + getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); + Http2SourceHandler http2SourceHandler = + getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); + if (outboundAccessLogMessage == null || http2SourceHandler == null) { + return; + } + + SocketAddress remoteAddress = targetHandler.getTargetChannel().getChannel().remoteAddress(); + if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { + InetAddress inetAddress = inetSocketAddress.getAddress(); + outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); + outboundAccessLogMessage.setHost(inetAddress.getHostName()); + outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); + } + if (outboundAccessLogMessage.getIp().startsWith("/")) { + outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); + } + + // Populate with header parameters + HttpHeaders headers = httpOutboundRequest.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? + forwardedHops.substring(0, firstCommaIndex) : forwardedHops); + } + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); + } + if (headers.contains(HttpHeaderNames.REFERER)) { + outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); + } + HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> + outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? + headers.get(customHeader) : "-")); + + outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); + outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); + HttpMessage outboundRequest = httpOutboundRequest.getNettyHttpRequest(); + if (outboundRequest != null) { + outboundAccessLogMessage.setScheme(outboundRequest.protocolVersion().toString()); + } else { + outboundAccessLogMessage.setScheme(httpOutboundRequest.getHttpVersion()); + } + outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); + outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); + outboundAccessLogMessage.setResponseBodySize(contentLength); + long requestTime = Calendar.getInstance().getTimeInMillis() - + outboundAccessLogMessage.getDateTime().getTimeInMillis(); + outboundAccessLogMessage.setRequestTime(requestTime); + + http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } + + private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { + Object property = request.getProperty(propertyName); + if (type.isInstance(property)) { + return type.cast(property); + } + return null; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index ac059d85c7..7ae278a2f0 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -18,7 +18,11 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states.http2; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; +import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2ClientChannel; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2TargetHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.OutboundMsgHolder; @@ -31,13 +35,24 @@ import io.netty.handler.codec.http.DefaultHttpContent; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http2.Http2Exception; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Calendar; + +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_GOAWAY_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_RST_STREAM_WHILE_READING_INBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.releaseContent; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.handleIncompleteInboundMessage; @@ -53,6 +68,7 @@ public class ReceivingEntityBody implements SenderState { private final Http2TargetHandler http2TargetHandler; private final Http2ClientChannel http2ClientChannel; private final Http2TargetHandler.Http2RequestWriter http2RequestWriter; + private Long contentLength = 0L; ReceivingEntityBody(Http2TargetHandler http2TargetHandler, Http2TargetHandler.Http2RequestWriter http2RequestWriter) { @@ -135,6 +151,7 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun Http2MessageStateContext http2MessageStateContext) { int streamId = http2DataFrame.getStreamId(); ByteBuf data = http2DataFrame.getData(); + contentLength += data.readableBytes(); boolean endOfStream = http2DataFrame.isEndOfStream(); if (serverPush) { @@ -143,6 +160,7 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun onResponseDataRead(outboundMsgHolder, streamId, endOfStream, data); } if (endOfStream) { + updateAccessLogInfo(outboundMsgHolder); http2MessageStateContext.setSenderState(new EntityBodyReceived(http2TargetHandler, http2RequestWriter)); } } @@ -170,4 +188,71 @@ private void onResponseDataRead(OutboundMsgHolder outboundMsgHolder, int streamI responseMessage.addHttpContent(new DefaultHttpContent(data.retain())); } } + + private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { + HttpCarbonMessage httpOutboundRequest = outboundMsgHolder.getRequest(); + HttpAccessLogMessage outboundAccessLogMessage = + getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); + Http2SourceHandler http2SourceHandler = + getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); + if (outboundAccessLogMessage == null || http2SourceHandler == null) { + return; + } + + SocketAddress remoteAddress = http2TargetHandler.getHttp2ClientChannel().getChannel().remoteAddress(); + if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { + InetAddress inetAddress = inetSocketAddress.getAddress(); + outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); + outboundAccessLogMessage.setHost(inetAddress.getHostName()); + outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); + } + if (outboundAccessLogMessage.getIp().startsWith("/")) { + outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); + } + + // Populate with header parameters + HttpHeaders headers = httpOutboundRequest.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? + forwardedHops.substring(0, firstCommaIndex) : forwardedHops); + } + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); + } + if (headers.contains(HttpHeaderNames.REFERER)) { + outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); + } + HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> + outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? + headers.get(customHeader) : "-")); + + outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); + outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); + HttpMessage outboundRequest = httpOutboundRequest.getNettyHttpRequest(); + if (outboundRequest != null) { + outboundAccessLogMessage.setScheme(outboundRequest.protocolVersion().toString()); + } else { + outboundAccessLogMessage.setScheme(httpOutboundRequest.getHttpVersion()); + } + outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); + outboundAccessLogMessage.setStatus(outboundMsgHolder.getResponse().getHttpStatusCode()); + outboundAccessLogMessage.setResponseBodySize(contentLength); + long requestTime = Calendar.getInstance().getTimeInMillis() - + outboundAccessLogMessage.getDateTime().getTimeInMillis(); + outboundAccessLogMessage.setRequestTime(requestTime); + + http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } + + private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { + Object property = request.getProperty(propertyName); + if (type.isInstance(property)) { + return type.cast(property); + } + return null; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java b/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java index c2d0e47f48..f10a1519c1 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java @@ -79,6 +79,7 @@ public class HttpCarbonMessage { private String httpMethod; private String requestUrl; private Integer httpStatusCode; + private Integer contentSize = 0; private boolean contentReleased = false; public HttpCarbonMessage(HttpMessage httpMessage, Listener contentListener) { @@ -144,6 +145,9 @@ public synchronized void addHttpContent(HttpContent httpContent) { public HttpContent getHttpContent() { HttpContent httpContent = this.blockingEntityCollector.getHttpContent(); this.contentObservable.notifyGetListener(httpContent); + if (httpContent != null) { + this.setContentSize(httpContent.content().readableBytes()); + } return httpContent; } @@ -547,6 +551,14 @@ public boolean isPipeliningEnabled() { return pipeliningEnabled; } + public void setContentSize(Integer contentSize) { + this.contentSize = contentSize; + } + + public Integer getContentSize() { + return contentSize; + } + public void setContentReleased(boolean contentReleased) { this.contentReleased = contentReleased; } From cfa3725f3efaf0b9972bff6df786527a33f6e029 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Mon, 3 Jun 2024 11:54:55 +0530 Subject: [PATCH 02/16] Implement HTTP/1.1 access logging --- .../DefaultHttpClientConnector.java | 1 - .../HttpOutboundRespListener.java | 13 ++++ .../HttpServerChannelInitializer.java | 16 ++-- .../contractimpl/listener/SourceHandler.java | 40 +++++++++- .../listener/states/SendingEntityBody.java | 77 +++++++++++++++++-- .../listener/states/SendingHeaders.java | 4 +- .../sender/states/ReceivingEntityBody.java | 15 +++- .../states/http2/ReceivingEntityBody.java | 15 +++- 8 files changed, 151 insertions(+), 30 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java index d570ef4619..ba92ddfa24 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java @@ -198,7 +198,6 @@ public HttpResponseFuture send(OutboundMsgHolder outboundMsgHolder, HttpCarbonMe new RequestWriteStarter(outboundMsgHolder, activeHttp2ClientChannel).startWritingContent(); httpResponseFuture = outboundMsgHolder.getResponseFuture(); httpResponseFuture.notifyResponseHandle(new ResponseHandle(outboundMsgHolder)); - return httpResponseFuture; } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java index 9cf1499b9f..932f39e741 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/HttpOutboundRespListener.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Calendar; import java.util.Locale; /** @@ -53,6 +54,8 @@ public class HttpOutboundRespListener implements HttpConnectorListener { private ChunkConfig chunkConfig; private KeepAliveConfig keepAliveConfig; private String serverName; + private Calendar inboundRequestArrivalTime; + private String remoteAddress = "-"; public HttpOutboundRespListener(HttpCarbonMessage requestMsg, SourceHandler sourceHandler) { this.requestDataHolder = new RequestDataHolder(requestMsg); @@ -64,6 +67,8 @@ public HttpOutboundRespListener(HttpCarbonMessage requestMsg, SourceHandler sour this.handlerExecutor = HttpTransportContextHolder.getInstance().getHandlerExecutor(); this.serverName = sourceHandler.getServerName(); this.listenerReqRespStateManager = requestMsg.listenerReqRespStateManager; + this.remoteAddress = sourceHandler.getRemoteHost(); + this.inboundRequestArrivalTime = Calendar.getInstance(); setBackPressureObservableToHttpResponseFuture(); } @@ -146,4 +151,12 @@ public void setKeepAliveConfig(KeepAliveConfig config) { public SourceHandler getSourceHandler() { return sourceHandler; } + + public Calendar getInboundRequestArrivalTime() { + return inboundRequestArrivalTime; + } + + public String getRemoteAddress() { + return remoteAddress; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java index 184731c93b..a18998bdcd 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java @@ -66,8 +66,6 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_ACCESS_LOG_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_TRACE_LOG_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.MAX_ENTITY_BODY_VALIDATION_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.SECURITY; @@ -220,9 +218,9 @@ public void configureHttpPipeline(ChannelPipeline serverPipeline, String initial if (httpTraceLogEnabled) { serverPipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } - if (httpAccessLogEnabled) { - serverPipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); - } +// if (httpAccessLogEnabled) { +// serverPipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); +// } } serverPipeline.addLast(URI_HEADER_LENGTH_VALIDATION_HANDLER, new UriAndHeaderLengthValidator(this.serverName)); if (reqSizeValidationConfig.getMaxEntityBodySize() > -1) { @@ -235,7 +233,7 @@ public void configureHttpPipeline(ChannelPipeline serverPipeline, String initial webSocketCompressionEnabled)); serverPipeline.addLast(Constants.BACK_PRESSURE_HANDLER, new BackPressureHandler()); serverPipeline.addLast(Constants.HTTP_SOURCE_HANDLER, - new SourceHandler(this.serverConnectorFuture, this.interfaceId, this.chunkConfig, + new SourceHandler(this.serverConnectorFuture, this, this.interfaceId, this.chunkConfig, keepAliveConfig, this.serverName, this.allChannels, this.listenerChannels, this.pipeliningEnabled, this.pipeliningLimit, this.pipeliningGroup)); @@ -267,9 +265,9 @@ private void configureH2cPipeline(ChannelPipeline pipeline) { pipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } - if (httpAccessLogEnabled) { - pipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); - } +// if (httpAccessLogEnabled) { +// pipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); +// } final HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java index 9e88ef6a97..75eb3c1dd0 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java @@ -19,6 +19,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.config.ChunkConfig; @@ -46,7 +47,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.Queue; @@ -73,8 +77,10 @@ public class SourceHandler extends ChannelInboundHandlerAdapter { private KeepAliveConfig keepAliveConfig; private ServerConnectorFuture serverConnectorFuture; + private HttpServerChannelInitializer serverChannelInitializer; private String interfaceId; private String serverName; + private String remoteHost; private boolean idleTimeout; private ChannelGroup allChannels; private ChannelGroup listenerChannels; @@ -87,12 +93,15 @@ public class SourceHandler extends ChannelInboundHandlerAdapter { private long sequenceId = 1L; //Keep track of the request order for http 1.1 pipelining private final Queue holdingQueue = new PriorityQueue<>(NUMBER_OF_INITIAL_EVENTS_HELD); private EventExecutorGroup pipeliningGroup; + private List httpAccessLogMessages; - public SourceHandler(ServerConnectorFuture serverConnectorFuture, String interfaceId, ChunkConfig chunkConfig, - KeepAliveConfig keepAliveConfig, String serverName, ChannelGroup allChannels, - ChannelGroup listenerChannels, boolean pipeliningEnabled, long pipeliningLimit, - EventExecutorGroup pipeliningGroup) { + public SourceHandler(ServerConnectorFuture serverConnectorFuture, + HttpServerChannelInitializer serverChannelInitializer, String interfaceId, + ChunkConfig chunkConfig, KeepAliveConfig keepAliveConfig, String serverName, + ChannelGroup allChannels, ChannelGroup listenerChannels, boolean pipeliningEnabled, + long pipeliningLimit, EventExecutorGroup pipeliningGroup) { this.serverConnectorFuture = serverConnectorFuture; + this.serverChannelInitializer = serverChannelInitializer; this.interfaceId = interfaceId; this.chunkConfig = chunkConfig; this.keepAliveConfig = keepAliveConfig; @@ -104,6 +113,7 @@ public SourceHandler(ServerConnectorFuture serverConnectorFuture, String interfa this.pipeliningEnabled = pipeliningEnabled; this.pipeliningLimit = pipeliningLimit; this.pipeliningGroup = pipeliningGroup; + this.httpAccessLogMessages = new ArrayList<>(); } @SuppressWarnings("unchecked") @@ -156,6 +166,12 @@ public void channelActive(final ChannelHandlerContext ctx) { handlerExecutor.executeAtSourceConnectionInitiation(Integer.toString(ctx.hashCode())); } this.remoteAddress = ctx.channel().remoteAddress(); + if (this.remoteAddress instanceof InetSocketAddress) { + remoteHost = ((InetSocketAddress) this.remoteAddress).getAddress().toString(); + if (remoteHost.startsWith("/")) { + remoteHost = remoteHost.substring(1); + } + } } @Override @@ -317,6 +333,10 @@ public ServerConnectorFuture getServerConnectorFuture() { return serverConnectorFuture; } + public HttpServerChannelInitializer getServerChannelInitializer() { + return serverChannelInitializer; + } + public ChunkConfig getChunkConfig() { return chunkConfig; } @@ -329,6 +349,10 @@ public String getServerName() { return serverName; } + public String getRemoteHost() { + return remoteHost; + } + public void setConnectedState(boolean connectedState) { this.connectedState = connectedState; } @@ -340,4 +364,12 @@ public void removeRequestEntry(HttpCarbonMessage inboundRequestMsg) { public void resetInboundRequestMsg() { this.inboundRequestMsg = null; } + + public void addHttpAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage) { + this.httpAccessLogMessages.add(httpAccessLogMessage); + } + + public List getHttpAccessLogMessages() { + return this.httpAccessLogMessages; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index edbfdfd102..2c1fc3e873 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -18,11 +18,14 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; import io.ballerina.stdlib.http.transport.contractimpl.HttpOutboundRespListener; +import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; import io.ballerina.stdlib.http.transport.internal.HandlerExecutor; import io.ballerina.stdlib.http.transport.internal.HttpTransportContextHolder; @@ -34,6 +37,9 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; @@ -44,13 +50,16 @@ import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import java.util.Queue; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_HEAD_METHOD; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_TO_HOST_CONNECTION_CLOSED; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.createFullHttpResponse; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.setupContentLengthRequest; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.ILLEGAL_STATE_ERROR; @@ -63,6 +72,7 @@ public class SendingEntityBody implements ListenerState { private static final Logger LOG = LoggerFactory.getLogger(SendingEntityBody.class); private final HandlerExecutor handlerExecutor; + private final HttpOutboundRespListener outboundRespListener; private final HttpResponseFuture outboundRespStatusFuture; private final ListenerReqRespStateManager listenerReqRespStateManager; private boolean headersWritten; @@ -73,13 +83,24 @@ public class SendingEntityBody implements ListenerState { private HttpCarbonMessage outboundResponseMsg; private ChannelHandlerContext sourceContext; private SourceHandler sourceHandler; + private final Calendar inboundRequestArrivalTime; + private String remoteAddress = "-"; - SendingEntityBody(ListenerReqRespStateManager listenerReqRespStateManager, + SendingEntityBody(HttpOutboundRespListener outboundRespListener, + ListenerReqRespStateManager listenerReqRespStateManager, HttpResponseFuture outboundRespStatusFuture, boolean headersWritten) { + this.outboundRespListener = outboundRespListener; this.listenerReqRespStateManager = listenerReqRespStateManager; this.outboundRespStatusFuture = outboundRespStatusFuture; this.headersWritten = headersWritten; this.handlerExecutor = HttpTransportContextHolder.getInstance().getHandlerExecutor(); + this.headRequest = + outboundRespListener.getRequestDataHolder().getHttpMethod().equalsIgnoreCase(HTTP_HEAD_METHOD); + this.inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); + this.sourceContext = outboundRespListener.getSourceContext(); + this.sourceHandler = outboundRespListener.getSourceHandler(); + this.inboundRequestArrivalTime = outboundRespListener.getInboundRequestArrivalTime(); + this.remoteAddress = outboundRespListener.getRemoteAddress(); } @Override @@ -100,11 +121,6 @@ public void writeOutboundResponseHeaders(HttpCarbonMessage outboundResponseMsg, @Override public void writeOutboundResponseBody(HttpOutboundRespListener outboundRespListener, HttpCarbonMessage outboundResponseMsg, HttpContent httpContent) { - - headRequest = outboundRespListener.getRequestDataHolder().getHttpMethod().equalsIgnoreCase(HTTP_HEAD_METHOD); - inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); - sourceContext = outboundRespListener.getSourceContext(); - sourceHandler = outboundRespListener.getSourceHandler(); this.outboundResponseMsg = outboundResponseMsg; ChannelFuture outboundChannelFuture; @@ -221,6 +237,9 @@ private void checkForResponseWriteStatus(HttpCarbonMessage inboundRequestMsg, } else { outboundRespStatusFuture.notifyHttpListener(inboundRequestMsg); } + if (sourceHandler.getServerChannelInitializer().isHttpAccessLogEnabled()) { + logAccessInfo(inboundRequestMsg, outboundResponseMsg); + } resetOutboundListenerState(); }); } @@ -270,6 +289,52 @@ private void triggerPipeliningLogic(HttpCarbonMessage outboundResponseMsg) { } } + private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { + if (!HttpAccessLogger.isEnabled()) { + return; + } + + HttpHeaders headers = inboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; + } + + // Populate request parameters + String userAgent = "-"; + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + userAgent = headers.get(HttpHeaderNames.USER_AGENT); + } + String referrer = "-"; + if (headers.contains(HttpHeaderNames.REFERER)) { + referrer = headers.get(HttpHeaderNames.REFERER); + } + String method = inboundRequestMsg.getHttpMethod(); + String uri = (String) inboundRequestMsg.getProperty(TO); + HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); + String protocol; + if (request != null) { + protocol = request.protocolVersion().toString(); + } else { + protocol = inboundRequestMsg.getHttpVersion(); + } + + // Populate response parameters + int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); + + long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); + HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, + inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); + inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); + inboundMessage.setRequestTime(requestTime); + + List outboundMessages = new ArrayList<>(sourceHandler.getHttpAccessLogMessages()); + + HttpAccessLogger.log(inboundMessage, outboundMessages); + } + private void resetOutboundListenerState() { contentList.clear(); contentLength = 0; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java index 2cf7c3dc14..047cca9212 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingHeaders.java @@ -123,8 +123,8 @@ public void writeOutboundResponseHeaders(HttpCarbonMessage outboundResponseMsg, } private void writeResponse(HttpCarbonMessage outboundResponseMsg, HttpContent httpContent, boolean headersWritten) { - listenerReqRespStateManager.state - = new SendingEntityBody(listenerReqRespStateManager, outboundRespStatusFuture, headersWritten); + listenerReqRespStateManager.state = new SendingEntityBody(outboundResponseListener, listenerReqRespStateManager, + outboundRespStatusFuture, headersWritten); listenerReqRespStateManager.writeOutboundResponseBody(outboundResponseListener, outboundResponseMsg, httpContent); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index 21313ad909..e696febcc0 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -24,6 +24,7 @@ import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contractimpl.common.states.SenderReqRespStateManager; import io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil; +import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.TargetHandler; import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; @@ -130,9 +131,7 @@ private void updateAccessLogInfo(TargetHandler targetHandler, HttpCarbonMessage httpOutboundRequest = targetHandler.getOutboundRequestMsg(); HttpAccessLogMessage outboundAccessLogMessage = getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - Http2SourceHandler http2SourceHandler = - getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); - if (outboundAccessLogMessage == null || http2SourceHandler == null) { + if (outboundAccessLogMessage == null) { return; } @@ -182,7 +181,15 @@ private void updateAccessLogInfo(TargetHandler targetHandler, outboundAccessLogMessage.getDateTime().getTimeInMillis(); outboundAccessLogMessage.setRequestTime(requestTime); - http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + SourceHandler srcHandler = getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, SourceHandler.class); + Http2SourceHandler http2SourceHandler = + getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); + + if (srcHandler != null) { + srcHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } else if (http2SourceHandler != null) { + http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } } private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index 7ae278a2f0..0783b58930 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -22,6 +22,7 @@ import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; +import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2ClientChannel; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2TargetHandler; @@ -193,9 +194,7 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { HttpCarbonMessage httpOutboundRequest = outboundMsgHolder.getRequest(); HttpAccessLogMessage outboundAccessLogMessage = getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - Http2SourceHandler http2SourceHandler = - getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); - if (outboundAccessLogMessage == null || http2SourceHandler == null) { + if (outboundAccessLogMessage == null) { return; } @@ -245,7 +244,15 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { outboundAccessLogMessage.getDateTime().getTimeInMillis(); outboundAccessLogMessage.setRequestTime(requestTime); - http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + SourceHandler srcHandler = getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, SourceHandler.class); + Http2SourceHandler http2SourceHandler = + getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); + + if (srcHandler != null) { + srcHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } else if (http2SourceHandler != null) { + http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + } } private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { From b378766dbe14fe1ff153d221043cca4f8bcec6f8 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Mon, 10 Jun 2024 00:52:07 +0530 Subject: [PATCH 03/16] Fix HTTP/1.1 stacked logging when Connection is keep-alive --- .../http/api/HttpConnectionManager.java | 2 +- .../client/actions/AbstractHTTPAction.java | 2 ++ .../api/client/actions/HttpClientAction.java | 3 +- .../endpoint/CreateSimpleHttpClient.java | 3 ++ .../formatters/HttpAccessLogFormatter.java | 1 + .../http/transport/contract/Constants.java | 1 + .../contract/config/SenderConfiguration.java | 9 ++++++ .../DefaultHttpClientConnector.java | 8 +++-- .../transport/contractimpl/common/Util.java | 3 ++ .../HttpServerChannelInitializer.java | 6 ---- .../contractimpl/listener/SourceHandler.java | 13 -------- .../listener/states/SendingEntityBody.java | 21 ++++++++++-- .../states/http2/SendingEntityBody.java | 3 +- .../sender/HttpClientChannelInitializer.java | 8 +++++ .../contractimpl/sender/TargetHandler.java | 9 ++++++ .../sender/http2/Http2TargetHandler.java | 10 ++++++ .../sender/states/ReceivingEntityBody.java | 32 ++++++++++++++++--- .../states/http2/ReceivingEntityBody.java | 32 ++++++++++++++++--- .../config/SenderConfigurationTest.java | 9 ++++++ 19 files changed, 138 insertions(+), 37 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java index 05a39b23ee..ceec5b68f3 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConnectionManager.java @@ -143,7 +143,7 @@ public boolean isHTTPTraceLoggerEnabled() { return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_TRACE_LOG_ENABLED)); } - private boolean isHTTPAccessLoggerEnabled() { + public boolean isHTTPAccessLoggerEnabled() { return Boolean.parseBoolean(System.getProperty(HttpConstants.HTTP_ACCESS_LOG_ENABLED)); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java index b4b718cf24..f553aefa9e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/AbstractHTTPAction.java @@ -290,6 +290,8 @@ protected static void executeNonBlockingAction(DataContext dataContext, boolean } outboundRequestMsg.setProperty(HttpConstants.ORIGIN_HOST, dataContext.getEnvironment().getStrandLocal(HttpConstants.ORIGIN_HOST)); + outboundRequestMsg.setProperty(HttpConstants.INBOUND_MESSAGE, + dataContext.getEnvironment().getStrandLocal(HttpConstants.INBOUND_MESSAGE)); sendOutboundRequest(dataContext, outboundRequestMsg, async); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java index 405c062e68..819a462be4 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/actions/HttpClientAction.java @@ -52,6 +52,7 @@ import static io.ballerina.stdlib.http.api.HttpConstants.CURRENT_TRANSACTION_CONTEXT_PROPERTY; import static io.ballerina.stdlib.http.api.HttpConstants.EMPTY; import static io.ballerina.stdlib.http.api.HttpConstants.EQUAL_SIGN; +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; import static io.ballerina.stdlib.http.api.HttpConstants.MAIN_STRAND; import static io.ballerina.stdlib.http.api.HttpConstants.ORIGIN_HOST; import static io.ballerina.stdlib.http.api.HttpConstants.POOLED_BYTE_BUFFER_FACTORY; @@ -242,7 +243,7 @@ public void notifyFailure(BError bError) { private static Map getPropertiesToPropagate(Environment env) { String[] keys = {CURRENT_TRANSACTION_CONTEXT_PROPERTY, KEY_OBSERVER_CONTEXT, SRC_HANDLER, MAIN_STRAND, - POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST}; + POOLED_BYTE_BUFFER_FACTORY, REMOTE_ADDRESS, ORIGIN_HOST, INBOUND_MESSAGE}; Map subMap = new HashMap<>(); for (String key : keys) { Object value = env.getStrandLocal(key); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java b/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java index 63bbf76422..af38f44b8e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/client/endpoint/CreateSimpleHttpClient.java @@ -93,6 +93,9 @@ public static Object createSimpleHttpClient(BObject httpClient, BMap globalPoolC if (connectionManager.isHTTPTraceLoggerEnabled()) { senderConfiguration.setHttpTraceLogEnabled(true); } + if (connectionManager.isHTTPAccessLoggerEnabled()) { + senderConfiguration.setHttpAccessLogEnabled(true); + } senderConfiguration.setTLSStoreType(HttpConstants.PKCS_STORE_TYPE); String httpVersion = clientEndpointConfig.getStringValue(HttpConstants.CLIENT_EP_HTTP_VERSION).getValue(); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java index 3330eaf7a1..ad676230c6 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/formatters/HttpAccessLogFormatter.java @@ -29,6 +29,7 @@ * @since 0.965 */ public class HttpAccessLogFormatter extends Formatter { + private static final String format = HttpLogManager.getLogManager().getProperty( HttpAccessLogFormatter.class.getCanonicalName() + ".format"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java index b21025ba78..7027a3bdcf 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/Constants.java @@ -158,6 +158,7 @@ public final class Constants { public static final String HTTP_REASON_PHRASE = "HTTP_REASON_PHRASE"; public static final String CHNL_HNDLR_CTX = "CHNL_HNDLR_CTX"; + public static final String OUTBOUND_ACCESS_LOG_MESSAGES = "OUTBOUND_ACCESS_LOG_MESSAGES"; public static final String OUTBOUND_ACCESS_LOG_MESSAGE = "OUTBOUND_ACCESS_LOG_MESSAGE"; public static final String SRC_HANDLER = "SRC_HANDLER"; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java index 9c6b41022b..8de2834ed9 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfiguration.java @@ -41,6 +41,7 @@ public static SenderConfiguration getDefault() { private String id = DEFAULT_KEY; private int socketIdleTimeout = 60000; private boolean httpTraceLogEnabled; + private boolean httpAccessLogEnabled; private ChunkConfig chunkingConfig = ChunkConfig.AUTO; private KeepAliveConfig keepAliveConfig = KeepAliveConfig.AUTO; private boolean forceHttp2 = false; @@ -93,6 +94,14 @@ public void setHttpTraceLogEnabled(boolean httpTraceLogEnabled) { this.httpTraceLogEnabled = httpTraceLogEnabled; } + public boolean isHttpAccessLogEnabled() { + return httpAccessLogEnabled; + } + + public void setHttpAccessLogEnabled(boolean httpAccessLogEnabled) { + this.httpAccessLogEnabled = httpAccessLogEnabled; + } + public ChunkConfig getChunkingConfig() { return chunkingConfig; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java index ba92ddfa24..ac3fd9cdff 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/DefaultHttpClientConnector.java @@ -151,9 +151,11 @@ public HttpResponseFuture send(HttpCarbonMessage httpOutboundRequest) { } public HttpResponseFuture send(OutboundMsgHolder outboundMsgHolder, HttpCarbonMessage httpOutboundRequest) { - HttpAccessLogMessage outboundAccessLogMessage = new HttpAccessLogMessage(); - outboundAccessLogMessage.setDateTime(Calendar.getInstance()); - httpOutboundRequest.setProperty(OUTBOUND_ACCESS_LOG_MESSAGE, outboundAccessLogMessage); + if (senderConfiguration.isHttpAccessLogEnabled()) { + HttpAccessLogMessage outboundAccessLogMessage = new HttpAccessLogMessage(); + outboundAccessLogMessage.setDateTime(Calendar.getInstance()); + httpOutboundRequest.setProperty(OUTBOUND_ACCESS_LOG_MESSAGE, outboundAccessLogMessage); + } final HttpResponseFuture httpResponseFuture; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java index b5b1349d20..f85bf648f2 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/Util.java @@ -98,6 +98,7 @@ import java.security.cert.X509Certificate; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Date; import java.util.Map; import java.util.concurrent.ScheduledFuture; @@ -123,6 +124,7 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_HANDSHAKE_RESULT; import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_PASSED; import static io.ballerina.stdlib.http.transport.contract.Constants.OK_200; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.PROTOCOL; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_HEADERS; import static io.ballerina.stdlib.http.transport.contract.Constants.TO; @@ -867,6 +869,7 @@ public static HttpCarbonMessage createInboundReqCarbonMsg(HttpRequest httpReques ctx.channel().attr(Constants.MUTUAL_SSL_RESULT_ATTRIBUTE).get()); inboundRequestMsg.setProperty(BASE_64_ENCODED_CERT, ctx.channel().attr(Constants.BASE_64_ENCODED_CERT_ATTRIBUTE).get()); + inboundRequestMsg.setProperty(OUTBOUND_ACCESS_LOG_MESSAGES, new ArrayList<>()); return inboundRequestMsg; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java index a18998bdcd..9cba0c95cf 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/HttpServerChannelInitializer.java @@ -218,9 +218,6 @@ public void configureHttpPipeline(ChannelPipeline serverPipeline, String initial if (httpTraceLogEnabled) { serverPipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } -// if (httpAccessLogEnabled) { -// serverPipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); -// } } serverPipeline.addLast(URI_HEADER_LENGTH_VALIDATION_HANDLER, new UriAndHeaderLengthValidator(this.serverName)); if (reqSizeValidationConfig.getMaxEntityBodySize() > -1) { @@ -265,9 +262,6 @@ private void configureH2cPipeline(ChannelPipeline pipeline) { pipeline.addLast(HTTP_TRACE_LOG_HANDLER, new HttpTraceLoggingHandler(TRACE_LOG_DOWNSTREAM)); } -// if (httpAccessLogEnabled) { -// pipeline.addLast(HTTP_ACCESS_LOG_HANDLER, new HttpAccessLoggingHandler(ACCESS_LOG)); -// } final HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java index 75eb3c1dd0..835bfd5995 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/SourceHandler.java @@ -19,7 +19,6 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.config.ChunkConfig; @@ -49,8 +48,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.Queue; @@ -93,7 +90,6 @@ public class SourceHandler extends ChannelInboundHandlerAdapter { private long sequenceId = 1L; //Keep track of the request order for http 1.1 pipelining private final Queue holdingQueue = new PriorityQueue<>(NUMBER_OF_INITIAL_EVENTS_HELD); private EventExecutorGroup pipeliningGroup; - private List httpAccessLogMessages; public SourceHandler(ServerConnectorFuture serverConnectorFuture, HttpServerChannelInitializer serverChannelInitializer, String interfaceId, @@ -113,7 +109,6 @@ public SourceHandler(ServerConnectorFuture serverConnectorFuture, this.pipeliningEnabled = pipeliningEnabled; this.pipeliningLimit = pipeliningLimit; this.pipeliningGroup = pipeliningGroup; - this.httpAccessLogMessages = new ArrayList<>(); } @SuppressWarnings("unchecked") @@ -364,12 +359,4 @@ public void removeRequestEntry(HttpCarbonMessage inboundRequestMsg) { public void resetInboundRequestMsg() { this.inboundRequestMsg = null; } - - public void addHttpAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage) { - this.httpAccessLogMessages.add(httpAccessLogMessage); - } - - public List getHttpAccessLogMessages() { - return this.httpAccessLogMessages; - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index 2c1fc3e873..e9b106c501 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -57,9 +57,9 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_HEAD_METHOD; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_TO_HOST_CONNECTION_CLOSED; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.createFullHttpResponse; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.setupContentLengthRequest; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.ILLEGAL_STATE_ERROR; @@ -312,7 +312,7 @@ private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessag referrer = headers.get(HttpHeaderNames.REFERER); } String method = inboundRequestMsg.getHttpMethod(); - String uri = (String) inboundRequestMsg.getProperty(TO); + String uri = inboundRequestMsg.getRequestUrl(); HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); String protocol; if (request != null) { @@ -330,7 +330,7 @@ private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessag inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); inboundMessage.setRequestTime(requestTime); - List outboundMessages = new ArrayList<>(sourceHandler.getHttpAccessLogMessages()); + List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); HttpAccessLogger.log(inboundMessage, outboundMessages); } @@ -340,4 +340,19 @@ private void resetOutboundListenerState() { contentLength = 0; headersWritten = false; } + + private List getHttpAccessLogMessages(HttpCarbonMessage request) { + Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + for (Object item : rawList) { + if (!(item instanceof HttpAccessLogMessage)) { + return null; + } + } + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return null; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index 7fe4eeae0e..df188054e1 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -61,7 +61,6 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.SRC_HANDLER; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; /** @@ -239,7 +238,7 @@ private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessag referrer = headers.get(HttpHeaderNames.REFERER); } String method = inboundRequestMsg.getHttpMethod(); - String uri = (String) inboundRequestMsg.getProperty(TO); + String uri = inboundRequestMsg.getRequestUrl(); HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); String protocol; if (request != null) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java index 20734fbfe5..580f5cfb9a 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java @@ -78,6 +78,7 @@ public class HttpClientChannelInitializer extends ChannelInitializer outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); + if (outboundAccessLogMessages != null) { + outboundAccessLogMessages.add(outboundAccessLogMessage); + } } else if (http2SourceHandler != null) { http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); } @@ -199,4 +208,19 @@ private T getTypedProperty(HttpCarbonMessage request, String propertyName, C } return null; } + + private List getHttpAccessLogMessages(HttpCarbonMessage request) { + Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + for (Object item : rawList) { + if (!(item instanceof HttpAccessLogMessage)) { + return null; + } + } + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return null; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index 0783b58930..bf48ea035a 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -47,9 +47,12 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Calendar; +import java.util.List; +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_GOAWAY_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_RST_STREAM_WHILE_READING_INBOUND_RESPONSE_BODY; @@ -161,7 +164,9 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun onResponseDataRead(outboundMsgHolder, streamId, endOfStream, data); } if (endOfStream) { - updateAccessLogInfo(outboundMsgHolder); + if (http2TargetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { + updateAccessLogInfo(outboundMsgHolder); + } http2MessageStateContext.setSenderState(new EntityBodyReceived(http2TargetHandler, http2RequestWriter)); } } @@ -244,12 +249,16 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { outboundAccessLogMessage.getDateTime().getTimeInMillis(); outboundAccessLogMessage.setRequestTime(requestTime); - SourceHandler srcHandler = getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, SourceHandler.class); + HttpCarbonMessage inboundReqMsg = + getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); Http2SourceHandler http2SourceHandler = getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); - if (srcHandler != null) { - srcHandler.addHttpAccessLogMessage(outboundAccessLogMessage); + if (inboundReqMsg != null) { + List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); + if (outboundAccessLogMessages != null) { + outboundAccessLogMessages.add(outboundAccessLogMessage); + } } else if (http2SourceHandler != null) { http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); } @@ -262,4 +271,19 @@ private T getTypedProperty(HttpCarbonMessage request, String propertyName, C } return null; } + + private List getHttpAccessLogMessages(HttpCarbonMessage request) { + Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + for (Object item : rawList) { + if (!(item instanceof HttpAccessLogMessage)) { + return null; + } + } + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return null; + } } diff --git a/native/src/test/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfigurationTest.java b/native/src/test/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfigurationTest.java index bbce59f66c..de2ee3e653 100644 --- a/native/src/test/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfigurationTest.java +++ b/native/src/test/java/io/ballerina/stdlib/http/transport/contract/config/SenderConfigurationTest.java @@ -57,6 +57,15 @@ public void testIsHttpTraceLogEnabled() { Assert.assertTrue(senderConfiguration.isHttpTraceLogEnabled()); } + @Test + public void testIsHttpAccessLogEnabled() { + SenderConfiguration senderConfiguration = new SenderConfiguration(); + Assert.assertFalse(senderConfiguration.isHttpAccessLogEnabled()); + + senderConfiguration.setHttpAccessLogEnabled(true); + Assert.assertTrue(senderConfiguration.isHttpAccessLogEnabled()); + } + @Test public void testGetHttpVersion() { SenderConfiguration senderConfiguration = new SenderConfiguration(); From 0cc3b2a9691679679964c7e6d111fc373bf852e7 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Mon, 10 Jun 2024 22:21:00 +0530 Subject: [PATCH 04/16] Fix HTTP/2.0 stacked logging --- ballerina/Ballerina.toml | 6 ++--- ballerina/CompilerPlugin.toml | 2 +- ballerina/Dependencies.toml | 4 +-- .../logging/accesslog/HttpAccessLogger.java | 14 +++++----- .../common/states/Http2StateUtil.java | 3 +++ .../listener/states/SendingEntityBody.java | 4 --- .../states/http2/SendingEntityBody.java | 27 ++++++++++++------- .../sender/HttpClientChannelInitializer.java | 2 +- .../sender/states/ReceivingEntityBody.java | 15 +++-------- .../states/http2/ReceivingEntityBody.java | 15 +++-------- 10 files changed, 42 insertions(+), 50 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 72331b2559..7570a49bbf 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "http" -version = "2.11.2" +version = "2.11.3" authors = ["Ballerina"] keywords = ["http", "network", "service", "listener", "client"] repository = "https://github.com/ballerina-platform/module-ballerina-http" @@ -16,8 +16,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.stdlib" artifactId = "http-native" -version = "2.11.2" -path = "../native/build/libs/http-native-2.11.2.jar" +version = "2.11.3" +path = "../native/build/libs/http-native-2.11.3-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "io.ballerina.stdlib" diff --git a/ballerina/CompilerPlugin.toml b/ballerina/CompilerPlugin.toml index d766246cdc..1f6001a559 100644 --- a/ballerina/CompilerPlugin.toml +++ b/ballerina/CompilerPlugin.toml @@ -3,4 +3,4 @@ id = "http-compiler-plugin" class = "io.ballerina.stdlib.http.compiler.HttpCompilerPlugin" [[dependency]] -path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.2.jar" +path = "../compiler-plugin/build/libs/http-compiler-plugin-2.11.3-SNAPSHOT.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 4def31d650..00e895c46a 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -50,7 +50,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.7.1" +version = "2.7.0" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} @@ -76,7 +76,7 @@ modules = [ [[package]] org = "ballerina" name = "http" -version = "2.11.2" +version = "2.11.3" dependencies = [ {org = "ballerina", name = "auth"}, {org = "ballerina", name = "cache"}, diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index f1b93bc2fb..b7ee0c5d5c 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -27,7 +27,6 @@ import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -52,10 +51,6 @@ public class HttpAccessLogger { private HttpAccessLogger() {} - public static boolean isEnabled() { - return ACCESS_LOGGER.isEnabled(InternalLogLevel.INFO); - } - public static void log(HttpAccessLogMessage inboundMessage, List outboundMessages) { String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, HttpAccessLogConfig.getInstance().getAccessLogFormat(), @@ -156,8 +151,13 @@ private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage http String attribute) { Map customHeaders = httpAccessLogMessage.getCustomHeaders(); if (attribute.startsWith("http_")) { - String customHeaderKey = attribute.substring(5).toLowerCase(Locale.getDefault()); - return customHeaders.getOrDefault(customHeaderKey, "-"); + String customHeaderKey = attribute.substring(5); + for (Map.Entry entry : customHeaders.entrySet()) { + if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { + return entry.getValue(); + } + } + return "-"; } return null; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java index e72fd6ae8a..4ed156e017 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/common/states/Http2StateUtil.java @@ -63,6 +63,7 @@ import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.util.ArrayList; import static io.ballerina.stdlib.http.transport.contract.Constants.BASE_64_ENCODED_CERT; import static io.ballerina.stdlib.http.transport.contract.Constants.CHNL_HNDLR_CTX; @@ -74,6 +75,7 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.LISTENER_PORT; import static io.ballerina.stdlib.http.transport.contract.Constants.LOCAL_ADDRESS; import static io.ballerina.stdlib.http.transport.contract.Constants.MUTUAL_SSL_HANDSHAKE_RESULT; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.POOLED_BYTE_BUFFER_FACTORY; import static io.ballerina.stdlib.http.transport.contract.Constants.PROMISED_STREAM_REJECTED_ERROR; import static io.ballerina.stdlib.http.transport.contract.Constants.PROTOCOL; @@ -158,6 +160,7 @@ public static HttpCarbonRequest setupCarbonRequest(HttpRequest httpRequest, Http String uri = httpRequest.uri(); sourceReqCMsg.setRequestUrl(uri); sourceReqCMsg.setProperty(TO, uri); + sourceReqCMsg.setProperty(OUTBOUND_ACCESS_LOG_MESSAGES, new ArrayList<>()); return sourceReqCMsg; } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index e9b106c501..1a63c36814 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -290,10 +290,6 @@ private void triggerPipeliningLogic(HttpCarbonMessage outboundResponseMsg) { } private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { - if (!HttpAccessLogger.isEnabled()) { - return; - } - HttpHeaders headers = inboundRequestMsg.getHeaders(); if (headers.contains(HTTP_X_FORWARDED_FOR)) { String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index df188054e1..865ed1d80b 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -53,14 +53,13 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.SRC_HANDLER; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; /** @@ -213,9 +212,6 @@ private void writeData(HttpContent httpContent, int streamId, boolean endStream) private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg, int streamId) { - if (!HttpAccessLogger.isEnabled()) { - return; - } if (originalStreamId != streamId) { // Skip access logs for server push messages LOG.debug("Access logging skipped for server push response"); return; @@ -256,12 +252,23 @@ private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessag inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); inboundMessage.setRequestTime(requestTime); - List outboundMessages = new ArrayList<>(); - Object sourceHandlerObject = inboundRequestMsg.getProperty(SRC_HANDLER); + List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); - if (sourceHandlerObject instanceof Http2SourceHandler http2SourceHandler) { - outboundMessages.addAll(http2SourceHandler.getHttpAccessLogMessages()); - } HttpAccessLogger.log(inboundMessage, outboundMessages); } + + private List getHttpAccessLogMessages(HttpCarbonMessage request) { + Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + for (Object item : rawList) { + if (!(item instanceof HttpAccessLogMessage)) { + return null; + } + } + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return null; + } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java index 580f5cfb9a..565e624895 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/HttpClientChannelInitializer.java @@ -124,7 +124,6 @@ public HttpClientChannelInitializer(SenderConfiguration senderConfiguration, Htt connectionHandlerBuilder.initialSettings().initialWindowSize(senderConfiguration.getHttp2InitialWindowSize()); http2ConnectionHandler = connectionHandlerBuilder.connection(connection).frameListener(frameListener).build(); http2TargetHandler = new Http2TargetHandler(connection, http2ConnectionHandler.encoder()); - http2TargetHandler.setHttpClientChannelInitializer(this); if (sslConfig != null) { sslHandlerFactory = new SSLHandlerFactory(sslConfig); } @@ -140,6 +139,7 @@ protected void initChannel(SocketChannel socketChannel) throws Exception { targetHandler.setHttp2TargetHandler(http2TargetHandler); targetHandler.setKeepAliveConfig(getKeepAliveConfig()); targetHandler.setHttpClientChannelInitializer(this); + http2TargetHandler.setHttpClientChannelInitializer(this); if (http2) { if (sslConfig != null) { configureSslForHttp2(socketChannel, clientPipeline, sslConfig); diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index f38bbefbab..f9714e9e44 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -20,12 +20,9 @@ import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contractimpl.common.states.SenderReqRespStateManager; import io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil; -import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; -import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.TargetHandler; import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; import io.netty.channel.ChannelHandlerContext; @@ -173,11 +170,11 @@ private void updateAccessLogInfo(TargetHandler targetHandler, outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage outboundRequest = httpOutboundRequest.getNettyHttpRequest(); - if (outboundRequest != null) { - outboundAccessLogMessage.setScheme(outboundRequest.protocolVersion().toString()); + HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); + if (inboundResponse != null) { + outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); } else { - outboundAccessLogMessage.setScheme(httpOutboundRequest.getHttpVersion()); + outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); } outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); @@ -188,16 +185,12 @@ private void updateAccessLogInfo(TargetHandler targetHandler, HttpCarbonMessage inboundReqMsg = getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - Http2SourceHandler http2SourceHandler = - getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); if (inboundReqMsg != null) { List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); if (outboundAccessLogMessages != null) { outboundAccessLogMessages.add(outboundAccessLogMessage); } - } else if (http2SourceHandler != null) { - http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index bf48ea035a..d165ce8a34 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -20,10 +20,7 @@ import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; -import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; -import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2ClientChannel; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2TargetHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.OutboundMsgHolder; @@ -236,11 +233,11 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage outboundRequest = httpOutboundRequest.getNettyHttpRequest(); - if (outboundRequest != null) { - outboundAccessLogMessage.setScheme(outboundRequest.protocolVersion().toString()); + HttpMessage inboundResponse = outboundMsgHolder.getResponse().getNettyHttpResponse(); + if (inboundResponse != null) { + outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); } else { - outboundAccessLogMessage.setScheme(httpOutboundRequest.getHttpVersion()); + outboundAccessLogMessage.setScheme(outboundMsgHolder.getResponse().getHttpVersion()); } outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); outboundAccessLogMessage.setStatus(outboundMsgHolder.getResponse().getHttpStatusCode()); @@ -251,16 +248,12 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { HttpCarbonMessage inboundReqMsg = getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - Http2SourceHandler http2SourceHandler = - getTypedProperty(httpOutboundRequest, Constants.SRC_HANDLER, Http2SourceHandler.class); if (inboundReqMsg != null) { List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); if (outboundAccessLogMessages != null) { outboundAccessLogMessages.add(outboundAccessLogMessage); } - } else if (http2SourceHandler != null) { - http2SourceHandler.addHttpAccessLogMessage(outboundAccessLogMessage); } } From 5dc1c06da200734480d45acc2f5c436a1624c40a Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Tue, 18 Jun 2024 11:49:19 +0530 Subject: [PATCH 05/16] Update changelog --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 748357226c..fc33420744 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - [Introduce default status code response record](https://github.com/ballerina-platform/ballerina-library/issues/6491) +- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options. The update support both JSON and flat logging formats](https://github.com/ballerina-platform/ballerina-library/issues/6111) ## [2.11.2] - 2024-06-14 From c8d1f1b575d65f5f76cc8fee02e5ee847a0a85b1 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Wed, 19 Jun 2024 09:39:55 +0530 Subject: [PATCH 06/16] Update specs --- ballerina/Dependencies.toml | 2 +- docs/spec/spec.md | 41 ++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 00e895c46a..ee2ab8a1f1 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -50,7 +50,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.7.0" +version = "2.7.2" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 63cc4c3637..d25e8f7f5d 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -2498,20 +2498,47 @@ path = "testTraceLog.txt" # Optional host = "localhost" # Optional port = 8080 # Optional ``` + #### 8.2.4 Access log +Ballerina supports HTTP access logs for HTTP services, providing insights into web traffic and request handling. +The access log feature is **disabled by default** to allow users to opt-in as per their requirements. -Ballerina supports HTTP access logs for HTTP services. The access log format used is the combined log format. -The HTTP access logs are **disabled as default**. -To enable access logs, set console=true under the ballerina.http.accessLogConfig in the Config.toml file. Also, -the path field can be used to specify the file path to save the access logs. +To enable access logs, configuration settings are provided under `ballerina.http.accessLogConfig` in the +`Config.toml` file. Users can specify whether logs should be output to the console, a file, or both, +and can select the format and specific attributes to log. ```toml [ballerina.http.accessLogConfig] # Enable printing access logs in console console = true # Default is false -# Specify the file path to save the access logs -path = "testAccessLog.txt" # Optional -``` +# Specify the file path to save the access logs +path = "testAccessLog.txt" # Optional, omit to disable file logging +# Select the format of the access logs +format = "json" # Options: "flat", "json"; Default is "flat". Omit to stick to the default. +# Specify which attributes to log. Omit to stick to the default set. +attributes = ["ip", "date_time", "request", "status", "response_body_size", "http_referrer", "http_user_agent"] +# Default attributes: ip, date_time, request, status, response_body_size, http_referrer, http_user_agent +``` +##### Configurable Attributes +Users can customize which parts of the access data are logged by specifying attributes in the configuration. +This allows for tailored logging that can focus on particular details relevant to the users' needs. + +| Attribute | Description | +|:----------------------:|:---------------------------------------------------:| +| ip | Client's IP address | +| date_time | HTTP request received time | +| request | Full HTTP request line (method, URI, protocol) | +| request_method | HTTP method of the request | +| request_uri | URI of the request, including parameters | +| scheme | Scheme of the request and HTTP version | +| status | HTTP status code returned to the client | +| request_body_size | Size of the request body in bytes | +| response_body_size | Size of the HTTP response body in bytes | +| request_time | Total time taken to process the request | +| http_referrer | HTTP Referer header, indicating the previous page | +| http_user_agent | User-Agent header, identifying the client software | +| http_x_forwarded_for | Originating IP address if using a proxy | +| http_(X-Custom-Header) | Header fields. Referring to them with `http` followed by the header name. (`x-request-id` ->; `http_x-request-id`) | #### 8.2.5 Panic inside resource From 4fe4e517107f665ef2c12a1b87a6da623623cbe7 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Wed, 19 Jun 2024 12:36:45 +0530 Subject: [PATCH 07/16] Address review comments --- .../accesslog/HttpAccessLogConfig.java | 23 ++++++++- .../logging/accesslog/HttpAccessLogUtil.java | 50 +++++++++++++++++++ .../listener/states/SendingEntityBody.java | 17 +------ .../states/http2/SendingEntityBody.java | 17 +------ .../sender/states/ReceivingEntityBody.java | 26 +--------- .../states/http2/ReceivingEntityBody.java | 26 +--------- 6 files changed, 77 insertions(+), 82 deletions(-) create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java index e0c4d6e7b9..5c02a38d68 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.stdlib.http.api.logging.accesslog; import io.ballerina.runtime.api.values.BArray; @@ -21,7 +39,8 @@ public class HttpAccessLogConfig { private static final HttpAccessLogConfig instance = new HttpAccessLogConfig(); - private static final Set EXCLUDED_ATTRIBUTES = new HashSet<>(List.of( + + private final Set excludedAttributes = new HashSet<>(List.of( ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR )); private BMap accessLogConfig; @@ -43,7 +62,7 @@ public List getCustomHeaders() { } return attributes.stream() - .filter(attr -> attr.startsWith("http_") && !EXCLUDED_ATTRIBUTES.contains(attr)) + .filter(attr -> attr.startsWith("http_") && !excludedAttributes.contains(attr)) .map(attr -> attr.substring(5)) .collect(Collectors.toList()); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java new file mode 100644 index 0000000000..f8421623a4 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; + +import java.util.List; + +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; + +public class HttpAccessLogUtil { + public static T getTypedProperty(HttpCarbonMessage carbonMessage, String propertyName, Class type) { + Object property = carbonMessage.getProperty(propertyName); + if (type.isInstance(property)) { + return type.cast(property); + } + return null; + } + + public static List getHttpAccessLogMessages(HttpCarbonMessage carbonMessage) { + Object outboundAccessLogMessagesObject = carbonMessage.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); + if (outboundAccessLogMessagesObject instanceof List rawList) { + for (Object item : rawList) { + if (!(item instanceof HttpAccessLogMessage)) { + return null; + } + } + @SuppressWarnings("unchecked") + List outboundAccessLogMessages = (List) rawList; + return outboundAccessLogMessages; + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index 1a63c36814..a06e5946d8 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -54,10 +54,10 @@ import java.util.List; import java.util.Queue; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_HEAD_METHOD; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_TO_HOST_CONNECTION_CLOSED; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.createFullHttpResponse; @@ -336,19 +336,4 @@ private void resetOutboundListenerState() { contentLength = 0; headersWritten = false; } - - private List getHttpAccessLogMessages(HttpCarbonMessage request) { - Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); - if (outboundAccessLogMessagesObject instanceof List rawList) { - for (Object item : rawList) { - if (!(item instanceof HttpAccessLogMessage)) { - return null; - } - } - @SuppressWarnings("unchecked") - List outboundAccessLogMessages = (List) rawList; - return outboundAccessLogMessages; - } - return null; - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index 865ed1d80b..dc9d3735cd 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -56,9 +56,9 @@ import java.util.Calendar; import java.util.List; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; @@ -256,19 +256,4 @@ private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessag HttpAccessLogger.log(inboundMessage, outboundMessages); } - - private List getHttpAccessLogMessages(HttpCarbonMessage request) { - Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); - if (outboundAccessLogMessagesObject instanceof List rawList) { - for (Object item : rawList) { - if (!(item instanceof HttpAccessLogMessage)) { - return null; - } - } - @SuppressWarnings("unchecked") - List outboundAccessLogMessages = (List) rawList; - return outboundAccessLogMessages; - } - return null; - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index f9714e9e44..75143910e5 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -42,11 +42,12 @@ import java.util.List; import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_STATE_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.isKeepAlive; @@ -193,27 +194,4 @@ private void updateAccessLogInfo(TargetHandler targetHandler, } } } - - private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { - Object property = request.getProperty(propertyName); - if (type.isInstance(property)) { - return type.cast(property); - } - return null; - } - - private List getHttpAccessLogMessages(HttpCarbonMessage request) { - Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); - if (outboundAccessLogMessagesObject instanceof List rawList) { - for (Object item : rawList) { - if (!(item instanceof HttpAccessLogMessage)) { - return null; - } - } - @SuppressWarnings("unchecked") - List outboundAccessLogMessages = (List) rawList; - return outboundAccessLogMessages; - } - return null; - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index d165ce8a34..a92b649830 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -47,9 +47,10 @@ import java.util.List; import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_GOAWAY_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_RST_STREAM_WHILE_READING_INBOUND_RESPONSE_BODY; @@ -256,27 +257,4 @@ private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { } } } - - private T getTypedProperty(HttpCarbonMessage request, String propertyName, Class type) { - Object property = request.getProperty(propertyName); - if (type.isInstance(property)) { - return type.cast(property); - } - return null; - } - - private List getHttpAccessLogMessages(HttpCarbonMessage request) { - Object outboundAccessLogMessagesObject = request.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); - if (outboundAccessLogMessagesObject instanceof List rawList) { - for (Object item : rawList) { - if (!(item instanceof HttpAccessLogMessage)) { - return null; - } - } - @SuppressWarnings("unchecked") - List outboundAccessLogMessages = (List) rawList; - return outboundAccessLogMessages; - } - return null; - } } From ebb1e7bf868b9d30d19580a303196bb25e6cf17c Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Wed, 19 Jun 2024 13:31:30 +0530 Subject: [PATCH 08/16] Warn on unsupported log-formats --- .../java/io/ballerina/stdlib/http/api/HttpConstants.java | 1 + .../ballerina/stdlib/http/api/logging/HttpLogManager.java | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java index 52ac12826e..cf094d30b4 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/HttpConstants.java @@ -326,6 +326,7 @@ public final class HttpConstants { public static final String HTTP_ACCESS_LOG = "http.accesslog"; public static final String HTTP_ACCESS_LOG_ENABLED = "http.accesslog.enabled"; public static final String HTTP_LOG_FORMAT_JSON = "json"; + public static final String HTTP_LOG_FORMAT_FLAT = "flat"; // TraceLog and AccessLog configs public static final BString HTTP_LOG_CONSOLE = StringUtils.fromString("console"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java index 5a12d023fa..54f4820986 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java @@ -39,6 +39,9 @@ import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG_ENABLED; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_CONSOLE; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FILE_PATH; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_FLAT; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_ENABLED; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_TRACE_LOG_HOST; @@ -163,6 +166,11 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) { } } + BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); + if (!(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) || logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) { + stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() + "'. Defaulting to 'flat'."); + } + if (accessLogsEnabled) { System.setProperty(HTTP_ACCESS_LOG_ENABLED, "true"); stdErr.println("ballerina: " + protocol + " access log enabled"); From bde27d66d48bdbf4b1b9175b0e1f1574f28aa71c Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Wed, 19 Jun 2024 14:16:21 +0530 Subject: [PATCH 09/16] Address review comments --- ballerina/Dependencies.toml | 2 +- changelog.md | 2 +- docs/spec/spec.md | 1 + .../stdlib/http/api/logging/HttpLogManager.java | 3 ++- .../listener/http2/Http2SourceHandler.java | 13 ------------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index ee2ab8a1f1..00e895c46a 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -50,7 +50,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.7.2" +version = "2.7.0" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} diff --git a/changelog.md b/changelog.md index fc33420744..2929fb5a25 100644 --- a/changelog.md +++ b/changelog.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - [Introduce default status code response record](https://github.com/ballerina-platform/ballerina-library/issues/6491) -- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options. The update support both JSON and flat logging formats](https://github.com/ballerina-platform/ballerina-library/issues/6111) +- [Enhanced the configurability of Ballerina access logging by introducing multiple configuration options.](https://github.com/ballerina-platform/ballerina-library/issues/6111) ## [2.11.2] - 2024-06-14 diff --git a/docs/spec/spec.md b/docs/spec/spec.md index d25e8f7f5d..29339bcc0c 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -2519,6 +2519,7 @@ format = "json" # Options: "flat", "json"; Default is "flat". Omit t attributes = ["ip", "date_time", "request", "status", "response_body_size", "http_referrer", "http_user_agent"] # Default attributes: ip, date_time, request, status, response_body_size, http_referrer, http_user_agent ``` + ##### Configurable Attributes Users can customize which parts of the access data are logged by specifying attributes in the configuration. This allows for tailored logging that can focus on particular details relevant to the users' needs. diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java index 54f4820986..e16a74b234 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java @@ -168,7 +168,8 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) { BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); if (!(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) || logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) { - stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() + "'. Defaulting to 'flat'."); + stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() + + "'. Defaulting to 'flat' format."); } if (accessLogsEnabled) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java index 4ea94255dc..b419b326d9 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/http2/Http2SourceHandler.java @@ -18,7 +18,6 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.http2; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -51,8 +50,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -84,7 +81,6 @@ public final class Http2SourceHandler extends ChannelInboundHandlerAdapter { private SocketAddress remoteAddress; private ChannelGroup allChannels; private ChannelGroup listenerChannels; - private List httpAccessLogMessages; Http2SourceHandler(HttpServerChannelInitializer serverChannelInitializer, Http2ConnectionEncoder encoder, String interfaceId, Http2Connection conn, ServerConnectorFuture serverConnectorFuture, @@ -98,7 +94,6 @@ public final class Http2SourceHandler extends ChannelInboundHandlerAdapter { this.targetChannelPool = new ConcurrentHashMap<>(); this.allChannels = allChannels; this.listenerChannels = listenerChannels; - this.httpAccessLogMessages = new ArrayList<>(); setRemoteFlowController(); setDataEventListeners(); } @@ -296,12 +291,4 @@ public Http2ServerChannel getHttp2ServerChannel() { public SocketAddress getRemoteAddress() { return remoteAddress; } - - public void addHttpAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage) { - this.httpAccessLogMessages.add(httpAccessLogMessage); - } - - public List getHttpAccessLogMessages() { - return this.httpAccessLogMessages; - } } From 35870c55503da88317058dd45c3c8628b301135b Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Fri, 28 Jun 2024 12:03:58 +0530 Subject: [PATCH 10/16] Address review comments --- .../stdlib/http/api/logging/HttpLogManager.java | 11 ++++++++++- .../api/logging/accesslog/HttpAccessLogConfig.java | 5 +---- .../http/api/logging/accesslog/HttpAccessLogger.java | 6 +++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java index e16a74b234..d3f39210ef 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/HttpLogManager.java @@ -18,6 +18,7 @@ package io.ballerina.stdlib.http.api.logging; +import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; @@ -37,6 +38,7 @@ import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_ACCESS_LOG_ENABLED; +import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_ATTRIBUTES; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_CONSOLE; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FILE_PATH; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; @@ -167,11 +169,18 @@ public void setHttpAccessLogHandler(BMap accessLogConfig) { } BString logFormat = accessLogConfig.getStringValue(HTTP_LOG_FORMAT); - if (!(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) || logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) { + if (logFormat != null && + !(logFormat.getValue().equals(HTTP_LOG_FORMAT_JSON) || + logFormat.getValue().equals(HTTP_LOG_FORMAT_FLAT))) { stdErr.println("WARNING: Unsupported log format '" + logFormat.getValue() + "'. Defaulting to 'flat' format."); } + BArray logAttributes = accessLogConfig.getArrayValue(HTTP_LOG_ATTRIBUTES); + if (logAttributes != null && logAttributes.getLength() == 0) { + accessLogsEnabled = false; + } + if (accessLogsEnabled) { System.setProperty(HTTP_ACCESS_LOG_ENABLED, "true"); stdErr.println("ballerina: " + protocol + " access log enabled"); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java index 5c02a38d68..d48d0f4c91 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -57,9 +57,6 @@ public void initializeHttpAccessLogConfig(BMap accessLogConfig) { public List getCustomHeaders() { List attributes = getAccessLogAttributes(); - if (attributes == null) { - return Collections.emptyList(); - } return attributes.stream() .filter(attr -> attr.startsWith("http_") && !excludedAttributes.contains(attr)) @@ -85,6 +82,6 @@ public List getAccessLogAttributes() { .collect(Collectors.toList()); } } - return null; + return Collections.emptyList(); } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index b7ee0c5d5c..468739fa00 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -61,8 +61,9 @@ public static void log(HttpAccessLogMessage inboundMessage, List outboundMessages, HttpAccessLogFormat format, List attributes) { + + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); if (format == HttpAccessLogFormat.FLAT) { - Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); String inboundFormatted = inboundMap.values().stream() .filter(Objects::nonNull) .collect(Collectors.joining(" ")); @@ -80,7 +81,6 @@ private static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage return inboundFormatted; } } else { - Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); Gson gson = new Gson(); JsonObject jsonObject = new JsonObject(); @@ -111,7 +111,7 @@ private static Map mapAccessLogMessage(HttpAccessLogMessage http Map attributeValues = new LinkedHashMap<>(); allAttributes.forEach(attr -> attributeValues.put(attr, null)); - if (attributes != null) { + if (!attributes.isEmpty()) { attributes.forEach(attr -> { attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); }); From 4382814906a9aac3a6713f4d7ad854678e9315dd Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Tue, 2 Jul 2024 14:17:58 +0530 Subject: [PATCH 11/16] Extract log related logics from states --- .../accesslog/HttpAccessLogFormatter.java | 156 ++++++++++++++++++ .../logging/accesslog/HttpAccessLogger.java | 148 +---------------- .../accesslog/ListenerHttpAccessLogger.java | 115 +++++++++++++ .../accesslog/SenderHttpAccessLogger.java | 125 ++++++++++++++ .../listener/states/SendingEntityBody.java | 61 +------ .../states/http2/SendingEntityBody.java | 83 ++-------- .../sender/states/ReceivingEntityBody.java | 100 ++--------- .../states/http2/ReceivingEntityBody.java | 97 ++--------- 8 files changed, 445 insertions(+), 440 deletions(-) create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java new file mode 100644 index 0000000000..79d5d5e4dc --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; + +public class HttpAccessLogFormatter { + + private HttpAccessLogFormatter() {} + + public static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, + List outboundMessages, HttpAccessLogFormat format, + List attributes) { + + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); + if (format == HttpAccessLogFormat.FLAT) { + String inboundFormatted = inboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" ")); + + if (!outboundMessages.isEmpty()) { + String outboundFormatted = outboundMessages.stream() + .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) + .map(outboundMap -> outboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" "))) + .collect(Collectors.joining(" ")); + + return inboundFormatted + " \"~\" " + outboundFormatted; + } else { + return inboundFormatted; + } + } else { + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + + inboundMap.forEach(jsonObject::addProperty); + + if (!outboundMessages.isEmpty()) { + JsonArray upstreamArray = new JsonArray(); + for (HttpAccessLogMessage outboundMessage : outboundMessages) { + Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); + JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); + upstreamArray.add(outboundJson); + } + jsonObject.add("upstream", upstreamArray); + } + return gson.toJson(jsonObject); + } + } + + private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, List attributes) { + List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, + ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, + ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); + List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, + ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); + + Map attributeValues = new LinkedHashMap<>(); + allAttributes.forEach(attr -> attributeValues.put(attr, null)); + + if (!attributes.isEmpty()) { + attributes.forEach(attr -> { + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); + }); + } else { + defaultAttributes.forEach(attr -> + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); + } + return attributeValues; + } + + private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, String attribute) { + return switch (attribute) { + case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); + case ATTRIBUTE_DATE_TIME -> + String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); + case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); + case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); + case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); + case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), + httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); + case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); + case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); + case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); + case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); + case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); + case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); + case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); + default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); + }; + } + + private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, + String attribute) { + Map customHeaders = httpAccessLogMessage.getCustomHeaders(); + if (attribute.startsWith("http_")) { + String customHeaderKey = attribute.substring(5); + for (Map.Entry entry : customHeaders.entrySet()) { + if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { + return entry.getValue(); + } + } + return "-"; + } + return null; + } + + private static String getHyphenForNull(String value) { + return value == null ? "-" : value; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index 468739fa00..1fde2e3267 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -18,151 +18,11 @@ package io.ballerina.stdlib.http.api.logging.accesslog; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.netty.util.internal.logging.InternalLogLevel; -import io.netty.util.internal.logging.InternalLogger; -import io.netty.util.internal.logging.InternalLoggerFactory; +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; +public interface HttpAccessLogger { -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; + void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); -public class HttpAccessLogger { - private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); - - private HttpAccessLogger() {} - - public static void log(HttpAccessLogMessage inboundMessage, List outboundMessages) { - String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, - HttpAccessLogConfig.getInstance().getAccessLogFormat(), - HttpAccessLogConfig.getInstance().getAccessLogAttributes()); - ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); - } - - private static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, - List outboundMessages, HttpAccessLogFormat format, - List attributes) { - - Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); - if (format == HttpAccessLogFormat.FLAT) { - String inboundFormatted = inboundMap.values().stream() - .filter(Objects::nonNull) - .collect(Collectors.joining(" ")); - - if (!outboundMessages.isEmpty()) { - String outboundFormatted = outboundMessages.stream() - .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) - .map(outboundMap -> outboundMap.values().stream() - .filter(Objects::nonNull) - .collect(Collectors.joining(" "))) - .collect(Collectors.joining(" ")); - - return inboundFormatted + " \"~\" " + outboundFormatted; - } else { - return inboundFormatted; - } - } else { - Gson gson = new Gson(); - JsonObject jsonObject = new JsonObject(); - - inboundMap.forEach(jsonObject::addProperty); - - if (!outboundMessages.isEmpty()) { - JsonArray upstreamArray = new JsonArray(); - for (HttpAccessLogMessage outboundMessage : outboundMessages) { - Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); - JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); - upstreamArray.add(outboundJson); - } - jsonObject.add("upstream", upstreamArray); - } - return gson.toJson(jsonObject); - } - } - - private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, - HttpAccessLogFormat format, List attributes) { - List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, - ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, - ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, - ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); - List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, - ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); - - Map attributeValues = new LinkedHashMap<>(); - allAttributes.forEach(attr -> attributeValues.put(attr, null)); - - if (!attributes.isEmpty()) { - attributes.forEach(attr -> { - attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); - }); - } else { - defaultAttributes.forEach(attr -> - attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); - } - return attributeValues; - } - - private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, - HttpAccessLogFormat format, String attribute) { - return switch (attribute) { - case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); - case ATTRIBUTE_DATE_TIME -> - String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); - case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); - case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); - case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); - case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), - httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); - case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); - case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); - case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); - case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); - case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); - case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); - case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); - default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); - }; - } - - private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, - String attribute) { - Map customHeaders = httpAccessLogMessage.getCustomHeaders(); - if (attribute.startsWith("http_")) { - String customHeaderKey = attribute.substring(5); - for (Map.Entry entry : customHeaders.entrySet()) { - if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { - return entry.getValue(); - } - } - return "-"; - } - return null; - } - - private static String getHyphenForNull(String value) { - return value == null ? "-" : value; - } + void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java new file mode 100644 index 0000000000..98594c5ed5 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.contractimpl.common.Util; +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.util.internal.logging.InternalLogLevel; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogFormatter.formatAccessLogMessage; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; + +public class ListenerHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(ListenerHttpAccessLogger.class); + private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); + + private final Calendar inboundRequestArrivalTime; + private Long contentLength = 0L; + private String remoteAddress; + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.remoteAddress = remoteAddress; + } + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, Long contentLength, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.contentLength = contentLength; + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { + HttpHeaders headers = inboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; + } + + // Populate request parameters + String userAgent = "-"; + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + userAgent = headers.get(HttpHeaderNames.USER_AGENT); + } + String referrer = "-"; + if (headers.contains(HttpHeaderNames.REFERER)) { + referrer = headers.get(HttpHeaderNames.REFERER); + } + String method = inboundRequestMsg.getHttpMethod(); + String uri = inboundRequestMsg.getRequestUrl(); + HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); + String protocol; + if (request != null) { + protocol = request.protocolVersion().toString(); + } else { + protocol = inboundRequestMsg.getHttpVersion(); + } + + // Populate response parameters + int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); + + long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); + HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, + inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); + inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); + inboundMessage.setRequestTime(requestTime); + + List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); + + String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, + HttpAccessLogConfig.getInstance().getAccessLogFormat(), + HttpAccessLogConfig.getInstance().getAccessLogAttributes()); + ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("updateAccessLogInfo is not a dependant action of this logger"); + } + + public void updateContentLength(HttpContent httpContent) { + contentLength += httpContent.content().readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java new file mode 100644 index 0000000000..a05915de8f --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; + +public class SenderHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(SenderHttpAccessLogger.class); + + private Long contentLength = 0L; + private final SocketAddress remoteAddress; + + public SenderHttpAccessLogger(SocketAddress remoteAddress) { + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("logAccessInfo is not a dependant action of this logger"); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage outboundRequestMsg, HttpCarbonMessage inboundResponseMsg) { + HttpAccessLogMessage outboundAccessLogMessage = + getTypedProperty(outboundRequestMsg, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); + if (outboundAccessLogMessage == null) { + return; + } + + if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { + InetAddress inetAddress = inetSocketAddress.getAddress(); + outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); + outboundAccessLogMessage.setHost(inetAddress.getHostName()); + outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); + } + if (outboundAccessLogMessage.getIp().startsWith("/")) { + outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); + } + + // Populate with header parameters + HttpHeaders headers = outboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? + forwardedHops.substring(0, firstCommaIndex) : forwardedHops); + } + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); + } + if (headers.contains(HttpHeaderNames.REFERER)) { + outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); + } + HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> + outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? + headers.get(customHeader) : "-")); + + outboundAccessLogMessage.setRequestMethod(outboundRequestMsg.getHttpMethod()); + outboundAccessLogMessage.setRequestUri((String) outboundRequestMsg.getProperty(TO)); + HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); + if (inboundResponse != null) { + outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); + } else { + outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); + } + outboundAccessLogMessage.setRequestBodySize((long) outboundRequestMsg.getContentSize()); + outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); + outboundAccessLogMessage.setResponseBodySize(contentLength); + long requestTime = Calendar.getInstance().getTimeInMillis() - + outboundAccessLogMessage.getDateTime().getTimeInMillis(); + outboundAccessLogMessage.setRequestTime(requestTime); + + HttpCarbonMessage inboundReqMsg = + getTypedProperty(outboundRequestMsg, INBOUND_MESSAGE, HttpCarbonMessage.class); + + if (inboundReqMsg != null) { + List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); + if (outboundAccessLogMessages != null) { + outboundAccessLogMessages.add(outboundAccessLogMessage); + } + } + } + + public void updateContentLength(ByteBuf content) { + contentLength += content.readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index a06e5946d8..51bfda13b8 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -18,14 +18,12 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; import io.ballerina.stdlib.http.transport.contractimpl.HttpOutboundRespListener; -import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; import io.ballerina.stdlib.http.transport.internal.HandlerExecutor; import io.ballerina.stdlib.http.transport.internal.HttpTransportContextHolder; @@ -37,9 +35,6 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; @@ -50,13 +45,10 @@ import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; import java.util.Queue; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_HEAD_METHOD; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_TO_HOST_CONNECTION_CLOSED; @@ -83,8 +75,6 @@ public class SendingEntityBody implements ListenerState { private HttpCarbonMessage outboundResponseMsg; private ChannelHandlerContext sourceContext; private SourceHandler sourceHandler; - private final Calendar inboundRequestArrivalTime; - private String remoteAddress = "-"; SendingEntityBody(HttpOutboundRespListener outboundRespListener, ListenerReqRespStateManager listenerReqRespStateManager, @@ -99,8 +89,6 @@ public class SendingEntityBody implements ListenerState { this.inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); this.sourceContext = outboundRespListener.getSourceContext(); this.sourceHandler = outboundRespListener.getSourceHandler(); - this.inboundRequestArrivalTime = outboundRespListener.getInboundRequestArrivalTime(); - this.remoteAddress = outboundRespListener.getRemoteAddress(); } @Override @@ -238,7 +226,10 @@ private void checkForResponseWriteStatus(HttpCarbonMessage inboundRequestMsg, outboundRespStatusFuture.notifyHttpListener(inboundRequestMsg); } if (sourceHandler.getServerChannelInitializer().isHttpAccessLogEnabled()) { - logAccessInfo(inboundRequestMsg, outboundResponseMsg); + ListenerHttpAccessLogger accessLogger = new ListenerHttpAccessLogger( + outboundRespListener.getInboundRequestArrivalTime(), contentLength, + outboundRespListener.getRemoteAddress()); + accessLogger.logAccessInfo(inboundRequestMsg, outboundResponseMsg); } resetOutboundListenerState(); }); @@ -289,48 +280,6 @@ private void triggerPipeliningLogic(HttpCarbonMessage outboundResponseMsg) { } } - private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { - HttpHeaders headers = inboundRequestMsg.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; - } - - // Populate request parameters - String userAgent = "-"; - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - userAgent = headers.get(HttpHeaderNames.USER_AGENT); - } - String referrer = "-"; - if (headers.contains(HttpHeaderNames.REFERER)) { - referrer = headers.get(HttpHeaderNames.REFERER); - } - String method = inboundRequestMsg.getHttpMethod(); - String uri = inboundRequestMsg.getRequestUrl(); - HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); - String protocol; - if (request != null) { - protocol = request.protocolVersion().toString(); - } else { - protocol = inboundRequestMsg.getHttpVersion(); - } - - // Populate response parameters - int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - - long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); - HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, - inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); - inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); - inboundMessage.setRequestTime(requestTime); - - List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); - - HttpAccessLogger.log(inboundMessage, outboundMessages); - } - private void resetOutboundListenerState() { contentList.clear(); contentLength = 0; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index dc9d3735cd..44b9c969eb 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states.http2; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -27,7 +26,6 @@ import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil; -import io.ballerina.stdlib.http.transport.contractimpl.listener.HttpServerChannelInitializer; import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2DataEventListener; import io.ballerina.stdlib.http.transport.message.Http2DataFrame; @@ -39,9 +37,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.codec.http2.Http2Connection; import io.netty.handler.codec.http2.Http2ConnectionEncoder; @@ -53,11 +49,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Calendar; -import java.util.List; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; @@ -73,32 +65,30 @@ public class SendingEntityBody implements ListenerState { private final Http2MessageStateContext http2MessageStateContext; private final ChannelHandlerContext ctx; - private final HttpServerChannelInitializer serverChannelInitializer; private final Http2Connection conn; private final Http2ConnectionEncoder encoder; private final HttpResponseFuture outboundRespStatusFuture; private final HttpCarbonMessage inboundRequestMsg; - private final Calendar inboundRequestArrivalTime; private final int originalStreamId; private final Http2OutboundRespListener http2OutboundRespListener; private HttpCarbonMessage outboundResponseMsg; - - private Long contentLength = 0L; - private String remoteAddress; + private ListenerHttpAccessLogger accessLogger; SendingEntityBody(Http2OutboundRespListener http2OutboundRespListener, Http2MessageStateContext http2MessageStateContext) { this.http2OutboundRespListener = http2OutboundRespListener; this.http2MessageStateContext = http2MessageStateContext; this.ctx = http2OutboundRespListener.getChannelHandlerContext(); - this.serverChannelInitializer = http2OutboundRespListener.getServerChannelInitializer(); this.conn = http2OutboundRespListener.getConnection(); this.encoder = http2OutboundRespListener.getEncoder(); this.inboundRequestMsg = http2OutboundRespListener.getInboundRequestMsg(); this.outboundRespStatusFuture = inboundRequestMsg.getHttpOutboundRespStatusFuture(); - this.inboundRequestArrivalTime = http2OutboundRespListener.getInboundRequestArrivalTime(); this.originalStreamId = http2OutboundRespListener.getOriginalStreamId(); - this.remoteAddress = http2OutboundRespListener.getRemoteAddress(); + if (http2OutboundRespListener.getServerChannelInitializer().isHttpAccessLogEnabled()) { + this.accessLogger = new ListenerHttpAccessLogger( + http2OutboundRespListener.getInboundRequestArrivalTime(), + http2OutboundRespListener.getRemoteAddress()); + } } @Override @@ -178,8 +168,12 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, inboundRequestMsg); } http2OutboundRespListener.removeDefaultResponseWriter(); - if (serverChannelInitializer.isHttpAccessLogEnabled()) { - logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg, streamId); + if (accessLogger != null) { + if (originalStreamId != streamId) { // Skip access logs for server push messages + LOG.debug("Access logging skipped for server push response"); + return; + } + accessLogger.logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg); } http2MessageStateContext .setListenerState(new ResponseCompleted(http2OutboundRespListener, http2MessageStateContext)); @@ -189,7 +183,9 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, } private void writeData(HttpContent httpContent, int streamId, boolean endStream) throws Http2Exception { - contentLength += httpContent.content().readableBytes(); + if (accessLogger != null) { + accessLogger.updateContentLength(httpContent); + } validatePromisedStreamState(originalStreamId, streamId, conn, inboundRequestMsg); final ByteBuf content = httpContent.content(); for (Http2DataEventListener dataEventListener : http2OutboundRespListener.getHttp2ServerChannel() @@ -209,51 +205,4 @@ private void writeData(HttpContent httpContent, int streamId, boolean endStream) Util.addResponseWriteFailureListener(outboundRespStatusFuture, channelFuture, http2OutboundRespListener); } } - - private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg, - int streamId) { - if (originalStreamId != streamId) { // Skip access logs for server push messages - LOG.debug("Access logging skipped for server push response"); - return; - } - HttpHeaders headers = inboundRequestMsg.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; - } - - // Populate request parameters - String userAgent = "-"; - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - userAgent = headers.get(HttpHeaderNames.USER_AGENT); - } - String referrer = "-"; - if (headers.contains(HttpHeaderNames.REFERER)) { - referrer = headers.get(HttpHeaderNames.REFERER); - } - String method = inboundRequestMsg.getHttpMethod(); - String uri = inboundRequestMsg.getRequestUrl(); - HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); - String protocol; - if (request != null) { - protocol = request.protocolVersion().toString(); - } else { - protocol = inboundRequestMsg.getHttpVersion(); - } - - // Populate response parameters - int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - - long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); - HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, - inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); - inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); - inboundMessage.setRequestTime(requestTime); - - List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); - - HttpAccessLogger.log(inboundMessage, outboundMessages); - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index 75143910e5..3764a2bd57 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.SenderHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contractimpl.common.states.SenderReqRespStateManager; import io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil; @@ -27,29 +26,14 @@ import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Calendar; -import java.util.List; - -import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_STATE_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.isKeepAlive; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.safelyRemoveHandlers; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.ILLEGAL_STATE_ERROR; @@ -64,11 +48,14 @@ public class ReceivingEntityBody implements SenderState { private final SenderReqRespStateManager senderReqRespStateManager; private final TargetHandler targetHandler; - private Long contentLength = 0L; + private SenderHttpAccessLogger accessLogger; ReceivingEntityBody(SenderReqRespStateManager senderReqRespStateManager, TargetHandler targetHandler) { this.senderReqRespStateManager = senderReqRespStateManager; this.targetHandler = targetHandler; + if (targetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { + accessLogger = new SenderHttpAccessLogger(targetHandler.getTargetChannel().getChannel().remoteAddress()); + } } @Override @@ -93,13 +80,12 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent if (httpContent instanceof LastHttpContent) { StateUtil.setInboundTrailersToNewMessage(((LastHttpContent) httpContent).trailingHeaders(), inboundResponseMsg); - inboundResponseMsg.addHttpContent(httpContent); - contentLength += httpContent.content().readableBytes(); + addHttpContent(inboundResponseMsg, httpContent); inboundResponseMsg.setLastHttpContentArrived(); targetHandler.resetInboundMsg(); safelyRemoveHandlers(targetHandler.getTargetChannel().getChannel().pipeline(), IDLE_STATE_HANDLER); - if (targetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { - updateAccessLogInfo(targetHandler, inboundResponseMsg); + if (accessLogger != null) { + accessLogger.updateAccessLogInfo(targetHandler.getOutboundRequestMsg(), inboundResponseMsg); } senderReqRespStateManager.state = new EntityBodyReceived(senderReqRespStateManager); @@ -109,8 +95,7 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent } targetHandler.getConnectionManager().returnChannel(targetHandler.getTargetChannel()); } else { - inboundResponseMsg.addHttpContent(httpContent); - contentLength += httpContent.content().readableBytes(); + addHttpContent(inboundResponseMsg, httpContent); } } @@ -129,69 +114,10 @@ public void handleIdleTimeoutConnectionClosure(TargetHandler targetHandler, IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY); } - private void updateAccessLogInfo(TargetHandler targetHandler, - HttpCarbonMessage inboundResponseMsg) { - HttpCarbonMessage httpOutboundRequest = targetHandler.getOutboundRequestMsg(); - HttpAccessLogMessage outboundAccessLogMessage = - getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - if (outboundAccessLogMessage == null) { - return; - } - - SocketAddress remoteAddress = targetHandler.getTargetChannel().getChannel().remoteAddress(); - if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { - InetAddress inetAddress = inetSocketAddress.getAddress(); - outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); - outboundAccessLogMessage.setHost(inetAddress.getHostName()); - outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); - } - if (outboundAccessLogMessage.getIp().startsWith("/")) { - outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); - } - - // Populate with header parameters - HttpHeaders headers = httpOutboundRequest.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? - forwardedHops.substring(0, firstCommaIndex) : forwardedHops); - } - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); - } - if (headers.contains(HttpHeaderNames.REFERER)) { - outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); - } - HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> - outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? - headers.get(customHeader) : "-")); - - outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); - outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); - if (inboundResponse != null) { - outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); - } else { - outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); - } - outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); - outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); - outboundAccessLogMessage.setResponseBodySize(contentLength); - long requestTime = Calendar.getInstance().getTimeInMillis() - - outboundAccessLogMessage.getDateTime().getTimeInMillis(); - outboundAccessLogMessage.setRequestTime(requestTime); - - HttpCarbonMessage inboundReqMsg = - getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - - if (inboundReqMsg != null) { - List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); - if (outboundAccessLogMessages != null) { - outboundAccessLogMessages.add(outboundAccessLogMessage); - } + private void addHttpContent(HttpCarbonMessage inboundResponseMsg, HttpContent httpContent) { + inboundResponseMsg.addHttpContent(httpContent); + if (accessLogger != null) { + accessLogger.updateContentLength(httpContent.content()); } } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index a92b649830..3a37c33faa 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states.http2; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.SenderHttpAccessLogger; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2ClientChannel; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2TargetHandler; @@ -33,28 +32,13 @@ import io.netty.handler.codec.http.DefaultHttpContent; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http2.Http2Exception; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Calendar; -import java.util.List; - -import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_GOAWAY_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_RST_STREAM_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.releaseContent; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.handleIncompleteInboundMessage; @@ -70,13 +54,17 @@ public class ReceivingEntityBody implements SenderState { private final Http2TargetHandler http2TargetHandler; private final Http2ClientChannel http2ClientChannel; private final Http2TargetHandler.Http2RequestWriter http2RequestWriter; - private Long contentLength = 0L; + private SenderHttpAccessLogger accessLogger; ReceivingEntityBody(Http2TargetHandler http2TargetHandler, Http2TargetHandler.Http2RequestWriter http2RequestWriter) { this.http2TargetHandler = http2TargetHandler; this.http2RequestWriter = http2RequestWriter; this.http2ClientChannel = http2TargetHandler.getHttp2ClientChannel(); + if (http2TargetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { + accessLogger = + new SenderHttpAccessLogger(http2TargetHandler.getHttp2ClientChannel().getChannel().remoteAddress()); + } } @Override @@ -153,7 +141,9 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun Http2MessageStateContext http2MessageStateContext) { int streamId = http2DataFrame.getStreamId(); ByteBuf data = http2DataFrame.getData(); - contentLength += data.readableBytes(); + if (accessLogger != null) { + accessLogger.updateContentLength(data); + } boolean endOfStream = http2DataFrame.isEndOfStream(); if (serverPush) { @@ -162,8 +152,8 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun onResponseDataRead(outboundMsgHolder, streamId, endOfStream, data); } if (endOfStream) { - if (http2TargetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { - updateAccessLogInfo(outboundMsgHolder); + if (accessLogger != null) { + accessLogger.updateAccessLogInfo(outboundMsgHolder.getRequest(), outboundMsgHolder.getResponse()); } http2MessageStateContext.setSenderState(new EntityBodyReceived(http2TargetHandler, http2RequestWriter)); } @@ -192,69 +182,4 @@ private void onResponseDataRead(OutboundMsgHolder outboundMsgHolder, int streamI responseMessage.addHttpContent(new DefaultHttpContent(data.retain())); } } - - private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { - HttpCarbonMessage httpOutboundRequest = outboundMsgHolder.getRequest(); - HttpAccessLogMessage outboundAccessLogMessage = - getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - if (outboundAccessLogMessage == null) { - return; - } - - SocketAddress remoteAddress = http2TargetHandler.getHttp2ClientChannel().getChannel().remoteAddress(); - if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { - InetAddress inetAddress = inetSocketAddress.getAddress(); - outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); - outboundAccessLogMessage.setHost(inetAddress.getHostName()); - outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); - } - if (outboundAccessLogMessage.getIp().startsWith("/")) { - outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); - } - - // Populate with header parameters - HttpHeaders headers = httpOutboundRequest.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? - forwardedHops.substring(0, firstCommaIndex) : forwardedHops); - } - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); - } - if (headers.contains(HttpHeaderNames.REFERER)) { - outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); - } - HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> - outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? - headers.get(customHeader) : "-")); - - outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); - outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage inboundResponse = outboundMsgHolder.getResponse().getNettyHttpResponse(); - if (inboundResponse != null) { - outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); - } else { - outboundAccessLogMessage.setScheme(outboundMsgHolder.getResponse().getHttpVersion()); - } - outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); - outboundAccessLogMessage.setStatus(outboundMsgHolder.getResponse().getHttpStatusCode()); - outboundAccessLogMessage.setResponseBodySize(contentLength); - long requestTime = Calendar.getInstance().getTimeInMillis() - - outboundAccessLogMessage.getDateTime().getTimeInMillis(); - outboundAccessLogMessage.setRequestTime(requestTime); - - HttpCarbonMessage inboundReqMsg = - getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - - if (inboundReqMsg != null) { - List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); - if (outboundAccessLogMessages != null) { - outboundAccessLogMessages.add(outboundAccessLogMessage); - } - } - } } From ae756e9978cef50014ed80e0653c7e1156883180 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Tue, 2 Jul 2024 15:31:27 +0530 Subject: [PATCH 12/16] Fix log formatting --- ballerina/Dependencies.toml | 2 +- .../accesslog/HttpAccessLogFormatter.java | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 00e895c46a..ee2ab8a1f1 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -50,7 +50,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.7.0" +version = "2.7.2" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java index 79d5d5e4dc..7f5ea3c28d 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -114,13 +114,14 @@ private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLo HttpAccessLogFormat format, String attribute) { return switch (attribute) { case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); - case ATTRIBUTE_DATE_TIME -> - String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); + case ATTRIBUTE_DATE_TIME -> String.format(format == HttpAccessLogFormat.FLAT ? + "[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]" : "%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz", + httpAccessLogMessage.getDateTime()); case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), + "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); @@ -130,22 +131,24 @@ private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLo "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); - case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); - default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); + case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor())); + default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, format, attribute); }; } private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, - String attribute) { + HttpAccessLogFormat format, String attribute) { Map customHeaders = httpAccessLogMessage.getCustomHeaders(); if (attribute.startsWith("http_")) { String customHeaderKey = attribute.substring(5); for (Map.Entry entry : customHeaders.entrySet()) { if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { - return entry.getValue(); + String value = entry.getValue(); + return format == HttpAccessLogFormat.FLAT ? String.format("\"%s\"", value) : value; } } - return "-"; + return format == HttpAccessLogFormat.FLAT ? "\"-\"" : "-"; } return null; } From e5c772d232f83f95e3efe2bb1913db8eb48d926e Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Wed, 3 Jul 2024 11:34:24 +0530 Subject: [PATCH 13/16] Address review comments --- .../accesslog/HttpAccessLogConfig.java | 8 ++++++- .../accesslog/HttpAccessLogFormat.java | 2 +- .../accesslog/HttpAccessLogFormatter.java | 9 ++++++- .../accesslog/HttpAccessLogMessage.java | 8 ++++++- .../logging/accesslog/HttpAccessLogUtil.java | 17 +++++++------ .../logging/accesslog/HttpAccessLogger.java | 9 ++++++- .../accesslog/ListenerHttpAccessLogger.java | 20 ++++++++++++++-- .../accesslog/SenderHttpAccessLogger.java | 24 +++++++++++++++---- .../transport/message/HttpCarbonMessage.java | 6 +---- 9 files changed, 79 insertions(+), 24 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java index d48d0f4c91..aebdb2c5d2 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -36,6 +36,12 @@ import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT; import static io.ballerina.stdlib.http.api.HttpConstants.HTTP_LOG_FORMAT_JSON; +/** + * Provides a singleton configuration manager for HTTP access logging within the application. + * This class manages settings related to log formats, attributes, and custom header. + * + * @since 2.11.3 + */ public class HttpAccessLogConfig { private static final HttpAccessLogConfig instance = new HttpAccessLogConfig(); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java index 4ac452c0f6..ab84efd7be 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java index 7f5ea3c28d..f47c57e527 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -42,6 +42,13 @@ import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; +/** + * Handles the formatting of HTTP access log messages based on the specified log format and attributes. + * This utility class supports both FLAT and JSON formats for the rendering of access log entries, + * accommodating custom attributes and handling multiple messages for detailed logging. + * + * @since 2.11.3 + */ public class HttpAccessLogFormatter { private HttpAccessLogFormatter() {} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java index 78947170c9..437f20b49e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -22,6 +22,12 @@ import java.util.HashMap; import java.util.Map; +/** + * Represents a single HTTP access log message, encapsulating all relevant data for a specific request-response cycle. + * This class stores details such as IP address, date and time, request and response attributes, and custom headers. + * + * @since 2.11.3 + */ public class HttpAccessLogMessage { private String ip; private Calendar dateTime; diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java index f8421623a4..d6bcd78b21 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -20,10 +20,18 @@ import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import java.util.ArrayList; import java.util.List; import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGES; +/** + * Provides utility methods for accessing and manipulating properties related to HTTP access logging + * from HttpCarbonMessage objects. This class includes type-safe property retrieval and handling specific + * logging data structures. + * + * @since 2.11.3 + */ public class HttpAccessLogUtil { public static T getTypedProperty(HttpCarbonMessage carbonMessage, String propertyName, Class type) { Object property = carbonMessage.getProperty(propertyName); @@ -36,15 +44,10 @@ public static T getTypedProperty(HttpCarbonMessage carbonMessage, String pro public static List getHttpAccessLogMessages(HttpCarbonMessage carbonMessage) { Object outboundAccessLogMessagesObject = carbonMessage.getProperty(OUTBOUND_ACCESS_LOG_MESSAGES); if (outboundAccessLogMessagesObject instanceof List rawList) { - for (Object item : rawList) { - if (!(item instanceof HttpAccessLogMessage)) { - return null; - } - } @SuppressWarnings("unchecked") List outboundAccessLogMessages = (List) rawList; return outboundAccessLogMessages; } - return null; + return new ArrayList<>(); } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index 1fde2e3267..ac9fc654c7 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -20,6 +20,13 @@ import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +/** + * Defines the contract for logging and updating HTTP access information. + * Implementations of this interface should handle the specifics of recording and maintaining HTTP access logs + * for both request and response messages. + * + * @since 2.11.3 + */ public interface HttpAccessLogger { void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java index 98594c5ed5..f688b76f7b 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -38,6 +38,12 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; +/** + * Implements {@link HttpAccessLogger} to log detailed HTTP access information for incoming requests + * and their corresponding responses. + * + * @since 2.11.3 + */ public class ListenerHttpAccessLogger implements HttpAccessLogger { private static final Logger LOG = LoggerFactory.getLogger(ListenerHttpAccessLogger.class); @@ -86,6 +92,16 @@ public void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage } else { protocol = inboundRequestMsg.getHttpVersion(); } + long requestBodySize; + if (headers.contains(HttpHeaderNames.CONTENT_LENGTH)) { + try { + requestBodySize = Long.parseLong(headers.get(HttpHeaderNames.CONTENT_LENGTH)); + } catch (Exception ignored) { + requestBodySize = 0L; + } + } else { + requestBodySize = (long) inboundRequestMsg.getContentSize(); + } // Populate response parameters int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); @@ -93,7 +109,7 @@ public void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); - inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); + inboundMessage.setRequestBodySize(requestBodySize); inboundMessage.setRequestTime(requestTime); List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java index a05915de8f..12278c7723 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org). * * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except @@ -39,6 +39,12 @@ import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.TO; +/** + * Implements {@link HttpAccessLogger} for the sender side, focusing on updating and enriching + * HTTP access log information for outbound requests and corresponding inbound responses. + * + * @since 2.11.3 + */ public class SenderHttpAccessLogger implements HttpAccessLogger { private static final Logger LOG = LoggerFactory.getLogger(SenderHttpAccessLogger.class); @@ -101,7 +107,17 @@ public void updateAccessLogInfo(HttpCarbonMessage outboundRequestMsg, HttpCarbon } else { outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); } - outboundAccessLogMessage.setRequestBodySize((long) outboundRequestMsg.getContentSize()); + long requestBodySize; + if (headers.contains(HttpHeaderNames.CONTENT_LENGTH)) { + try { + requestBodySize = Long.parseLong(headers.get(HttpHeaderNames.CONTENT_LENGTH)); + } catch (Exception ignored) { + requestBodySize = 0L; + } + } else { + requestBodySize = (long) outboundRequestMsg.getContentSize(); + } + outboundAccessLogMessage.setRequestBodySize(requestBodySize); outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); outboundAccessLogMessage.setResponseBodySize(contentLength); long requestTime = Calendar.getInstance().getTimeInMillis() - @@ -113,9 +129,7 @@ public void updateAccessLogInfo(HttpCarbonMessage outboundRequestMsg, HttpCarbon if (inboundReqMsg != null) { List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); - if (outboundAccessLogMessages != null) { - outboundAccessLogMessages.add(outboundAccessLogMessage); - } + outboundAccessLogMessages.add(outboundAccessLogMessage); } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java b/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java index f10a1519c1..f3ed3d5739 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/message/HttpCarbonMessage.java @@ -146,7 +146,7 @@ public HttpContent getHttpContent() { HttpContent httpContent = this.blockingEntityCollector.getHttpContent(); this.contentObservable.notifyGetListener(httpContent); if (httpContent != null) { - this.setContentSize(httpContent.content().readableBytes()); + this.contentSize += httpContent.content().readableBytes(); } return httpContent; } @@ -551,10 +551,6 @@ public boolean isPipeliningEnabled() { return pipeliningEnabled; } - public void setContentSize(Integer contentSize) { - this.contentSize = contentSize; - } - public Integer getContentSize() { return contentSize; } From 1824cca3dff7a6df6ac48c7615e2eb88acbfa30b Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Thu, 11 Jul 2024 12:20:11 +0530 Subject: [PATCH 14/16] Update since to 2.12.0 --- .../http/api/logging/accesslog/HttpAccessLogConfig.java | 2 +- .../http/api/logging/accesslog/HttpAccessLogFormat.java | 6 ++++++ .../http/api/logging/accesslog/HttpAccessLogFormatter.java | 2 +- .../http/api/logging/accesslog/HttpAccessLogMessage.java | 2 +- .../http/api/logging/accesslog/HttpAccessLogUtil.java | 2 +- .../stdlib/http/api/logging/accesslog/HttpAccessLogger.java | 2 +- .../api/logging/accesslog/ListenerHttpAccessLogger.java | 2 +- .../http/api/logging/accesslog/SenderHttpAccessLogger.java | 2 +- 8 files changed, 13 insertions(+), 7 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java index aebdb2c5d2..ff63969ce9 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogConfig.java @@ -40,7 +40,7 @@ * Provides a singleton configuration manager for HTTP access logging within the application. * This class manages settings related to log formats, attributes, and custom header. * - * @since 2.11.3 + * @since 2.12.0 */ public class HttpAccessLogConfig { diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java index ab84efd7be..4c670ad4e6 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormat.java @@ -18,6 +18,12 @@ package io.ballerina.stdlib.http.api.logging.accesslog; +/** + * Represents the format types available for HTTP access logging. + * This enum is used to specify the preferred logging format for HTTP access logs. + * + * @since 2.12.0 + */ public enum HttpAccessLogFormat { FLAT, JSON } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java index f47c57e527..8890888289 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -47,7 +47,7 @@ * This utility class supports both FLAT and JSON formats for the rendering of access log entries, * accommodating custom attributes and handling multiple messages for detailed logging. * - * @since 2.11.3 + * @since 2.12.0 */ public class HttpAccessLogFormatter { diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java index 437f20b49e..c929f7570a 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogMessage.java @@ -26,7 +26,7 @@ * Represents a single HTTP access log message, encapsulating all relevant data for a specific request-response cycle. * This class stores details such as IP address, date and time, request and response attributes, and custom headers. * - * @since 2.11.3 + * @since 2.12.0 */ public class HttpAccessLogMessage { private String ip; diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java index d6bcd78b21..72f4c36f5f 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogUtil.java @@ -30,7 +30,7 @@ * from HttpCarbonMessage objects. This class includes type-safe property retrieval and handling specific * logging data structures. * - * @since 2.11.3 + * @since 2.12.0 */ public class HttpAccessLogUtil { public static T getTypedProperty(HttpCarbonMessage carbonMessage, String propertyName, Class type) { diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index ac9fc654c7..505c455990 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -25,7 +25,7 @@ * Implementations of this interface should handle the specifics of recording and maintaining HTTP access logs * for both request and response messages. * - * @since 2.11.3 + * @since 2.12.0 */ public interface HttpAccessLogger { diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java index f688b76f7b..8389ba68f9 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java @@ -42,7 +42,7 @@ * Implements {@link HttpAccessLogger} to log detailed HTTP access information for incoming requests * and their corresponding responses. * - * @since 2.11.3 + * @since 2.12.0 */ public class ListenerHttpAccessLogger implements HttpAccessLogger { diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java index 12278c7723..fa6eede67e 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java @@ -43,7 +43,7 @@ * Implements {@link HttpAccessLogger} for the sender side, focusing on updating and enriching * HTTP access log information for outbound requests and corresponding inbound responses. * - * @since 2.11.3 + * @since 2.12.0 */ public class SenderHttpAccessLogger implements HttpAccessLogger { From 2eb84b633716c16e06bf43933e8960e05e13fa14 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Thu, 11 Jul 2024 12:25:31 +0530 Subject: [PATCH 15/16] Enable accesslog for http-dispatching-tests package --- ballerina-tests/http-dispatching-tests/Config.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 ballerina-tests/http-dispatching-tests/Config.toml diff --git a/ballerina-tests/http-dispatching-tests/Config.toml b/ballerina-tests/http-dispatching-tests/Config.toml new file mode 100644 index 0000000000..77330aad47 --- /dev/null +++ b/ballerina-tests/http-dispatching-tests/Config.toml @@ -0,0 +1,3 @@ +[ballerina.http.accessLogConfig] +console = true +format = "json" From a9092dac607b50dce6c743311e8f5603165661f0 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Fri, 12 Jul 2024 18:28:04 +0530 Subject: [PATCH 16/16] Move Config.toml file to tests dir --- ballerina-tests/http-dispatching-tests/{ => tests}/Config.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ballerina-tests/http-dispatching-tests/{ => tests}/Config.toml (100%) diff --git a/ballerina-tests/http-dispatching-tests/Config.toml b/ballerina-tests/http-dispatching-tests/tests/Config.toml similarity index 100% rename from ballerina-tests/http-dispatching-tests/Config.toml rename to ballerina-tests/http-dispatching-tests/tests/Config.toml