Group Logs and Spans by Resource and Instrumentation Scope in OTLP Exporter

opentelemetry-otlp/CHANGELOG.md

@@ -19,6 +19,9 @@ now use `.with_resource(RESOURCE::default())` to configure Resource when using
 - Fixing the OTLP HTTP/JSON exporter. [#1882](https://github.com/open-telemetry/opentelemetry-rust/pull/1882) - The exporter was broken in the
  previous release.
 - **Breaking** [1869](https://github.com/open-telemetry/opentelemetry-rust/pull/1869) The OTLP logs exporter now overrides the [InstrumentationScope::name](https://github.com/open-telemetry/opentelemetry-proto/blob/b3060d2104df364136d75a35779e6bd48bac449a/opentelemetry/proto/common/v1/common.proto#L73) field with the `target` from `LogRecord`, if target is populated.
+- Group batch of `LogRecord` and `Span` by their resource and instrumentation scope before exporting, for better efficiency [#1873](https://github.com/open-telemetry/opentelemetry-rust/pull/1873).
+ - This optimization reduces redundancy and improves the efficiency of log export. 
+ - The OTLP compliant Collector or Agents would be able to successfully parse these events.
 
 ## v0.16.0
 

opentelemetry-otlp/src/exporter/http/logs.rs

@@ -25,7 +25,7 @@ impl LogExporter for OtlpHttpClient {
  .map(|cow_log_data| cow_log_data.into_owned()) // Converts Cow to owned LogData
  .collect::<Vec<LogData>>();
 
- let (body, content_type) = { self.build_logs_export_body(owned_batch, &self.resource)? };
+ let (body, content_type) = { self.build_logs_export_body(owned_batch)? };
  let mut request = http::Request::builder()
  .method(Method::POST)
  .uri(&self.collector_endpoint)

opentelemetry-otlp/src/exporter/http/mod.rs

@@ -9,7 +9,10 @@ use crate::{
 use http::{HeaderName, HeaderValue, Uri};
 use opentelemetry_http::HttpClient;
 use opentelemetry_proto::transform::common::tonic::ResourceAttributesWithSchema;
-
+#[cfg(feature = "logs")]
+use opentelemetry_proto::transform::logs::tonic::group_logs_by_resource_and_scope;
+#[cfg(feature = "trace")]
+use opentelemetry_proto::transform::trace::tonic::group_spans_by_resource_and_scope;
 #[cfg(feature = "logs")]
 use opentelemetry_sdk::export::logs::LogData;
 #[cfg(feature = "trace")]
@@ -307,16 +310,9 @@ impl OtlpHttpClient {
  fn build_trace_export_body(
  &self,
  spans: Vec<SpanData>,
- resource: &opentelemetry_proto::transform::common::tonic::ResourceAttributesWithSchema,
  ) -> opentelemetry::trace::TraceResult<(Vec<u8>, &'static str)> {
- use opentelemetry_proto::tonic::{
- collector::trace::v1::ExportTraceServiceRequest, trace::v1::ResourceSpans,
- };
-
- let resource_spans = spans
- .into_iter()
- .map(|span| ResourceSpans::new(span, resource))
- .collect::<Vec<_>>();
+ use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
+ let resource_spans = group_spans_by_resource_and_scope(spans, &self.resource);
 
  let req = ExportTraceServiceRequest { resource_spans };
  match self.protocol {
@@ -333,13 +329,9 @@ impl OtlpHttpClient {
  fn build_logs_export_body(
  &self,
  logs: Vec<LogData>,
- resource: &opentelemetry_proto::transform::common::tonic::ResourceAttributesWithSchema,
  ) -> opentelemetry::logs::LogResult<(Vec<u8>, &'static str)> {
  use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
- let resource_logs = logs
- .into_iter()
- .map(|log_event| (log_event, resource).into())
- .collect::<Vec<_>>();
+ let resource_logs = group_logs_by_resource_and_scope(logs, &self.resource);
  let req = ExportLogsServiceRequest { resource_logs };
 
  match self.protocol {

opentelemetry-otlp/src/exporter/http/trace.rs

@@ -21,7 +21,7 @@ impl SpanExporter for OtlpHttpClient {
  Err(err) => return Box::pin(std::future::ready(Err(err))),
  };
 
- let (body, content_type) = match self.build_trace_export_body(batch, &self.resource) {
+ let (body, content_type) = match self.build_trace_export_body(batch) {
  Ok(body) => body,
  Err(e) => return Box::pin(std::future::ready(Err(e))),
  };

opentelemetry-otlp/src/exporter/tonic/logs.rs

@@ -7,6 +7,8 @@ use opentelemetry_proto::tonic::collector::logs::v1::{
 use opentelemetry_sdk::export::logs::{LogData, LogExporter};
 use tonic::{codegen::CompressionEncoding, service::Interceptor, transport::Channel, Request};
 
+use opentelemetry_proto::transform::logs::tonic::group_logs_by_resource_and_scope;
+
 use super::BoxInterceptor;
 
 pub(crate) struct TonicLogsClient {
@@ -65,15 +67,13 @@ impl LogExporter for TonicLogsClient {
  None => return Err(LogError::Other("exporter is already shut down".into())),
  };
 
- // TODO: Avoid cloning here.
- let resource_logs = {
- batch
- .into_iter()
- .map(|log_data_cow| (log_data_cow.into_owned()))
- .map(|log_data| (log_data, &self.resource))
- .map(Into::into)
- .collect()
- };
+ //TODO: avoid cloning here.
+ let owned_batch = batch
+ .into_iter()
+ .map(|cow_log_data| cow_log_data.into_owned()) // Converts Cow to owned LogData
+ .collect::<Vec<LogData>>();
+
+ let resource_logs = group_logs_by_resource_and_scope(owned_batch, &self.resource);
 
  client
  .export(Request::from_parts(

opentelemetry-otlp/src/exporter/tonic/trace.rs

@@ -5,10 +5,11 @@ use opentelemetry::trace::TraceError;
 use opentelemetry_proto::tonic::collector::trace::v1::{
  trace_service_client::TraceServiceClient, ExportTraceServiceRequest,
 };
-use opentelemetry_proto::tonic::trace::v1::ResourceSpans;
 use opentelemetry_sdk::export::trace::{ExportResult, SpanData, SpanExporter};
 use tonic::{codegen::CompressionEncoding, service::Interceptor, transport::Channel, Request};
 
+use opentelemetry_proto::transform::trace::tonic::group_spans_by_resource_and_scope;
+
 use super::BoxInterceptor;
 
 pub(crate) struct TonicTracesClient {
@@ -71,13 +72,7 @@ impl SpanExporter for TonicTracesClient {
  }
  };
 
- // TODO: Avoid cloning here.
- let resource_spans = {
- batch
- .into_iter()
- .map(|log_data| ResourceSpans::new(log_data, &self.resource))
- .collect()
- };
+ let resource_spans = group_spans_by_resource_and_scope(batch, &self.resource);
 
  Box::pin(async move {
  client

opentelemetry-proto/CHANGELOG.md

@@ -3,6 +3,9 @@
 ## vNext
 
 - Bump MSRV to 1.70 [1864](https://github.com/open-telemetry/opentelemetry-rust/pull/1874)
+- **BREAKING** Group log and Span batch by their resource and instrumentation scope before exporting [#1873](https://github.com/open-telemetry/opentelemetry-rust/pull/1873).
+ - Introduced `group_logs_by_resource_and_scope()` and `group_spans_by_resource_and_scope()` methods to group logs and spans by the resource and scope respectively.
+ - This is a breaking change for exporters consuming the OTLP format. Refer to the OTLP Log and Span exporter in the opentelemetry-otlp crate for the required changes.
 
 ## v0.6.0
 

opentelemetry-proto/Cargo.toml

@@ -29,7 +29,7 @@ path = "tests/json_deserialize.rs"
 
 
 [features]
-default = []
+default = ["full"]
 
 full = ["gen-tonic", "trace", "logs", "metrics", "zpages", "with-serde"]
 
@@ -42,6 +42,7 @@ trace = ["opentelemetry/trace", "opentelemetry_sdk/trace"]
 metrics = ["opentelemetry/metrics", "opentelemetry_sdk/metrics"]
 logs = ["opentelemetry/logs", "opentelemetry_sdk/logs"]
 zpages = ["trace"]
+testing = ["opentelemetry/testing"]
 
 # add ons
 with-schemars = ["schemars"]
@@ -57,7 +58,8 @@ serde = { workspace = true, optional = true, features = ["serde_derive"] }
 hex = { version = "0.4.3", optional = true }
 
 [dev-dependencies]
+opentelemetry = { version = "0.23", features = ["testing"], path = "../opentelemetry" }
 tonic-build = { workspace = true }
 prost-build = { workspace = true }
 tempfile = "3.3.0"
-serde_json = { workspace = true }
+serde_json = { workspace = true }

opentelemetry-proto/src/transform/logs.rs

@@ -10,6 +10,7 @@ pub mod tonic {
  transform::common::{to_nanos, tonic::ResourceAttributesWithSchema},
  };
  use opentelemetry::logs::{AnyValue as LogsAnyValue, Severity};
+ use std::collections::HashMap;
 
  impl From<LogsAnyValue> for AnyValue {
  fn from(value: LogsAnyValue) -> Self {
@@ -143,4 +144,116 @@ pub mod tonic {
  }
  }
  }
+
+ pub fn group_logs_by_resource_and_scope(
+ logs: Vec<opentelemetry_sdk::export::logs::LogData>,
+ resource: &ResourceAttributesWithSchema,
+ ) -> Vec<ResourceLogs> {
+ // no need of explicit grouping by resource, as all the logs belong to single resource.
+ let scope_map = logs.iter().fold(
+ HashMap::new(),
+ |mut scope_map: HashMap<
+ &opentelemetry_sdk::InstrumentationLibrary,
+ Vec<&opentelemetry_sdk::export::logs::LogData>,
+ >,
+ log| {
+ let instrumentation = &log.instrumentation;
+ scope_map.entry(instrumentation).or_default().push(log);
+ scope_map
+ },
+ );
+
+ let scope_logs = scope_map
+ .into_iter()
+ .map(|(instrumentation, log_records)| ScopeLogs {
+ scope: Some(instrumentation.into()),
+ schema_url: resource.schema_url.clone().unwrap_or_default(),
+ log_records: log_records
+ .into_iter()
+ .map(|log_data| log_data.record.clone().into())
+ .collect(),
+ })
+ .collect();
+
+ vec![ResourceLogs {
+ resource: Some(Resource {
+ attributes: resource.attributes.0.clone(),
+ dropped_attributes_count: 0,
+ }),
+ scope_logs,
+ schema_url: resource.schema_url.clone().unwrap_or_default(),
+ }]
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::transform::common::tonic::ResourceAttributesWithSchema;
+ use opentelemetry::logs::LogRecord as _;
+ use opentelemetry_sdk::export::logs::LogData;
+ use opentelemetry_sdk::{logs::LogRecord, Resource};
+ use std::time::SystemTime;
+
+ fn create_test_log_data(instrumentation_name: &str, _message: &str) -> LogData {
+ let mut logrecord = LogRecord::default();
+ logrecord.set_timestamp(SystemTime::now());
+ logrecord.set_observed_timestamp(SystemTime::now());
+ LogData {
+ instrumentation: opentelemetry_sdk::InstrumentationLibrary::builder(
+ instrumentation_name.to_string(),
+ )
+ .build(),
+ record: logrecord,
+ }
+ }
+
+ #[test]
+ fn test_group_logs_by_resource_and_scope_single_scope() {
+ let resource = Resource::default();
+ let log1 = create_test_log_data("test-lib", "Log 1");
+ let log2 = create_test_log_data("test-lib", "Log 2");
+
+ let logs = vec![log1, log2];
+ let resource: ResourceAttributesWithSchema = (&resource).into(); // Convert Resource to ResourceAttributesWithSchema
+
+ let grouped_logs =
+ crate::transform::logs::tonic::group_logs_by_resource_and_scope(logs, &resource);
+
+ assert_eq!(grouped_logs.len(), 1);
+ let resource_logs = &grouped_logs[0];
+ assert_eq!(resource_logs.scope_logs.len(), 1);
+
+ let scope_logs = &resource_logs.scope_logs[0];
+ assert_eq!(scope_logs.log_records.len(), 2);
+ }
+
+ #[test]
+ fn test_group_logs_by_resource_and_scope_multiple_scopes() {
+ let resource = Resource::default();
+ let log1 = create_test_log_data("lib1", "Log 1");
+ let log2 = create_test_log_data("lib2", "Log 2");
+
+ let logs = vec![log1, log2];
+ let resource: ResourceAttributesWithSchema = (&resource).into(); // Convert Resource to ResourceAttributesWithSchema
+ let grouped_logs =
+ crate::transform::logs::tonic::group_logs_by_resource_and_scope(logs, &resource);
+
+ assert_eq!(grouped_logs.len(), 1);
+ let resource_logs = &grouped_logs[0];
+ assert_eq!(resource_logs.scope_logs.len(), 2);
+
+ let scope_logs_1 = &resource_logs
+ .scope_logs
+ .iter()
+ .find(|scope| scope.scope.as_ref().unwrap().name == "lib1")
+ .unwrap();
+ let scope_logs_2 = &resource_logs
+ .scope_logs
+ .iter()
+ .find(|scope| scope.scope.as_ref().unwrap().name == "lib2")
+ .unwrap();
+
+ assert_eq!(scope_logs_1.log_records.len(), 1);
+ assert_eq!(scope_logs_2.log_records.len(), 1);
+ }
 }