Skip to content

Commit a9fd65c

Browse files
banoollarry-aptos
banool
authored and
larry-aptos
committed
[Data Service] Push metrics parameterized by identifier, not API key name + email (aptos-labs#12547)
1 parent e7663c3 commit a9fd65c

File tree

4 files changed

+28
-48
lines changed

4 files changed

+28
-48
lines changed

‎ecosystem/indexer-grpc/indexer-grpc-data-service/src/metrics.rs

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,15 @@ use aptos_metrics_core::{
77
};
88
use once_cell::sync::Lazy;
99

10+
// The `identifier` label at the time of writing (2024-04-08) is always the
11+
// application ID, a globally unique ID.
12+
1013
/// Latest processed transaction version.
1114
pub static LATEST_PROCESSED_VERSION: Lazy<IntGaugeVec> = Lazy::new(|| {
1215
register_int_gauge_vec!(
1316
"indexer_grpc_data_service_with_user_latest_processed_version",
1417
"Latest processed transaction version",
15-
&["request_token", "email", "processor"],
18+
&["identifier", "processor"],
1619
)
1720
.unwrap()
1821
});
@@ -22,7 +25,7 @@ pub static PROCESSED_VERSIONS_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
2225
register_int_counter_vec!(
2326
"indexer_grpc_data_service_with_user_processed_versions",
2427
"Number of transactions that have been processed by data service",
25-
&["request_token", "email", "processor"],
28+
&["identifier", "processor"],
2629
)
2730
.unwrap()
2831
});
@@ -42,7 +45,7 @@ pub static PROCESSED_LATENCY_IN_SECS: Lazy<GaugeVec> = Lazy::new(|| {
4245
register_gauge_vec!(
4346
"indexer_grpc_data_service_with_user_latest_data_latency_in_secs",
4447
"Latency of data service based on latest processed transaction",
45-
&["request_token", "email", "processor"],
48+
&["identifier", "processor"],
4649
)
4750
.unwrap()
4851
});
@@ -52,7 +55,7 @@ pub static PROCESSED_LATENCY_IN_SECS_ALL: Lazy<HistogramVec> = Lazy::new(|| {
5255
register_histogram_vec!(
5356
"indexer_grpc_data_service_latest_data_latency_in_secs_all",
5457
"Latency of data service based on latest processed transaction",
55-
&["request_token"]
58+
&[]
5659
)
5760
.unwrap()
5861
});
@@ -62,7 +65,7 @@ pub static PROCESSED_BATCH_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
6265
register_int_gauge_vec!(
6366
"indexer_grpc_data_service_with_user_processed_batch_size",
6467
"Size of latest processed batch by data service",
65-
&["request_token", "email", "processor"],
68+
&["identifier", "processor"],
6669
)
6770
.unwrap()
6871
});
@@ -72,7 +75,7 @@ pub static CONNECTION_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
7275
register_int_counter_vec!(
7376
"indexer_grpc_data_service_connection_count_v2",
7477
"Count of connections that data service has established",
75-
&["request_token", "email", "processor"],
78+
&["identifier", "processor"],
7679
)
7780
.unwrap()
7881
});
@@ -82,7 +85,7 @@ pub static SHORT_CONNECTION_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
8285
register_int_counter_vec!(
8386
"indexer_grpc_data_service_short_connection_by_user_processor_count",
8487
"Count of the short connections; i.e., < 10 seconds",
85-
&["request_token", "email", "processor"],
88+
&["identifier", "processor"],
8689
)
8790
.unwrap()
8891
});
@@ -93,7 +96,7 @@ pub static BYTES_READY_TO_TRANSFER_FROM_SERVER: Lazy<IntCounterVec> = Lazy::new(
9396
register_int_counter_vec!(
9497
"indexer_grpc_data_service_bytes_ready_to_transfer_from_server",
9598
"Count of bytes ready to transfer to the client",
96-
&["request_token", "email", "processor"],
99+
&["identifier", "processor"],
97100
)
98101
.unwrap()
99102
});

‎ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs

Lines changed: 14 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,9 @@ const RESPONSE_CHANNEL_SEND_TIMEOUT: Duration = Duration::from_secs(120);
6262

6363
const SHORT_CONNECTION_DURATION_IN_SECS: u64 = 10;
6464

65-
const REQUEST_HEADER_APTOS_EMAIL_HEADER: &str = "x-aptos-email";
66-
const REQUEST_HEADER_APTOS_USER_CLASSIFICATION_HEADER: &str = "x-aptos-user-classification";
67-
const REQUEST_HEADER_APTOS_API_KEY_NAME: &str = "x-aptos-api-key-name";
65+
/// This comes from API Gateway. The identifier uniquely identifies the requester, which
66+
/// in the case of indexer-grpc is always an application.
67+
const REQUEST_HEADER_APTOS_IDENTIFIER: &str = "x-aptos-identifier";
6868
const RESPONSE_HEADER_APTOS_CONNECTION_ID_HEADER: &str = "x-aptos-connection-id";
6969
const SERVICE_TYPE: &str = "data_service";
7070

@@ -141,8 +141,7 @@ impl RawData for RawDataServerWrapper {
141141
};
142142
CONNECTION_COUNT
143143
.with_label_values(&[
144-
&request_metadata.request_api_key_name,
145-
&request_metadata.request_email,
144+
&request_metadata.request_identifier,
146145
&request_metadata.processor_name,
147146
])
148147
.inc();
@@ -349,12 +348,9 @@ async fn get_data_in_task(
349348
Ok(TransactionsDataStatus::AheadOfCache) => {
350349
info!(
351350
start_version = start_version,
352-
request_name = request_metadata.processor_name.as_str(),
353-
request_email = request_metadata.request_email.as_str(),
354-
request_api_key_name = request_metadata.request_api_key_name.as_str(),
351+
request_identifier = request_metadata.request_identifier.as_str(),
355352
processor_name = request_metadata.processor_name.as_str(),
356353
connection_id = request_metadata.request_connection_id.as_str(),
357-
request_user_classification = request_metadata.request_user_classification.as_str(),
358354
duration_in_secs = current_batch_start_time.elapsed().as_secs_f64(),
359355
service_type = SERVICE_TYPE,
360356
"[Data Service] Requested data is ahead of cache. Sleeping for {} ms.",
@@ -513,8 +509,7 @@ async fn data_fetcher_task(
513509
.sum::<usize>();
514510
BYTES_READY_TO_TRANSFER_FROM_SERVER
515511
.with_label_values(&[
516-
&request_metadata.request_api_key_name,
517-
&request_metadata.request_email,
512+
&request_metadata.request_identifier,
518513
&request_metadata.processor_name,
519514
])
520515
.inc_by(bytes_ready_to_transfer as u64);
@@ -542,36 +537,32 @@ async fn data_fetcher_task(
542537
Ok(_) => {
543538
PROCESSED_BATCH_SIZE
544539
.with_label_values(&[
545-
request_metadata.request_api_key_name.as_str(),
546-
request_metadata.request_email.as_str(),
540+
request_metadata.request_identifier.as_str(),
547541
request_metadata.processor_name.as_str(),
548542
])
549543
.set(current_batch_size as i64);
550544
// TODO: Reasses whether this metric useful
551545
LATEST_PROCESSED_VERSION_OLD
552546
.with_label_values(&[
553-
request_metadata.request_api_key_name.as_str(),
554-
request_metadata.request_email.as_str(),
547+
request_metadata.request_identifier.as_str(),
555548
request_metadata.processor_name.as_str(),
556549
])
557550
.set(end_of_batch_version as i64);
558551
PROCESSED_VERSIONS_COUNT
559552
.with_label_values(&[
560-
request_metadata.request_api_key_name.as_str(),
561-
request_metadata.request_email.as_str(),
553+
request_metadata.request_identifier.as_str(),
562554
request_metadata.processor_name.as_str(),
563555
])
564556
.inc_by(current_batch_size as u64);
565557
if let Some(data_latency_in_secs) = data_latency_in_secs {
566558
PROCESSED_LATENCY_IN_SECS
567559
.with_label_values(&[
568-
request_metadata.request_api_key_name.as_str(),
569-
request_metadata.request_email.as_str(),
560+
request_metadata.request_identifier.as_str(),
570561
request_metadata.processor_name.as_str(),
571562
])
572563
.set(data_latency_in_secs);
573564
PROCESSED_LATENCY_IN_SECS_ALL
574-
.with_label_values(&[request_metadata.request_user_classification.as_str()])
565+
.with_label_values(&[])
575566
.observe(data_latency_in_secs);
576567
}
577568
},
@@ -589,22 +580,17 @@ async fn data_fetcher_task(
589580
current_version = end_of_batch_version + 1;
590581
}
591582
info!(
592-
request_name = request_metadata.processor_name.as_str(),
593-
request_email = request_metadata.request_email.as_str(),
594-
request_api_key_name = request_metadata.request_api_key_name.as_str(),
583+
request_identifier = request_metadata.request_identifier.as_str(),
595584
processor_name = request_metadata.processor_name.as_str(),
596585
connection_id = request_metadata.request_connection_id.as_str(),
597-
request_user_classification = request_metadata.request_user_classification.as_str(),
598-
request_user_classification = request_metadata.request_user_classification.as_str(),
599586
service_type = SERVICE_TYPE,
600587
"[Data Service] Client disconnected."
601588
);
602589
if let Some(start_time) = connection_start_time {
603590
if start_time.elapsed().as_secs() < SHORT_CONNECTION_DURATION_IN_SECS {
604591
SHORT_CONNECTION_COUNT
605592
.with_label_values(&[
606-
request_metadata.request_api_key_name.as_str(),
607-
request_metadata.request_email.as_str(),
593+
request_metadata.request_identifier.as_str(),
608594
request_metadata.processor_name.as_str(),
609595
])
610596
.inc();
@@ -869,12 +855,7 @@ fn get_request_metadata(
869855
req: &Request<GetTransactionsRequest>,
870856
) -> tonic::Result<IndexerGrpcRequestMetadata> {
871857
let request_metadata_pairs = vec![
872-
("request_api_key_name", REQUEST_HEADER_APTOS_API_KEY_NAME),
873-
("request_email", REQUEST_HEADER_APTOS_EMAIL_HEADER),
874-
(
875-
"request_user_classification",
876-
REQUEST_HEADER_APTOS_USER_CLASSIFICATION_HEADER,
877-
),
858+
("request_identifier", REQUEST_HEADER_APTOS_IDENTIFIER),
878859
("request_token", GRPC_AUTH_TOKEN_HEADER),
879860
("processor_name", GRPC_REQUEST_NAME_HEADER),
880861
];

‎ecosystem/indexer-grpc/indexer-grpc-utils/src/constants.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@ pub const MESSAGE_SIZE_LIMIT: usize = 1024 * 1024 * 15;
1717
#[derive(Clone, Serialize, Deserialize, Debug)]
1818
pub struct IndexerGrpcRequestMetadata {
1919
pub processor_name: String,
20-
pub request_email: String,
21-
pub request_user_classification: String,
22-
pub request_api_key_name: String,
20+
/// See `REQUEST_HEADER_APTOS_IDENTIFIER` for more information.
21+
pub request_identifier: String,
2322
pub request_connection_id: String,
2423
// Token is no longer needed behind api gateway.
2524
#[deprecated]

‎ecosystem/indexer-grpc/indexer-grpc-utils/src/counters.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -278,12 +278,9 @@ pub fn log_grpc_step(
278278
duration_in_secs,
279279
size_in_bytes,
280280
// Request metadata variables
281-
request_name = &request_metadata.processor_name,
282-
request_email = &request_metadata.request_email,
283-
request_api_key_name = &request_metadata.request_api_key_name,
281+
request_identifier = &request_metadata.request_identifier,
284282
processor_name = &request_metadata.processor_name,
285283
connection_id = &request_metadata.request_connection_id,
286-
request_user_classification = &request_metadata.request_user_classification,
287284
service_type,
288285
step = step.get_step(),
289286
"{}",

0 commit comments

Comments
 (0)