From 50f2a88292d67be76075ab1a6ba0b5ec80808867 Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Tue, 4 Feb 2025 20:42:41 +0000 Subject: [PATCH] collect mongodb activity lsid and transaction --- .../mongo/dbm/operation_samples.py | 26 ++++++++++++++++++- .../mongo/dbm/slow_operations.py | 11 ++++++++ mongo/datadog_checks/mongo/dbm/types.py | 15 ++++++++++- mongo/tests/fixtures/$currentOp-mongos | 22 ++++++++++++++++ .../results/operation-activities-mongos.json | 19 +++++++++++--- .../operation-activities-standalone.json | 8 ++++-- .../results/operation-samples-mongos.json | 11 +++++++- .../results/operation-samples-standalone.json | 5 +++- 8 files changed, 108 insertions(+), 9 deletions(-) diff --git a/mongo/datadog_checks/mongo/dbm/operation_samples.py b/mongo/datadog_checks/mongo/dbm/operation_samples.py index d706a1f925290..fb428052f945b 100644 --- a/mongo/datadog_checks/mongo/dbm/operation_samples.py +++ b/mongo/datadog_checks/mongo/dbm/operation_samples.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) +import binascii import time from typing import List, Optional @@ -259,6 +260,28 @@ def _get_operation_cursor(self, operation: dict) -> Optional[OperationSampleOper "operation_using_cursor_id": cursor.get("operationUsingCursorId"), } + def _get_operation_lsid(self, operation: dict) -> Optional[dict]: + lsid = operation.get("lsid") + if not lsid or not lsid.get("id"): + return None + + return { + "id": binascii.hexlify(lsid['id']).decode(), + } + + def _get_operation_transaction(self, operation: dict) -> Optional[dict]: + transaction = operation.get("transaction") + if not transaction: + return None + + return { + "txn_number": transaction.get("parameters", {}).get("txnNumber"), + "txn_retry_counter": transaction.get("parameters", {}).get("txnRetryCounter"), + "time_open_micros": transaction.get("timeOpenMicros"), + "time_active_micros": transaction.get("timeActiveMicros"), + "time_inactive_micros": transaction.get("timeInactiveMicros"), + } + def _get_operation_stats(self, operation: dict) -> OperationSampleOperationStats: return { "active": operation.get("active", False), # bool @@ -269,7 +292,6 @@ def _get_operation_stats(self, operation: dict) -> OperationSampleOperationStats "query_framework": operation.get("queryFramework"), # str "current_op_time": operation.get("currentOpTime"), # str start time of the operation "microsecs_running": operation.get("microsecs_running"), # int - "transaction_time_open_micros": operation.get("transaction", {}).get("timeOpenMicros"), # int # Conflicts "prepare_read_conflicts": operation.get("prepareReadConflicts", 0), # int "write_conflicts": operation.get("writeConflicts", 0), # int @@ -291,6 +313,8 @@ def _get_operation_stats(self, operation: dict) -> OperationSampleOperationStats ), # dict # cursor "cursor": self._get_operation_cursor(operation), # dict + "transaction": self._get_operation_transaction(operation), # dict + "lsid": self._get_operation_lsid(operation), # dict } def _get_query_signature(self, obfuscated_command: str) -> str: diff --git a/mongo/datadog_checks/mongo/dbm/slow_operations.py b/mongo/datadog_checks/mongo/dbm/slow_operations.py index fe388fa2f79fe..2f39367bb275a 100644 --- a/mongo/datadog_checks/mongo/dbm/slow_operations.py +++ b/mongo/datadog_checks/mongo/dbm/slow_operations.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) +import binascii import time from datetime import datetime @@ -353,6 +354,7 @@ def _create_slow_operation_explain_plan_payload(self, slow_operation: dict, expl "lock_stats": self._get_slow_operation_lock_stats(slow_operation), "flow_control_stats": self._get_slow_operation_flow_control_stats(slow_operation), "cursor": self._get_slow_operation_cursor(slow_operation), + "lsid": self._get_slow_operation_lsid(slow_operation["command"]), } ), } @@ -378,6 +380,15 @@ def _get_slow_operation_cursor(self, slow_operation): } return None + def _get_slow_operation_lsid(self, command): + lsid = command.get("lsid") + if not lsid or not lsid.get("id"): + return None + + return { + "id": binascii.hexlify(lsid['id']).decode(), + } + def _get_slow_operation_lock_stats(self, slow_operation): lock_stats = slow_operation.get("locks") if lock_stats: diff --git a/mongo/datadog_checks/mongo/dbm/types.py b/mongo/datadog_checks/mongo/dbm/types.py index 7e4dea323c4d9..baf04f6824c7c 100644 --- a/mongo/datadog_checks/mongo/dbm/types.py +++ b/mongo/datadog_checks/mongo/dbm/types.py @@ -125,6 +125,18 @@ class OperationSampleOperationStatsCursor(TypedDict, total=False): operation_using_cursor_id: Optional[str] +class OperationSampleOperationStatsTransaction(TypedDict, total=False): + txn_number: int + txn_retry_counter: int + time_open_micros: int + time_active_micros: int + time_inactive_micros: int + + +class OperationSampleOperationStatsLsid(TypedDict, total=False): + id: str + + class OperationSampleOperationStats(TypedDict, total=False): active: bool desc: Optional[str] @@ -134,7 +146,6 @@ class OperationSampleOperationStats(TypedDict, total=False): query_framework: Optional[str] current_op_time: str microsecs_running: Optional[int] - transaction_time_open_micros: Optional[int] prepare_read_conflicts: Optional[int] write_conflicts: Optional[int] num_yields: Optional[int] @@ -145,6 +156,8 @@ class OperationSampleOperationStats(TypedDict, total=False): flow_control_stats: Optional[OperationSampleOperationStatsFlowControlStats] waiting_for_latch: Optional[OperationSampleOperationStatsWaitingForLatch] cursor = Optional[OperationSampleOperationStatsCursor] + transaction = Optional[OperationSampleOperationStatsTransaction] + lsid = Optional[OperationSampleOperationStatsLsid] class OperationSampleActivityBase(TypedDict, total=False): diff --git a/mongo/tests/fixtures/$currentOp-mongos b/mongo/tests/fixtures/$currentOp-mongos index 299961375511c..c967cc2d0ffb1 100644 --- a/mongo/tests/fixtures/$currentOp-mongos +++ b/mongo/tests/fixtures/$currentOp-mongos @@ -54,6 +54,28 @@ } } }, + "transaction": { + "parameters": { + "txnNumber": 392, + "txnRetryCounter": 0, + "autocommit": false, + "readConcern": { + "level": "local", + "provenance": "implicitDefault" + } + }, + "readTimestamp": { + "$timestamp": { + "t": 0, + "i": 0 + } + }, + "startWallClockTime": "2025-02-04T18:45:22.915+00:00", + "timeOpenMicros": 1750, + "timeActiveMicros": 435, + "timeInactiveMicros": 1315, + "expiryTime": "2025-02-04T18:46:22.915+00:00" + }, "secs_running": 0, "microsecs_running": 29232, "op": "query", diff --git a/mongo/tests/results/operation-activities-mongos.json b/mongo/tests/results/operation-activities-mongos.json index 8c79c6a66e9e5..699e212998676 100644 --- a/mongo/tests/results/operation-activities-mongos.json +++ b/mongo/tests/results/operation-activities-mongos.json @@ -27,11 +27,13 @@ "query_framework": null, "current_op_time": "2024-05-20T14:36:56.231+00:00", "microsecs_running": 29232, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 0, "waiting_for_lock": false, + "lsid": { + "id": "ff95bad791504c36ab749ae8c640d54e" + }, "locks": { "feature_compatibility_version": "r", "global": "r" @@ -63,6 +65,13 @@ "type": "op", "op": "query", "shard": "shard04", + "transaction": { + "time_active_micros": 435, + "time_inactive_micros": 1315, + "time_open_micros": 1750, + "txn_number": 392, + "txn_retry_counter": 0 + }, "dbname": "integration", "application": null, "collection": "users", @@ -96,11 +105,13 @@ "query_framework": "classic", "current_op_time": "2024-06-13T20:50:10.834+00:00", "microsecs_running": 26789, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 1, "waiting_for_lock": false, + "lsid": { + "id": "ccb77e0cda6441daa458dd4067cdbc48" + }, "locks": { "feature_compatibility_version": "r", "global": "r" @@ -142,6 +153,7 @@ "type": "op", "op": "getmore", "shard": null, + "transaction": null, "dbname": "integration", "application": "orders-mongo", "collection": "movies", @@ -173,11 +185,11 @@ "query_framework": "classic", "current_op_time": "2024-08-06T20:59:35.394+00:00", "microsecs_running": 13134, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 2, "waiting_for_lock": false, + "lsid": null, "locks": {}, "lock_stats": { "feature_compatibility_version": { @@ -211,6 +223,7 @@ "type": "op", "op": "getmore", "shard": null, + "transaction": null, "dbname": "local", "application": "OplogFetcher", "collection": "oplog.rs", diff --git a/mongo/tests/results/operation-activities-standalone.json b/mongo/tests/results/operation-activities-standalone.json index 57340a791dc91..c43e1790445da 100644 --- a/mongo/tests/results/operation-activities-standalone.json +++ b/mongo/tests/results/operation-activities-standalone.json @@ -26,11 +26,13 @@ "query_framework": null, "current_op_time": "2024-05-16T18:06:38.419+00:00", "microsecs_running": 167, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 0, "waiting_for_lock": false, + "lsid": { + "id": "fe7f64529521475ca6263130bc2c926e" + }, "locks": { "feature_compatibility_version": "r", "global": "r" @@ -64,6 +66,7 @@ "type": "op", "op": "query", "shard": null, + "transaction": null, "dbname": "integration", "application": null, "collection": "products", @@ -97,12 +100,12 @@ "query_framework": "classic", "current_op_time": "2024-08-06T20:59:35.394+00:00", "microsecs_running": 13134, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 2, "waiting_for_lock": false, "locks": {}, + "lsid": null, "lock_stats": { "feature_compatibility_version": { "acquire_count": { @@ -135,6 +138,7 @@ "type": "op", "op": "getmore", "shard": null, + "transaction": null, "dbname": "local", "application": "OplogFetcher", "collection": "oplog.rs", diff --git a/mongo/tests/results/operation-samples-mongos.json b/mongo/tests/results/operation-samples-mongos.json index 63a63e4d08693..1f409084f3465 100644 --- a/mongo/tests/results/operation-samples-mongos.json +++ b/mongo/tests/results/operation-samples-mongos.json @@ -723,11 +723,20 @@ "query_framework": null, "current_op_time": "2024-05-20T14:36:56.231+00:00", "microsecs_running": 29232, - "transaction_time_open_micros": null, + "transaction": { + "time_active_micros": 435, + "time_inactive_micros": 1315, + "time_open_micros": 1750, + "txn_number": 392, + "txn_retry_counter": 0 + }, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 0, "waiting_for_lock": false, + "lsid": { + "id": "ff95bad791504c36ab749ae8c640d54e" + }, "locks": { "feature_compatibility_version": "r", "global": "r" diff --git a/mongo/tests/results/operation-samples-standalone.json b/mongo/tests/results/operation-samples-standalone.json index e6ca905264813..bff379371d6d6 100644 --- a/mongo/tests/results/operation-samples-standalone.json +++ b/mongo/tests/results/operation-samples-standalone.json @@ -144,11 +144,13 @@ "query_framework": null, "current_op_time": "2024-05-16T18:06:38.419+00:00", "microsecs_running": 167, - "transaction_time_open_micros": null, "prepare_read_conflicts": 0, "write_conflicts": 0, "num_yields": 0, "waiting_for_lock": false, + "lsid": { + "id": "fe7f64529521475ca6263130bc2c926e" + }, "locks": { "feature_compatibility_version": "r", "global": "r" @@ -182,6 +184,7 @@ "type": "op", "op": "query", "shard": null, + "transaction": null, "collection": "products", "comment": "find", "truncated": "not_truncated"