[PLINT-246] Add pagination support for ironic nodes endpoint (#16400)

* Add support for pagination in ironic nodes endpoint

* Update log line and remove first_try

* rename resources var

* Reformat logging

* Add changelog

* Address review

* fix config spec

openstack_controller/assets/configuration/spec.yaml

@@ -132,7 +132,7 @@ files:
             paginated_limit sets the number of items some api calls should return.
           value:
             example: 1000
-            display_default: 1000
+            default: null
             type: integer
         - name: openstack_config_file_path
           description: |

openstack_controller/changelog.d/16400.added

@@ -0,0 +1 @@
+Add pagination support for ironic nodes endpoint

openstack_controller/datadog_checks/openstack_controller/api/api_rest.py

@@ -292,6 +292,43 @@ def get_network_quota(self, project_id):
         response.raise_for_status()
         return response.json().get('quota', [])
 
+    def make_paginated_request(self, url, resource_name, marker_name, params=None):
+        marker = None
+        item_list = []
+        params = {} if params is None else params
+        while True:
+            self.log.debug(
+                "making paginated request [limit=%s, marker=%s]",
+                self.config.paginated_limit,
+                marker,
+            )
+
+            params['limit'] = self.config.paginated_limit
+            if marker is not None:
+                params['marker'] = marker
+
+            response = self.http.get(url, params=params)
+            response.raise_for_status()
+
+            response_json = response.json()
+            resources = response_json.get(resource_name, [])
+            if len(resources) > 0:
+                last_item = resources[-1]
+                next = last_item.get('next')
+                item_list.extend(resources)
+                if next is None:
+                    break
+
+                marker = last_item.get(marker_name)
+            else:
+                marker = None
+                break
+
+            if marker is None:
+                break
+
+        return item_list
+
     def get_baremetal_nodes(self):
         def use_legacy_nodes_resource(microversion):
             self.log.debug("Configured ironic microversion: %s", microversion)
@@ -308,18 +345,24 @@ def use_legacy_nodes_resource(microversion):
             self.log.debug("Collecting baremetal nodes with use_legacy_nodes_resource =%s", legacy_microversion)
             return legacy_microversion
 
-        self.log.debug("getting baremetal nodes [microversion=%s]", self.config.ironic_microversion)
         ironic_endpoint = self._catalog.get_endpoint_by_type(Component.Types.BAREMETAL.value)
 
+        params = {}
         if use_legacy_nodes_resource(self.config.ironic_microversion):
-            response = self.http.get('{}/v1/nodes/detail'.format(ironic_endpoint))
+            url = '{}/v1/nodes/detail'.format(ironic_endpoint)
         else:
             params = {'detail': True}
+            url = '{}/v1/nodes'.format(ironic_endpoint)
 
-            response = self.http.get('{}/v1/nodes'.format(ironic_endpoint), params=params)
+        if self.config.paginated_limit is not None:
+            return self.make_paginated_request(url, 'nodes', 'uuid', params)
 
-        response.raise_for_status()
-        return response.json().get('nodes', [])
+        else:
+            response = self.http.get(url, params=params)
+            response.raise_for_status()
+
+            response_json = response.json()
+            return response_json.get("nodes", [])
 
     def get_baremetal_conductors(self):
         response = self.http.get(

openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py

@@ -118,6 +118,12 @@ def get_response_time(self, endpoint_types):
         response.raise_for_status()
         return response.elapsed.total_seconds() * 1000
 
+    def call_paginated_api(self, method_name, *args, **kwargs):
+        if kwargs.get('limit') is None:
+            kwargs.pop('limit')
+
+        return method_name(*args, **kwargs)
+
     def get_identity_regions(self):
         return [region.to_dict(original_names=True) for region in self.connection.identity.regions()]
 
@@ -202,7 +208,12 @@ def get_network_quota(self, project_id):
         return self.connection.network.get_quota(project_id, details=True).to_dict(original_names=True)
 
     def get_baremetal_nodes(self):
-        return [node.to_dict(original_names=True) for node in self.connection.baremetal.nodes(details=True)]
+        return [
+            node.to_dict(original_names=True)
+            for node in self.call_paginated_api(
+                self.connection.baremetal.nodes, details=True, limit=self.config.paginated_limit
+            )
+        ]
 
     def get_baremetal_conductors(self):
         return [conductor.to_dict(original_names=True) for conductor in self.connection.baremetal.conductors()]

openstack_controller/datadog_checks/openstack_controller/config.py

@@ -47,6 +47,7 @@ def __init__(self, logger, config):
         self.ironic_microversion = config.ironic_microversion
         self.endpoint_interface = config.endpoint_interface
         self.endpoint_region_id = config.endpoint_region_id
+        self.paginated_limit = config.paginated_limit
         self.api_type = None
         self.validate()
 

openstack_controller/datadog_checks/openstack_controller/config_models/defaults.py

@@ -97,7 +97,7 @@ def instance_min_collection_interval():
 
 
 def instance_paginated_limit():
-    return 1000
+    return None
 
 
 def instance_persist_connections():

openstack_controller/tests/conftest.py

@@ -167,7 +167,7 @@ def method(method, url, file='response', headers=None, params=None):
         filename = file
         request_path = url
         request_path = request_path.replace('?', '/')
-        if params is not None:
+        if params:
             param_string = '/'.join('{}={}'.format(key, str(val)) for key, val in params.items())
             request_path = '{}/{}'.format(url, param_string)
         if any(re.search(pattern, request_path) for pattern in NOVA_ENDPOINTS):
@@ -546,7 +546,7 @@ def connection_baremetal(request, mock_responses):
     param = request.param if hasattr(request, 'param') and request.param is not None else {}
     http_error = param.get('http_error')
 
-    def nodes(details):
+    def nodes(details, limit=None):
         if http_error and 'nodes' in http_error:
             raise requests.exceptions.HTTPError(response=http_error['nodes'])
         return [

openstack_controller/tests/endpoints.py

@@ -3,17 +3,17 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
 NOVA_ENDPOINTS = [
-    '^/compute/v2\\.1/limits$',
-    '^/compute/v2\\.1/os-services$',
-    '^/compute/v2\\.1/flavors/detail$',
-    '^/compute/v2\\.1/os-hypervisors/detail$',
-    '^/compute/v2\\.1/os-quota-sets/[^/]+$',
-    '^/compute/v2\\.1/servers/detail/project_id=[^/]+$',
-    '^/compute/v2\\.1/servers/[^/]+/diagnostics$',
+    '^/compute/v2\\.1/limits',
+    '^/compute/v2\\.1/os-services',
+    '^/compute/v2\\.1/flavors/detail',
+    '^/compute/v2\\.1/os-hypervisors/detail',
+    '^/compute/v2\\.1/os-quota-sets/[^/]+',
+    '^/compute/v2\\.1/servers/detail/project_id=[^/]+',
+    '^/compute/v2\\.1/servers/[^/]+/diagnostics',
 ]
 
 IRONIC_ENDPOINTS = [
-    '^/baremetal/v1/nodes/detail$',
-    '^/baremetal/v1/nodes/detail=True$',
-    '^/baremetal/v1/conductors$',
+    '^/baremetal/v1/nodes/detail',
+    '^/baremetal/v1/nodes/detail=True',
+    '^/baremetal/v1/conductors',
 ]

openstack_controller/tests/fixtures/GET/baremetal/v1/nodes/detail/limit=1/marker=54855e59-83ca-46f8-a78f-55d3370e0656/response-1.80.json

@@ -0,0 +1,116 @@
+{
+    "nodes": [
+      {
+        "uuid": "20512deb-e493-4796-a046-5d6e4e072c95",
+        "created_at": "2023-04-06T16:55:34+00:00",
+        "updated_at": "2023-04-06T16:57:38+00:00",
+        "automated_clean": null,
+        "bios_interface": "fake",
+        "boot_interface": "fake",
+        "boot_mode": null,
+        "clean_step": {},
+        "conductor_group": "",
+        "console_enabled": false,
+        "console_interface": "fake",
+        "deploy_interface": "direct",
+        "deploy_step": {},
+        "description": null,
+        "driver": "fake-hardware",
+        "driver_info": {},
+        "driver_internal_info": {
+          "last_power_state_change": "2023-04-06T16:55:42.474124"
+        },
+        "extra": {},
+        "fault": null,
+        "inspection_finished_at": null,
+        "inspection_started_at": null,
+        "inspect_interface": "fake",
+        "instance_info": {},
+        "instance_uuid": null,
+        "last_error": null,
+        "lessee": null,
+        "maintenance": false,
+        "maintenance_reason": null,
+        "management_interface": "fake",
+        "name": "test",
+        "network_data": {},
+        "network_interface": "flat",
+        "owner": "01b21103a92d4997ab09e46ff8346bd5",
+        "power_interface": "fake",
+        "power_state": "power on",
+        "properties": {},
+        "protected": false,
+        "protected_reason": null,
+        "provision_state": "active",
+        "provision_updated_at": "2023-04-06T16:56:21+00:00",
+        "raid_config": {},
+        "raid_interface": "fake",
+        "rescue_interface": "fake",
+        "reservation": null,
+        "resource_class": null,
+        "retired": false,
+        "retired_reason": null,
+        "secure_boot": null,
+        "storage_interface": "noop",
+        "target_power_state": null,
+        "target_provision_state": null,
+        "target_raid_config": {},
+        "traits": [],
+        "vendor_interface": "fake",
+        "links": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
+            "rel": "bookmark"
+          }
+        ],
+        "conductor": "agent-integrations-openstack-ironic",
+        "allocation_uuid": null,
+        "chassis_uuid": null,
+        "ports": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
+            "rel": "bookmark"
+          }
+        ],
+        "states": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/states",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/states",
+            "rel": "bookmark"
+          }
+        ],
+        "portgroups": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/portgroups",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/portgroups",
+            "rel": "bookmark"
+          }
+        ],
+        "volume": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/volume",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/volume",
+            "rel": "bookmark"
+          }
+        ]
+      }
+    ]
+  }
+

openstack_controller/tests/fixtures/GET/baremetal/v1/nodes/detail/limit=1/marker=54855e59-83ca-46f8-a78f-55d3370e0656/response.json

@@ -0,0 +1,47 @@
+{
+    "nodes": [
+      {
+        "uuid": "20512deb-e493-4796-a046-5d6e4e072c95",
+        "created_at": "2023-04-06T16:55:34+00:00",
+        "updated_at": "2023-04-06T16:57:38+00:00",
+        "console_enabled": false,
+        "driver": "fake-hardware",
+        "driver_info": {},
+        "extra": {},
+        "instance_info": {},
+        "instance_uuid": null,
+        "last_error": null,
+        "maintenance": false,
+        "maintenance_reason": null,
+        "power_state": "power on",
+        "properties": {},
+        "provision_state": "active",
+        "provision_updated_at": "2023-04-06T16:56:21+00:00",
+        "reservation": null,
+        "target_power_state": null,
+        "target_provision_state": null,
+        "links": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
+            "rel": "bookmark"
+          }
+        ],
+        "chassis_uuid": null,
+        "ports": [
+          {
+            "href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
+            "rel": "self"
+          },
+          {
+            "href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
+            "rel": "bookmark"
+          }
+        ]
+      }
+    ]
+  }
+