Skip to content

Commit d647f80

Browse files
sarah-wittsarah-witt
authored
[PLINT-246] Add pagination support for ironic nodes endpoint (#16400)
* Add support for pagination in ironic nodes endpoint * Update log line and remove first_try * rename resources var * Reformat logging * Add changelog * Address review * fix config spec
1 parent 0f79c1f commit d647f80

File tree

25 files changed

+1789
-20
lines changed

25 files changed

+1789
-20
lines changed

‎openstack_controller/assets/configuration/spec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ files:
132132
paginated_limit sets the number of items some api calls should return.
133133
value:
134134
example: 1000
135-
display_default: 1000
135+
default: null
136136
type: integer
137137
- name: openstack_config_file_path
138138
description: |

‎openstack_controller/changelog.d/16400.added

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add pagination support for ironic nodes endpoint

‎openstack_controller/datadog_checks/openstack_controller/api/api_rest.py

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,43 @@ def get_network_quota(self, project_id):
292292
response.raise_for_status()
293293
return response.json().get('quota', [])
294294

295+
def make_paginated_request(self, url, resource_name, marker_name, params=None):
296+
marker = None
297+
item_list = []
298+
params = {} if params is None else params
299+
while True:
300+
self.log.debug(
301+
"making paginated request [limit=%s, marker=%s]",
302+
self.config.paginated_limit,
303+
marker,
304+
)
305+
306+
params['limit'] = self.config.paginated_limit
307+
if marker is not None:
308+
params['marker'] = marker
309+
310+
response = self.http.get(url, params=params)
311+
response.raise_for_status()
312+
313+
response_json = response.json()
314+
resources = response_json.get(resource_name, [])
315+
if len(resources) > 0:
316+
last_item = resources[-1]
317+
next = last_item.get('next')
318+
item_list.extend(resources)
319+
if next is None:
320+
break
321+
322+
marker = last_item.get(marker_name)
323+
else:
324+
marker = None
325+
break
326+
327+
if marker is None:
328+
break
329+
330+
return item_list
331+
295332
def get_baremetal_nodes(self):
296333
def use_legacy_nodes_resource(microversion):
297334
self.log.debug("Configured ironic microversion: %s", microversion)
@@ -308,18 +345,24 @@ def use_legacy_nodes_resource(microversion):
308345
self.log.debug("Collecting baremetal nodes with use_legacy_nodes_resource =%s", legacy_microversion)
309346
return legacy_microversion
310347

311-
self.log.debug("getting baremetal nodes [microversion=%s]", self.config.ironic_microversion)
312348
ironic_endpoint = self._catalog.get_endpoint_by_type(Component.Types.BAREMETAL.value)
313349

350+
params = {}
314351
if use_legacy_nodes_resource(self.config.ironic_microversion):
315-
response = self.http.get('{}/v1/nodes/detail'.format(ironic_endpoint))
352+
url = '{}/v1/nodes/detail'.format(ironic_endpoint)
316353
else:
317354
params = {'detail': True}
355+
url = '{}/v1/nodes'.format(ironic_endpoint)
318356

319-
response = self.http.get('{}/v1/nodes'.format(ironic_endpoint), params=params)
357+
if self.config.paginated_limit is not None:
358+
return self.make_paginated_request(url, 'nodes', 'uuid', params)
320359

321-
response.raise_for_status()
322-
return response.json().get('nodes', [])
360+
else:
361+
response = self.http.get(url, params=params)
362+
response.raise_for_status()
363+
364+
response_json = response.json()
365+
return response_json.get("nodes", [])
323366

324367
def get_baremetal_conductors(self):
325368
response = self.http.get(

‎openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ def get_response_time(self, endpoint_types):
118118
response.raise_for_status()
119119
return response.elapsed.total_seconds() * 1000
120120

121+
def call_paginated_api(self, method_name, *args, **kwargs):
122+
if kwargs.get('limit') is None:
123+
kwargs.pop('limit')
124+
125+
return method_name(*args, **kwargs)
126+
121127
def get_identity_regions(self):
122128
return [region.to_dict(original_names=True) for region in self.connection.identity.regions()]
123129

@@ -202,7 +208,12 @@ def get_network_quota(self, project_id):
202208
return self.connection.network.get_quota(project_id, details=True).to_dict(original_names=True)
203209

204210
def get_baremetal_nodes(self):
205-
return [node.to_dict(original_names=True) for node in self.connection.baremetal.nodes(details=True)]
211+
return [
212+
node.to_dict(original_names=True)
213+
for node in self.call_paginated_api(
214+
self.connection.baremetal.nodes, details=True, limit=self.config.paginated_limit
215+
)
216+
]
206217

207218
def get_baremetal_conductors(self):
208219
return [conductor.to_dict(original_names=True) for conductor in self.connection.baremetal.conductors()]

‎openstack_controller/datadog_checks/openstack_controller/config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def __init__(self, logger, config):
4747
self.ironic_microversion = config.ironic_microversion
4848
self.endpoint_interface = config.endpoint_interface
4949
self.endpoint_region_id = config.endpoint_region_id
50+
self.paginated_limit = config.paginated_limit
5051
self.api_type = None
5152
self.validate()
5253

‎openstack_controller/datadog_checks/openstack_controller/config_models/defaults.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def instance_min_collection_interval():
9797

9898

9999
def instance_paginated_limit():
100-
return 1000
100+
return None
101101

102102

103103
def instance_persist_connections():

‎openstack_controller/tests/conftest.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ def method(method, url, file='response', headers=None, params=None):
167167
filename = file
168168
request_path = url
169169
request_path = request_path.replace('?', '/')
170-
if params is not None:
170+
if params:
171171
param_string = '/'.join('{}={}'.format(key, str(val)) for key, val in params.items())
172172
request_path = '{}/{}'.format(url, param_string)
173173
if any(re.search(pattern, request_path) for pattern in NOVA_ENDPOINTS):
@@ -546,7 +546,7 @@ def connection_baremetal(request, mock_responses):
546546
param = request.param if hasattr(request, 'param') and request.param is not None else {}
547547
http_error = param.get('http_error')
548548

549-
def nodes(details):
549+
def nodes(details, limit=None):
550550
if http_error and 'nodes' in http_error:
551551
raise requests.exceptions.HTTPError(response=http_error['nodes'])
552552
return [

‎openstack_controller/tests/endpoints.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@
33
# Licensed under a 3-clause BSD style license (see LICENSE)
44

55
NOVA_ENDPOINTS = [
6-
'^/compute/v2\\.1/limits$',
7-
'^/compute/v2\\.1/os-services$',
8-
'^/compute/v2\\.1/flavors/detail$',
9-
'^/compute/v2\\.1/os-hypervisors/detail$',
10-
'^/compute/v2\\.1/os-quota-sets/[^/]+$',
11-
'^/compute/v2\\.1/servers/detail/project_id=[^/]+$',
12-
'^/compute/v2\\.1/servers/[^/]+/diagnostics$',
6+
'^/compute/v2\\.1/limits',
7+
'^/compute/v2\\.1/os-services',
8+
'^/compute/v2\\.1/flavors/detail',
9+
'^/compute/v2\\.1/os-hypervisors/detail',
10+
'^/compute/v2\\.1/os-quota-sets/[^/]+',
11+
'^/compute/v2\\.1/servers/detail/project_id=[^/]+',
12+
'^/compute/v2\\.1/servers/[^/]+/diagnostics',
1313
]
1414

1515
IRONIC_ENDPOINTS = [
16-
'^/baremetal/v1/nodes/detail$',
17-
'^/baremetal/v1/nodes/detail=True$',
18-
'^/baremetal/v1/conductors$',
16+
'^/baremetal/v1/nodes/detail',
17+
'^/baremetal/v1/nodes/detail=True',
18+
'^/baremetal/v1/conductors',
1919
]

‎openstack_controller/tests/fixtures/GET/baremetal/v1/nodes/detail/limit=1/marker=54855e59-83ca-46f8-a78f-55d3370e0656/response-1.80.json

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"nodes": [
3+
{
4+
"uuid": "20512deb-e493-4796-a046-5d6e4e072c95",
5+
"created_at": "2023-04-06T16:55:34+00:00",
6+
"updated_at": "2023-04-06T16:57:38+00:00",
7+
"automated_clean": null,
8+
"bios_interface": "fake",
9+
"boot_interface": "fake",
10+
"boot_mode": null,
11+
"clean_step": {},
12+
"conductor_group": "",
13+
"console_enabled": false,
14+
"console_interface": "fake",
15+
"deploy_interface": "direct",
16+
"deploy_step": {},
17+
"description": null,
18+
"driver": "fake-hardware",
19+
"driver_info": {},
20+
"driver_internal_info": {
21+
"last_power_state_change": "2023-04-06T16:55:42.474124"
22+
},
23+
"extra": {},
24+
"fault": null,
25+
"inspection_finished_at": null,
26+
"inspection_started_at": null,
27+
"inspect_interface": "fake",
28+
"instance_info": {},
29+
"instance_uuid": null,
30+
"last_error": null,
31+
"lessee": null,
32+
"maintenance": false,
33+
"maintenance_reason": null,
34+
"management_interface": "fake",
35+
"name": "test",
36+
"network_data": {},
37+
"network_interface": "flat",
38+
"owner": "01b21103a92d4997ab09e46ff8346bd5",
39+
"power_interface": "fake",
40+
"power_state": "power on",
41+
"properties": {},
42+
"protected": false,
43+
"protected_reason": null,
44+
"provision_state": "active",
45+
"provision_updated_at": "2023-04-06T16:56:21+00:00",
46+
"raid_config": {},
47+
"raid_interface": "fake",
48+
"rescue_interface": "fake",
49+
"reservation": null,
50+
"resource_class": null,
51+
"retired": false,
52+
"retired_reason": null,
53+
"secure_boot": null,
54+
"storage_interface": "noop",
55+
"target_power_state": null,
56+
"target_provision_state": null,
57+
"target_raid_config": {},
58+
"traits": [],
59+
"vendor_interface": "fake",
60+
"links": [
61+
{
62+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
63+
"rel": "self"
64+
},
65+
{
66+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
67+
"rel": "bookmark"
68+
}
69+
],
70+
"conductor": "agent-integrations-openstack-ironic",
71+
"allocation_uuid": null,
72+
"chassis_uuid": null,
73+
"ports": [
74+
{
75+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
76+
"rel": "self"
77+
},
78+
{
79+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
80+
"rel": "bookmark"
81+
}
82+
],
83+
"states": [
84+
{
85+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/states",
86+
"rel": "self"
87+
},
88+
{
89+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/states",
90+
"rel": "bookmark"
91+
}
92+
],
93+
"portgroups": [
94+
{
95+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/portgroups",
96+
"rel": "self"
97+
},
98+
{
99+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/portgroups",
100+
"rel": "bookmark"
101+
}
102+
],
103+
"volume": [
104+
{
105+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/volume",
106+
"rel": "self"
107+
},
108+
{
109+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/volume",
110+
"rel": "bookmark"
111+
}
112+
]
113+
}
114+
]
115+
}
116+

‎openstack_controller/tests/fixtures/GET/baremetal/v1/nodes/detail/limit=1/marker=54855e59-83ca-46f8-a78f-55d3370e0656/response.json

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"nodes": [
3+
{
4+
"uuid": "20512deb-e493-4796-a046-5d6e4e072c95",
5+
"created_at": "2023-04-06T16:55:34+00:00",
6+
"updated_at": "2023-04-06T16:57:38+00:00",
7+
"console_enabled": false,
8+
"driver": "fake-hardware",
9+
"driver_info": {},
10+
"extra": {},
11+
"instance_info": {},
12+
"instance_uuid": null,
13+
"last_error": null,
14+
"maintenance": false,
15+
"maintenance_reason": null,
16+
"power_state": "power on",
17+
"properties": {},
18+
"provision_state": "active",
19+
"provision_updated_at": "2023-04-06T16:56:21+00:00",
20+
"reservation": null,
21+
"target_power_state": null,
22+
"target_provision_state": null,
23+
"links": [
24+
{
25+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
26+
"rel": "self"
27+
},
28+
{
29+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95",
30+
"rel": "bookmark"
31+
}
32+
],
33+
"chassis_uuid": null,
34+
"ports": [
35+
{
36+
"href": "http://34.141.226.224/baremetal/v1/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
37+
"rel": "self"
38+
},
39+
{
40+
"href": "http://34.141.226.224/baremetal/nodes/20512deb-e493-4796-a046-5d6e4e072c95/ports",
41+
"rel": "bookmark"
42+
}
43+
]
44+
}
45+
]
46+
}
47+

0 commit comments

Comments
 (0)