fix: rewrite get_rest_pages so it knows the API provides fully formed… (snyk-labs#193)

* fix: rewrite get_rest_pages so it knows the API provides fully formed next urls

* chore: black

* fix: fix version of Poetry

Author: nathan-roys

.github/workflows/release.yml

@@ -20,6 +20,8 @@ jobs:
 
       - name: Install poetry
         uses: snok/install-poetry@v1
+        with:
+            version: 1.5.1
 
       - name: Install Dependencies
         run: poetry install -v

.github/workflows/test.yml

@@ -21,7 +21,8 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           virtualenvs-path: .venv
-          
+          version: 1.5.1
+
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v3

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pysnyk"
-version = "0.9.11"
+version = "0.9.12"
 description = "A Python client for the Snyk API"
 authors = [
   "Gareth Rushgrove <[email protected]>",

snyk/client.py

@@ -152,7 +152,7 @@ def get(
                 params = {}
 
             # we use the presence of version to determine if we are REST or not
-            if "version" not in params.keys() and self.version:
+            if "version" not in params.keys() and self.version and not exclude_version:
                 params["version"] = version or self.version
 
             # Python Bools are True/False, JS Bools are true/false
@@ -212,40 +212,24 @@ def get_rest_pages(self, path: str, params: dict = {}) -> List:
         This collects the "data" list from the first reponse and then appends the
         any further "data" lists if a next link is found in the links field.
         """
+        first_page_response = self.get(path, params)
+        page_data = first_page_response.json()
+        return_data = page_data["data"]
 
-        # this is a raw primative but a higher level module might want something that does an
-        # arbitrary path + origin=foo + limit=100 url construction instead before being sent here
-
-        limit = params["limit"]
-
-        data = list()
-
-        page = self.get(path, params).json()
-
-        data.extend(page["data"])
-
-        while "next" in page["links"].keys():
+        while page_data.get("links", {}).get("next"):
             logger.debug(
-                f"GET_REST_PAGES: Another link exists: {page['links']['next']}"
+                f"GET_REST_PAGES: Another link exists: {page_data['links']['next']}"
             )
+            next_url = page_data.get("links", {}).get("next")
 
-            next_url = urllib.parse.urlsplit(page["links"]["next"])
-            query = urllib.parse.parse_qs(next_url.query)
-
-            for k, v in query.items():
-                params[k] = v
-
-            params["limit"] = limit
-
-            page = self.get(next_url.path, params).json()
-
-            data.extend(page["data"])
-
+            # The next url comes back fully formed (i.e. with all the params already set, so no need to do it here)
+            next_page_response = self.get(next_url, {}, exclude_version=True)
+            page_data = next_page_response.json()
+            return_data.extend(page_data["data"])
             logger.debug(
-                f"GET_REST_PAGES: Added another {len(page['data'])} items to the response"
+                f"GET_REST_PAGES: Added another {len(page_data['data'])} items to the response"
             )
-
-        return data
+        return return_data
 
     # alias for backwards compatibility where V3 was the old name
     get_v3_pages = get_rest_pages