Retry when an IATA fails to load

Jonty · Sep 20, 2024 · 6f2fa0a · 6f2fa0a
1 parent 5f46a5d
commit 6f2fa0a
Showing 1 changed file with 13 additions and 7 deletions.
diff --git a/scrape_airline_routes.py b/scrape_airline_routes.py
@@ -31,13 +31,19 @@
 
         print("Fetching #%s: %s" % (len(airports), iata))
 
-        response = requests.get(
-            "https://www.flightsfrom.com/%s/destinations" % iata, impersonate="chrome"
-        )
-        root = lxml.html.document_fromstring(response.content)
-        metadata_nodes = root.xpath('//script[contains(., "window.airport")]')
-        metadata_tag = metadata_nodes[0].text_content()
-        metadata_bits = metadata_tag.split("window.")
+        while True:
+            try:
+                response = requests.get(
+                    "https://www.flightsfrom.com/%s/destinations" % iata, impersonate="chrome"
+                )
+                root = lxml.html.document_fromstring(response.content)
+                metadata_nodes = root.xpath('//script[contains(., "window.airport")]')
+                metadata_tag = metadata_nodes[0].text_content()
+                metadata_bits = metadata_tag.split("window.")
+                break
+            except Exception as e:
+                print("! Error while fetching IATA, having a little 5m sleep before retrying: %s" % e)
+                time.sleep(60*5)
 
         metadata = {}
         for bit in metadata_bits: