Skip to content

Commit

Permalink
details and fix for __geo_interface__, enforce black formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
mvexel committed Jan 28, 2025
1 parent 25f2f29 commit cdaeff4
Show file tree
Hide file tree
Showing 6 changed files with 332 additions and 89 deletions.
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,6 @@ dev = [
"mkdocs-material-extensions>=1.3.1",
"mkdocstrings[python]>=0.26.1",
"pytest>=8.3.4",
"shapely>=2.0.6",
"typing-extensions>=4.12.2",
]
63 changes: 35 additions & 28 deletions src/osmdiff/augmenteddiff.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,12 @@ def __init__(
sequence_number: Optional[int] = None,
timestamp: Optional[datetime] = None,
base_url: Optional[str] = None,
timeout: Optional[int] = None
timeout: Optional[int] = None,
) -> None:
# Initialize with defaults from config
self.base_url = base_url or API_CONFIG["overpass"]["base_url"]
self.timeout = timeout or API_CONFIG["overpass"]["timeout"]

# Initialize other config values
self.minlon = minlon
self.minlat = minlat
Expand Down Expand Up @@ -120,8 +120,7 @@ def get_state(self) -> bool:
"""
state_url = urljoin(self.base_url, "augmented_diff_status")
response = requests.get(
state_url,
timeout=self.timeout or 5 # Use instance timeout with fallback
state_url, timeout=self.timeout or 5 # Use instance timeout with fallback
)
if response.status_code != 200:
return False
Expand All @@ -143,12 +142,12 @@ def _build_adiff_url(self):

def _build_action(self, elem):
"""Parse an action element from an augmented diff.
Actions in augmented diffs are ordered: nodes first, then ways, then relations.
Within each type, elements are ordered by ID.
"""
action_type = elem.attrib["type"]

if action_type == "create":
for child in elem:
osm_obj = OSMObject.from_xml(child)
Expand Down Expand Up @@ -182,7 +181,13 @@ def _parse_stream(self, stream):
if elem.tag == "action":
self._build_action(elem)

def retrieve(self, clear_cache: bool = False, timeout: Optional[int] = None, auto_increment: bool = True, max_retries: int = 3) -> int:
def retrieve(
self,
clear_cache: bool = False,
timeout: Optional[int] = None,
auto_increment: bool = True,
max_retries: int = 3,
) -> int:
"""Retrieve the Augmented diff corresponding to the sequence_number.
Args:
Expand All @@ -200,60 +205,62 @@ def retrieve(self, clear_cache: bool = False, timeout: Optional[int] = None, aut
"""
if not self.sequence_number:
raise Exception("invalid sequence number")

if clear_cache:
self.create, self.modify, self.delete = ([], [], [])

url = self._build_adiff_url()

# Store current data before making request
prev_create = self.create.copy()
prev_modify = self.modify.copy()
prev_delete = self.delete.copy()

# Use a longer timeout if none specified
request_timeout = timeout or self.timeout or 120 # 2 minutes default, this will still fail for very large diffs, like 12346

request_timeout = (
timeout or self.timeout or 120
) # 2 minutes default, this will still fail for very large diffs, like 12346

for attempt in range(max_retries):
try:
# Exponential backoff between retries
if attempt > 0:
time.sleep(2 ** attempt) # 2, 4, 8 seconds...
time.sleep(2**attempt) # 2, 4, 8 seconds...

r = requests.get(
url,
stream=True,
timeout=request_timeout,
headers=DEFAULT_HEADERS
url, stream=True, timeout=request_timeout, headers=DEFAULT_HEADERS
)

if r.status_code != 200:
return r.status_code

r.raw.decode_content = True

# Clear current lists but keep previous data
self.create, self.modify, self.delete = ([], [], [])

# Parse new data
self._parse_stream(r.raw)

# Merge with previous data
self.create = prev_create + self.create
self.modify = prev_modify + self.modify
self.delete = prev_delete + self.delete

# Automatically increment sequence number after successful retrieval
if auto_increment:
self.sequence_number += 1

return r.status_code

except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e:

except (
requests.exceptions.ReadTimeout,
requests.exceptions.ConnectionError,
) as e:
if attempt == max_retries - 1: # Last attempt
raise
continue

return 0 # Should never reach here due to raise in except block

@property
Expand Down
9 changes: 3 additions & 6 deletions src/osmdiff/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"nominatim": {
"base_url": "https://nominatim.openstreetmap.org",
"timeout": 30,
}
},
}

# Default parameters for AugmentedDiff operations
Expand All @@ -41,15 +41,12 @@
"minlat": None, # Minimum latitude for bounding box
"maxlon": None, # Maximum longitude for bounding box
"maxlat": None, # Maximum latitude for bounding box
"timestamp": None # Timestamp for diff operations
"timestamp": None, # Timestamp for diff operations
}

# User agent string following OSM API guidelines
# https://operations.osmfoundation.org/policies/api/
USER_AGENT = "osmdiff/1.0" # Replace with your actual user agent

# Standard headers used in all API requests
DEFAULT_HEADERS = {
"User-Agent": USER_AGENT,
"Accept": "application/json, text/xml"
}
DEFAULT_HEADERS = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}
Loading

0 comments on commit cdaeff4

Please sign in to comment.