Skip to content

Commit

Permalink
Merge pull request #44 from openaddresses/pmtiles-support
Browse files Browse the repository at this point in the history 
Pmtiles support
  • Loading branch information
@iandees
iandees authored Aug 31, 2023
2 parents db3abc5 + aff4611 commit 2088164
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 12 deletions.
41 changes: 31 additions & 10 deletions openaddr/process_one.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def process(source, destination, layer, layersource, do_preview, mapbox_key=None
with wait_lock:
proc_wait.start()
cache_result, conform_result = CacheResult.empty(), ConformResult.empty()
preview_path, slippymap_path, skipped_source = None, None, False
preview_path, mbtiles_path, pmtiles_path, skipped_source = None, None, None, False
tests_passed = None

try:
Expand Down Expand Up @@ -153,7 +153,8 @@ def process(source, destination, layer, layersource, do_preview, mapbox_key=None
preview_path = render_preview(conform_result.path, temp_dir, mapbox_key)

if do_preview:
slippymap_path = render_slippymap(conform_result.path, temp_dir)
mbtiles_path = render_mbtiles(conform_result.path, temp_dir)
pmtiles_path = render_pmtiles(conform_result.path, temp_dir)

if not preview_path:
_L.warning('Nothing previewed')
Expand All @@ -176,7 +177,7 @@ def process(source, destination, layer, layersource, do_preview, mapbox_key=None
logging.getLogger('openaddr').removeHandler(log_handler)

state_path = write_state(temp_src, layer, data_source['name'], skipped_source, destination, log_handler,
tests_passed, cache_result, conform_result, preview_path, slippymap_path,
tests_passed, cache_result, conform_result, preview_path, mbtiles_path, pmtiles_path,
temp_dir)

log_handler.close()
Expand Down Expand Up @@ -207,18 +208,33 @@ def render_preview(csv_filename, temp_dir, mapbox_key):

return png_filename

def render_slippymap(csv_filename, temp_dir):
def render_mbtiles(csv_filename, temp_dir):
'''
'''
try:
mbtiles_filename = join(temp_dir, 'slippymap.mbtiles')
slippymap.generate(mbtiles_filename, csv_filename)
except Exception as e:
_L.error('%s in render_slippymap: %s', type(e), e)
_L.error('%s in render_mbtiles: %s', type(e), e)
return None
else:
return mbtiles_filename

def render_pmtiles(csv_filename, temp_dir):
'''
:param csv_filename:
:param temp_dir:
:return:
'''
try:
pmtiles_filename = join(temp_dir, 'slippymap.pmtiles')
slippymap.generate(pmtiles_filename, csv_filename)
except Exception as e:
_L.error('%s in render_pmtiles: %s', type(e), e)
return None
else:
return pmtiles_filename

class LogFilterCurrentThread:
''' Logging filter object to match only record in the current thread.
'''
Expand Down Expand Up @@ -288,7 +304,7 @@ def find_source_problem(log_contents, source):
return None

def write_state(source, layer, data_source_name, skipped, destination, log_handler, tests_passed,
cache_result, conform_result, preview_path, slippymap_path,
cache_result, conform_result, preview_path, mbtiles_path, pmtiles_path,
temp_dir):
'''
'''
Expand Down Expand Up @@ -332,9 +348,13 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl
preview_path2 = join(statedir, 'preview.png')
copy(preview_path, preview_path2)

if slippymap_path:
slippymap_path2 = join(statedir, 'slippymap.mbtiles')
copy(slippymap_path, slippymap_path2)
if mbtiles_path:
mbtiles_path2 = join(statedir, 'slippymap.mbtiles')
copy(mbtiles_path, mbtiles_path2)

if pmtiles_path:
pmtiles_path2 = join(statedir, 'slippymap.pmtiles')
copy(pmtiles_path, pmtiles_path2)

log_handler.flush()
output_path = join(statedir, 'output.txt')
Expand Down Expand Up @@ -366,7 +386,8 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl
('process time', conform_result.elapsed and str(conform_result.elapsed)),
('output', relpath(output_path, statedir)),
('preview', preview_path and relpath(preview_path2, statedir)),
('slippymap', slippymap_path and relpath(slippymap_path2, statedir)),
('slippymap', mbtiles_path and relpath(mbtiles_path2, statedir)),
('pmtiles', pmtiles_path and relpath(pmtiles_path2, statedir)),
('source problem', getattr(source_problem, 'value', None)),
('code version', __version__),
('tests passed', tests_passed),
Expand Down
23 changes: 21 additions & 2 deletions openaddr/tests/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,8 @@ def test_single_ac_local(self):

self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles'))
self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.csv'))
self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles'))
self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.csv'))

with open(state_path) as file:
state = dict(zip(*json.load(file)))
Expand All @@ -355,6 +357,7 @@ def test_single_ac_local(self):
self.assertIsNotNone(state['processed'])
self.assertIsNotNone(state['preview'])
self.assertIsNotNone(state['slippymap'])
self.assertIsNotNone(state['pmtiles'])

output_path = join(dirname(state_path), state['processed'])
with open(output_path, encoding='utf8') as input:
Expand Down Expand Up @@ -390,6 +393,8 @@ def test_single_ac(self):

self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles'))
self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.csv'))
self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles'))
self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.csv'))

with open(state_path) as file:
state = dict(zip(*json.load(file)))
Expand All @@ -398,6 +403,7 @@ def test_single_ac(self):
self.assertIsNotNone(state['processed'])
self.assertIsNotNone(state['preview'])
self.assertIsNotNone(state['slippymap'])
self.assertIsNotNone(state['pmtiles'])

output_path = join(dirname(state_path), state['processed'])
with open(output_path, encoding='utf8') as input:
Expand Down Expand Up @@ -433,6 +439,8 @@ def test_single_ac_mixedcase(self):

self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles'))
self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.csv'))
self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles'))
self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.csv'))

with open(state_path) as file:
state = dict(zip(*json.load(file)))
Expand All @@ -441,6 +449,7 @@ def test_single_ac_mixedcase(self):
self.assertIsNotNone(state['processed'])
self.assertIsNotNone(state['preview'])
self.assertIsNotNone(state['slippymap'])
self.assertIsNotNone(state['pmtiles'])

output_path = join(dirname(state_path), state['processed'])

Expand Down Expand Up @@ -473,6 +482,8 @@ def test_single_sf(self):

self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles'))
self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.csv'))
self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles'))
self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.csv'))

with open(state_path) as file:
state = dict(zip(*json.load(file)))
Expand All @@ -481,6 +492,7 @@ def test_single_sf(self):
self.assertIsNotNone(state['processed'])
self.assertIsNotNone(state['preview'])
self.assertIsNotNone(state['slippymap'])
self.assertIsNotNone(state['pmtiles'])

output_path = join(dirname(state_path), state['processed'])

Expand Down Expand Up @@ -517,6 +529,8 @@ def test_single_car(self):

self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles'))
self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.csv'))
self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles'))
self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.csv'))

with open(state_path) as file:
state = dict(zip(*json.load(file)))
Expand All @@ -526,6 +540,7 @@ def test_single_car(self):
self.assertIsNotNone(state['processed'])
self.assertIsNotNone(state['preview'])
self.assertIsNotNone(state['slippymap'])
self.assertIsNotNone(state['pmtiles'])

with open(join(dirname(state_path), state['processed'])) as file:
rows = list(DictReader(file, dialect='excel'))
Expand Down Expand Up @@ -1441,7 +1456,10 @@ def test_write_state(self):
preview_path = file.name

with open(join(self.output_dir, 'slippymap.mbtiles'), 'w') as file:
slippymap_path = file.name
mbtiles_path = file.name

with open(join(self.output_dir, 'slippymap.pmtiles'), 'w') as file:
pmtiles_path = file.name

conform_result = ConformResult(processed=None,
feat_count=999,
Expand All @@ -1460,7 +1478,8 @@ def test_write_state(self):
destination=self.output_dir, log_handler=log_handler,
cache_result=cache_result, conform_result=conform_result,
temp_dir=self.output_dir, preview_path=preview_path,
slippymap_path=slippymap_path, tests_passed=True)
mbtiles_path=mbtiles_path, pmtiles_path=pmtiles_path,
tests_passed=True)

path1 = process_one.write_state(**args)

Expand Down

0 comments on commit 2088164

Please sign in to comment.