From 3afa8821aab270e3ef217f54408af33cc4b7ad62 Mon Sep 17 00:00:00 2001 From: ingalls Date: Wed, 6 Sep 2023 23:37:08 +0100 Subject: [PATCH 01/26] Update CHANGELOG --- CHANGELOG | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 6524fe8a..01855f42 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +2023-09-06 v8.3.0 +- Update to next major alpine version +- Remove mock library in favour of unittest.mock +- Update all python deps to latest versions + 2023-09-05 v8.2.4 - Update TippeCanoe to latest version - Make GeoJSON+LD output a default From 050a4bcd56eec8298e194dc6a8c37d6297198c00 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 17:52:21 +0100 Subject: [PATCH 02/26] Output to lne delimited GeoJSON --- openaddr/__init__.py | 8 ++++---- openaddr/conform.py | 49 +++++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/openaddr/__init__.py b/openaddr/__init__.py index f376b211..ef23cd32 100644 --- a/openaddr/__init__.py +++ b/openaddr/__init__.py @@ -19,7 +19,7 @@ from .conform import ( ConformResult, DecompressionTask, - ConvertToCsvTask, + ConvertToGeojsonTask, elaborate_filenames, ADDRESSES_SCHEMA, BUILDINGS_SCHEMA, @@ -134,7 +134,7 @@ def conform(source_config, destdir, extras): decompressed_paths = task2.decompress(downloaded_path, workdir, names) _L.info("Decompressed to %d files", len(decompressed_paths)) - task4 = ConvertToCsvTask() + task4 = ConvertToGeojsonTask() try: csv_path, feat_count = task4.convert(source_config, decompressed_paths, workdir) if feat_count > 0: @@ -148,8 +148,8 @@ def conform(source_config, destdir, extras): out_path = None if csv_path is not None and exists(csv_path): - move(csv_path, join(destdir, 'out.csv')) - out_path = realpath(join(destdir, 'out.csv')) + move(csv_path, join(destdir, 'out.geojsonl')) + out_path = realpath(join(destdir, 'out.geojsonl')) rmtree(workdir) diff --git a/openaddr/conform.py b/openaddr/conform.py index 212bb709..937f64b9 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -365,7 +365,7 @@ def find_source_path(data_source, source_paths): _L.warning("Unknown source conform format %s", format_string) return None -class ConvertToCsvTask(object): +class ConvertToGeojsonTask(object): known_types = ('.shp', '.json', '.csv', '.kml', '.gdb') def convert(self, source_config, source_paths, workdir): @@ -381,7 +381,7 @@ def convert(self, source_config, source_paths, workdir): source_path = find_source_path(source_config.data_source, source_paths) if source_path is not None: basename, ext = os.path.splitext(os.path.basename(source_path)) - dest_path = os.path.join(convert_path, basename + ".csv") + dest_path = os.path.join(convert_path, basename + ".geojsonl") rc = conform_cli(source_config, source_path, dest_path) if rc == 0: with open(dest_path) as file: @@ -832,13 +832,16 @@ def row_transform_and_convert(source_config, row): # Make up a random fingerprint if none exists cache_fingerprint = source_config.data_source.get('fingerprint', str(uuid4())) - row = row_convert_to_out(source_config, row) - if source_config.layer == "addresses": - row = row_canonicalize_unit_and_number(source_config.data_source, row) row = row_round_lat_lon(source_config.data_source, row) row = row_calculate_hash(cache_fingerprint, row) + + feat = row_convert_to_out(source_config, row) + + if source_config.layer == "addresses": + feat = feat_canonicalize_unit_and_number(source_config.data_source, feat) + return row def fxn_smash_case(fxn): @@ -1041,14 +1044,14 @@ def row_fxn_constant(sc, row, key, fxn): return row -def row_canonicalize_unit_and_number(sc, row): +def feat_canonicalize_unit_and_number(sc, feat): "Canonicalize address unit and number" - row["UNIT"] = (row["UNIT"] or '').strip() - row["NUMBER"] = (row["NUMBER"] or '').strip() - if row["NUMBER"].endswith(".0"): - row["NUMBER"] = row["NUMBER"][:-2] - row["STREET"] = (row["STREET"] or '').strip() - return row + feat["properties"]["UNIT"] = (feat["properties"]["UNIT"] or '').strip() + feat["properties"]["NUMBER"] = (feat["properties"]["NUMBER"] or '').strip() + if feat["properties"]["NUMBER"].endswith(".0"): + feat["properties"]["NUMBER"] = feat["properties"]["NUMBER"][:-2] + feat["properties"]["STREET"] = (feat["properties"]["STREET"] or '').strip() + return feat def _round_wgs84_to_7(n): "Round a WGS84 coordinate to 7 decimal points. Input and output both strings." @@ -1086,21 +1089,23 @@ def row_convert_to_out(source_config, row): "Convert a row from the source schema to OpenAddresses output schema" output = { - "GEOM": row.get(GEOM_FIELDNAME.lower(), None), + "type": "Feature", + "properties": {}, + "geometry": row.get(GEOM_FIELDNAME.lower(), None), } for field in source_config.SCHEMA: if row.get('oa:{}'.format(field.lower())) is not None: # If there is an OA prefix, it is not a native field and was compiled # via an attrib funciton or concatentation - output[field] = row.get('oa:{}'.format(field.lower())) + output["properties"][field] = row.get('oa:{}'.format(field.lower())) else: # Get a native field as specified in the conform object cfield = source_config.data_source['conform'].get(field.lower()) if cfield: - output[field] = row.get(cfield.lower()) + output["properties"][field] = row.get(cfield.lower()) else: - output[field] = '' + output["properties"][field] = '' return output @@ -1134,8 +1139,8 @@ def extract_to_source_csv(source_config, source_path, extract_path): else: raise Exception("Unsupported source format %s" % format_string) -def transform_to_out_csv(source_config, extract_path, dest_path): - ''' Transform an extracted source CSV to the OpenAddresses output CSV by applying conform rules. +def transform_to_out_geojson(source_config, extract_path, dest_path): + ''' Transform an extracted source CSV to the OpenAddresses output GeoJSON by applying conform rules. source_config: description of the source, containing the conform object extract_path: extracted CSV file to process @@ -1147,14 +1152,12 @@ def transform_to_out_csv(source_config, extract_path, dest_path): # Read through the extract CSV with open(extract_path, 'r', encoding='utf-8') as extract_fp: reader = csv.DictReader(extract_fp) - # Write to the destination CSV + # Write to the destination GeoJSON with open(dest_path, 'w', encoding='utf-8') as dest_fp: - writer = csv.DictWriter(dest_fp, ['GEOM', 'HASH', *source_config.SCHEMA]) - writer.writeheader() # For every row in the extract for extract_row in reader: out_row = row_transform_and_convert(source_config, extract_row) - writer.writerow(out_row) + dest_fp.write(json.dumps(out_row) + '\n') def conform_cli(source_config, source_path, dest_path): "Command line entry point for conforming a downloaded source to an output CSV." @@ -1176,7 +1179,7 @@ def conform_cli(source_config, source_path, dest_path): try: extract_to_source_csv(source_config, source_path, extract_path) - transform_to_out_csv(source_config, extract_path, dest_path) + transform_to_out_geojson(source_config, extract_path, dest_path) finally: os.remove(extract_path) From 0d4408dd346356106bddb66d5b4bd603b2aab239 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 18:02:03 +0100 Subject: [PATCH 03/26] Parse WKT --- openaddr/conform.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openaddr/conform.py b/openaddr/conform.py index 937f64b9..55683280 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -15,6 +15,9 @@ from .geojson import stream_geojson +from shapely.wkt import loads as wkt_loads +from shapely.geometry import mapping + from zipfile import ZipFile from locale import getpreferredencoding from os.path import splitext @@ -1094,6 +1097,11 @@ def row_convert_to_out(source_config, row): "geometry": row.get(GEOM_FIELDNAME.lower(), None), } + if output["geometry"] is not None: + wkt_parsed = wkt_loads(geom_wkt) + output["geometry"] = mapping(wkt_parsed) + + for field in source_config.SCHEMA: if row.get('oa:{}'.format(field.lower())) is not None: # If there is an OA prefix, it is not a native field and was compiled From 6173699599d61d20d906305b322501c881fb10a6 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 18:04:48 +0100 Subject: [PATCH 04/26] Remove GeoJSONLD Generation in favour of native --- openaddr/conform.py | 2 +- openaddr/process_one.py | 23 ++--------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 55683280..ed2b8dc9 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1098,7 +1098,7 @@ def row_convert_to_out(source_config, row): } if output["geometry"] is not None: - wkt_parsed = wkt_loads(geom_wkt) + wkt_parsed = wkt_loads(output["geometry"]) output["geometry"] = mapping(wkt_parsed) diff --git a/openaddr/process_one.py b/openaddr/process_one.py index 10cfd2d3..6081737f 100644 --- a/openaddr/process_one.py +++ b/openaddr/process_one.py @@ -56,7 +56,7 @@ def boolstr(value): raise ValueError(repr(value)) def process(source, destination, layer, layersource, - do_preview, do_mbtiles, do_pmtiles, + do_preview, do_pmtiles, mapbox_key=None, extras=dict()): ''' Process a single source and destination, return path to JSON state file. @@ -163,29 +163,11 @@ def process(source, destination, layer, layersource, else: _L.info('Preview image in {}'.format(preview_path)) - geojsonld_path = render_geojsonld(conform_result.path, temp_dir) - - if not geojsonld_path: - _L.warning('No GeoJSON-LD generated') - else: - _L.info('GeoJSON-LD file in {}'.format(geojsonld_path)) - - if do_mbtiles: - if not geojsonld_path: - _L.error("Can't generate mbtiles without GeoJSON-LD") - else: - mbtiles_path = render_mbtiles(geojsonld_path, temp_dir) - - if not mbtiles_path: - _L.warning('No mbtiles generated') - else: - _L.info('mbtiles file in {}'.format(mbtiles_path)) - if do_pmtiles: if not geojsonld_path: _L.error("Can't generate pmtiles without GeoJSON-LD") else: - pmtiles_path = render_pmtiles(geojsonld_path, temp_dir) + pmtiles_path = render_pmtiles(conform_result.path, temp_dir) if not pmtiles_path: _L.warning('No pmtiles generated') @@ -531,7 +513,6 @@ def main(): args.layer, args.layersource, args.render_preview, args.render_preview, - args.render_preview, mapbox_key=args.mapbox_key) except Exception as e: _L.error(e, exc_info=True) From 3b80efbfe9b9a4a52d88280c06cc98f8f3d0fdd0 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 18:10:55 +0100 Subject: [PATCH 05/26] Return Feat --- openaddr/conform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index ed2b8dc9..df872482 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -845,7 +845,7 @@ def row_transform_and_convert(source_config, row): if source_config.layer == "addresses": feat = feat_canonicalize_unit_and_number(source_config.data_source, feat) - return row + return feat def fxn_smash_case(fxn): if "field" in fxn: From d287e22c760f31d69c885e83e802b0cdca1353b8 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 22:22:38 +0100 Subject: [PATCH 06/26] Get Preview and Slippy working --- openaddr/__init__.py | 4 +-- openaddr/conform.py | 2 +- openaddr/preview.py | 61 ++++++++------------------------------ openaddr/process_one.py | 65 +++++------------------------------------ 4 files changed, 23 insertions(+), 109 deletions(-) diff --git a/openaddr/__init__.py b/openaddr/__init__.py index ef23cd32..7c6df756 100644 --- a/openaddr/__init__.py +++ b/openaddr/__init__.py @@ -148,8 +148,8 @@ def conform(source_config, destdir, extras): out_path = None if csv_path is not None and exists(csv_path): - move(csv_path, join(destdir, 'out.geojsonl')) - out_path = realpath(join(destdir, 'out.geojsonl')) + move(csv_path, join(destdir, 'out.geojson')) + out_path = realpath(join(destdir, 'out.geojson')) rmtree(workdir) diff --git a/openaddr/conform.py b/openaddr/conform.py index df872482..130052c1 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -384,7 +384,7 @@ def convert(self, source_config, source_paths, workdir): source_path = find_source_path(source_config.data_source, source_paths) if source_path is not None: basename, ext = os.path.splitext(os.path.basename(source_path)) - dest_path = os.path.join(convert_path, basename + ".geojsonl") + dest_path = os.path.join(convert_path, basename + ".geojson") rc = conform_cli(source_config, source_path, dest_path) if rc == 0: with open(dest_path) as file: diff --git a/openaddr/preview.py b/openaddr/preview.py index 811db3bc..c44eaa20 100644 --- a/openaddr/preview.py +++ b/openaddr/preview.py @@ -1,13 +1,8 @@ -from __future__ import division import logging; _L = logging.getLogger('openaddr.preview') -from zipfile import ZipFile -from io import TextIOWrapper -from csv import DictReader from tempfile import mkstemp from math import pow, sqrt, pi, log from argparse import ArgumentParser -from urllib.parse import urlparse import json, itertools, os, struct import requests, uritemplate, mapbox_vector_tile @@ -30,10 +25,9 @@ # Web Mercator, https://trac.osgeo.org/openlayers/wiki/SphericalMercator EPSG900913 = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs' -def render(filename_or_url, png_filename, width, resolution, mapbox_key): +def render(src_filename, png_filename, width, resolution, mapbox_key): ''' ''' - src_filename = get_local_filename(filename_or_url) _, points_filename = mkstemp(prefix='points-', suffix='.bin') try: @@ -90,50 +84,19 @@ def render(filename_or_url, png_filename, width, resolution, mapbox_key): os.remove(points_filename) surface.write_to_png(png_filename) -def get_local_filename(filename_or_url): - ''' - ''' - parsed = urlparse(filename_or_url) - suffix = os.path.splitext(parsed.path)[1] - - if parsed.scheme in ('', 'file'): - return filename_or_url - - if parsed.scheme not in ('http', 'https'): - raise ValueError('Unknown URL type: {}'.format(filename_or_url)) - - _L.info('Downloading {}...'.format(filename_or_url)) - - got = requests.get(filename_or_url) - _, filename = mkstemp(prefix='Preview-', suffix=suffix) - - with open(filename, 'wb') as file: - file.write(got.content) - _L.debug('Saved to {}'.format(filename)) - - return filename - def iterate_file_lonlats(filename): - ''' Stream (lon, lat) coordinates from an input .csv or .zip file. + ''' Stream (lon, lat) coordinates from an input GeoJSON ''' - suffix = os.path.splitext(filename)[1].lower() - - if suffix == '.csv': - open_file = open(filename, 'r') - elif suffix == '.zip': - open_file = open(filename, 'rb') - - with open_file as file: - if suffix == '.csv': - csv_file = file - elif suffix == '.zip': - zip = ZipFile(file) - csv_names = [name for name in zip.namelist() if name.endswith('.csv')] - csv_file = TextIOWrapper(zip.open(csv_names[0])) - - for row in DictReader(csv_file): + + print('FILENAME', filename) + with open(filename, 'r') as file: + print('FILE', file); + for line in file: try: - lon, lat, x = ogr.CreateGeometryFromWkt(row['GEOM']).PointOnSurface().GetPoint() + line = json.loads(line) + lon, lat, x = ogr.CreateGeometryFromJson(json.dumps(line['geometry'])).PointOnSurface().GetPoint() + except Exception as e: + print('ERROR', e) except: continue @@ -412,7 +375,7 @@ def draw_line(ctx, start, points): parser = ArgumentParser(description='Draw a map of a single source preview.') -parser.add_argument('src_filename', help='Input Zip or CSV filename or URL.') +parser.add_argument('src_filename', help='Input GeoJSON') parser.add_argument('png_filename', help='Output PNG filename.') parser.set_defaults(resolution=1, width=668) diff --git a/openaddr/process_one.py b/openaddr/process_one.py index 6081737f..ad9c3f46 100644 --- a/openaddr/process_one.py +++ b/openaddr/process_one.py @@ -81,7 +81,7 @@ def process(source, destination, layer, layersource, with wait_lock: proc_wait.start() cache_result, conform_result = CacheResult.empty(), ConformResult.empty() - geojsonld_path, preview_path, mbtiles_path, pmtiles_path, skipped_source = None, None, None, None, False + preview_path, pmtiles_path, skipped_source = None, None, False tests_passed = None try: @@ -164,15 +164,12 @@ def process(source, destination, layer, layersource, _L.info('Preview image in {}'.format(preview_path)) if do_pmtiles: - if not geojsonld_path: - _L.error("Can't generate pmtiles without GeoJSON-LD") - else: - pmtiles_path = render_pmtiles(conform_result.path, temp_dir) + pmtiles_path = render_pmtiles(conform_result.path, temp_dir) - if not pmtiles_path: - _L.warning('No pmtiles generated') - else: - _L.info('pmtiles file in {}'.format(pmtiles_path)) + if not pmtiles_path: + _L.warning('No pmtiles generated') + else: + _L.info('pmtiles file in {}'.format(pmtiles_path)) except SourceSaysSkip: _L.info('Source says to skip in process_one.process()') @@ -190,8 +187,7 @@ def process(source, destination, layer, layersource, logging.getLogger('openaddr').removeHandler(log_handler) state_path = write_state(temp_src, layer, data_source['name'], skipped_source, destination, log_handler, - tests_passed, cache_result, conform_result, preview_path, mbtiles_path, pmtiles_path, geojsonld_path, - temp_dir) + tests_passed, cache_result, conform_result, preview_path, pmtiles_path, temp_dir) log_handler.close() rmtree(temp_dir) @@ -221,17 +217,6 @@ def render_preview(csv_filename, temp_dir, mapbox_key): return png_filename -def render_mbtiles(csv_filename, temp_dir): - ''' - ''' - try: - mbtiles_filename = join(temp_dir, 'slippymap.mbtiles') - slippymap.generate(mbtiles_filename, csv_filename) - except Exception as e: - _L.error('%s in render_mbtiles: %s', type(e), e) - return None - else: - return mbtiles_filename def render_pmtiles(csv_filename, temp_dir): ''' @@ -248,30 +233,6 @@ def render_pmtiles(csv_filename, temp_dir): else: return pmtiles_filename -def render_geojsonld(csv_filename, temp_dir): - ''' - Convert the given CSV file to GeoJSON-LD. - :return: Path to the generated GeoJSON-LD file. - ''' - geojsonld_filename = join(temp_dir, 'out.geojson') - with open(csv_filename, encoding='utf8') as csv_file: - csv_rows = csv.DictReader(csv_file) - with open(geojsonld_filename, 'w', encoding='utf8') as geojsonld_file: - for row in csv_rows: - feat = {"type": "Feature", "properties": {}, "geometry": None} - - if geom_wkt := row.pop("GEOM", None): - wkt_parsed = wkt_loads(geom_wkt) - feat["geometry"] = mapping(wkt_parsed) - - for k, v in row.items(): - feat["properties"][k.lower()] = v - - geojsonld_file.write(json.dumps(feat, sort_keys=True, separators=(',', ':'))) - geojsonld_file.write('\n') - - return geojsonld_filename - class LogFilterCurrentThread: ''' Logging filter object to match only record in the current thread. ''' @@ -341,7 +302,7 @@ def find_source_problem(log_contents, source): return None def write_state(source, layer, data_source_name, skipped, destination, log_handler, tests_passed, - cache_result, conform_result, preview_path, mbtiles_path, pmtiles_path, geojsonld_path, + cache_result, conform_result, preview_path, pmtiles_path, temp_dir): ''' ''' @@ -375,10 +336,6 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl processed_path2 = join(statedir, 'out{1}'.format(*splitext(processed_path1))) copy(processed_path1, processed_path2) - if geojsonld_path: - geojsonld_path2 = join(statedir, 'out.geojson') - copy(geojsonld_path, geojsonld_path2) - # Write the sample data to a sample.json file if conform_result.sample: sample_path = join(statedir, 'sample.json') @@ -389,10 +346,6 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl preview_path2 = join(statedir, 'preview.png') copy(preview_path, preview_path2) - if mbtiles_path: - mbtiles_path2 = join(statedir, 'slippymap.mbtiles') - copy(mbtiles_path, mbtiles_path2) - if pmtiles_path: pmtiles_path2 = join(statedir, 'slippymap.pmtiles') copy(pmtiles_path, pmtiles_path2) @@ -427,9 +380,7 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl ('process time', conform_result.elapsed and str(conform_result.elapsed)), ('output', relpath(output_path, statedir)), ('preview', preview_path and relpath(preview_path2, statedir)), - ('slippymap', mbtiles_path and relpath(mbtiles_path2, statedir)), ('pmtiles', pmtiles_path and relpath(pmtiles_path2, statedir)), - ('geojsonld', geojsonld_path and relpath(geojsonld_path2, statedir)), ('source problem', getattr(source_problem, 'value', None)), ('code version', __version__), ('tests passed', tests_passed), From a3ec11cfe5449e2b49cab778797e34162febee11 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 22:28:48 +0100 Subject: [PATCH 07/26] Lowercase Fieldnames --- openaddr/conform.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 130052c1..ff7bf8fe 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1049,11 +1049,11 @@ def row_fxn_constant(sc, row, key, fxn): def feat_canonicalize_unit_and_number(sc, feat): "Canonicalize address unit and number" - feat["properties"]["UNIT"] = (feat["properties"]["UNIT"] or '').strip() - feat["properties"]["NUMBER"] = (feat["properties"]["NUMBER"] or '').strip() - if feat["properties"]["NUMBER"].endswith(".0"): - feat["properties"]["NUMBER"] = feat["properties"]["NUMBER"][:-2] - feat["properties"]["STREET"] = (feat["properties"]["STREET"] or '').strip() + feat["properties"]["unit"] = (feat["properties"]["unit"] or '').strip() + feat["properties"]["number"] = (feat["properties"]["number"] or '').strip() + if feat["properties"]["number"].endswith(".0"): + feat["properties"]["number"] = feat["properties"]["number"][:-2] + feat["properties"]["street"] = (feat["properties"]["street"] or '').strip() return feat def _round_wgs84_to_7(n): @@ -1106,14 +1106,14 @@ def row_convert_to_out(source_config, row): if row.get('oa:{}'.format(field.lower())) is not None: # If there is an OA prefix, it is not a native field and was compiled # via an attrib funciton or concatentation - output["properties"][field] = row.get('oa:{}'.format(field.lower())) + output["properties"][field.lower()] = row.get('oa:{}'.format(field.lower())) else: # Get a native field as specified in the conform object cfield = source_config.data_source['conform'].get(field.lower()) if cfield: - output["properties"][field] = row.get(cfield.lower()) + output["properties"][field.lower()] = row.get(cfield.lower()) else: - output["properties"][field] = '' + output["properties"][field.lower()] = '' return output From d53bac6ef42d042724bc7586963a8b8c0e4e6c9b Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 22:38:47 +0100 Subject: [PATCH 08/26] Get tests running --- openaddr/tests/conform.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 671d7365..99cb6389 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -23,7 +23,7 @@ row_fxn_postfixed_unit, row_fxn_remove_prefix, row_fxn_remove_postfix, row_fxn_chain, row_fxn_first_non_empty, row_fxn_constant, - row_canonicalize_unit_and_number, conform_smash_case, conform_cli, + feat_canonicalize_unit_and_number, conform_smash_case, conform_cli, convert_regexp_replace, normalize_ogr_filename_case, is_in, geojson_source_to_csv, check_source_tests ) @@ -450,8 +450,8 @@ def test_transform_and_convert(self): 'HASH': '9574c16dfc3cc7b1' }, r) - def test_row_canonicalize_unit_and_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": "324 ", "STREET": " OAK DR.", "UNIT": "1"}) + def test_feat_canonicalize_unit_and_number(self): + r = feat_canonicalize_unit_and_number({}, {"NUMBER": "324 ", "STREET": " OAK DR.", "UNIT": "1"}) self.assertEqual("324", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("1", r["UNIT"]) @@ -462,17 +462,17 @@ def test_row_canonicalize_unit_and_number(self): ("3240", "3240"), ("INVALID", "INVALID"), ("324.5", "324.5")): - r = row_canonicalize_unit_and_number({}, {"NUMBER": a, "STREET": "", "UNIT": ""}) + r = feat_canonicalize_unit_and_number({}, {"NUMBER": a, "STREET": "", "UNIT": ""}) self.assertEqual(e, r["NUMBER"]) - def test_row_canonicalize_street_and_no_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) + def test_feat_canonicalize_street_and_no_number(self): + r = feat_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) self.assertEqual("", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("", r["UNIT"]) - def test_row_canonicalize_street_with_no_unit_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) + def test_feat_canonicalize_street_with_no_unit_number(self): + r = feat_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) self.assertEqual("", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("", r["UNIT"]) From 1384fd84a2be13a37806a4d31a00b32ab7d4ae30 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 8 Sep 2023 22:55:21 +0100 Subject: [PATCH 09/26] Rename to csv => out --- openaddr/__init__.py | 12 ++++++------ openaddr/tests/conform.py | 27 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/openaddr/__init__.py b/openaddr/__init__.py index 7c6df756..dcd0e68b 100644 --- a/openaddr/__init__.py +++ b/openaddr/__init__.py @@ -136,19 +136,19 @@ def conform(source_config, destdir, extras): task4 = ConvertToGeojsonTask() try: - csv_path, feat_count = task4.convert(source_config, decompressed_paths, workdir) + out_path, feat_count = task4.convert(source_config, decompressed_paths, workdir) if feat_count > 0: - _L.info("Converted to %s with %d features", csv_path, feat_count) + _L.info("Converted to %s with %d features", out_path, feat_count) else: _L.warning('Found no features in source data') - csv_path = None + out_path = None except Exception as e: _L.warning("Error doing conform; skipping", exc_info=True) - csv_path, feat_count = None, 0 + out_path, feat_count = None, 0 out_path = None - if csv_path is not None and exists(csv_path): - move(csv_path, join(destdir, 'out.geojson')) + if out_path is not None and exists(out_path): + move(out_path, join(destdir, 'out.geojson')) out_path = realpath(join(destdir, 'out.geojson')) rmtree(workdir) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 99cb6389..1574e857 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -377,16 +377,21 @@ def test_transform_and_convert(self): r = row_transform_and_convert(d, { "n": "123", "s1": "MAPLE", "s2": "ST", "oa:geom": "POINT (-119.2 39.3)"}) self.assertEqual({ - "STREET": "MAPLE ST", - "UNIT": "", - "NUMBER": "123", - "GEOM": "POINT (-119.2 39.3)", - "CITY": "", - "REGION": "", - "DISTRICT": "", - "POSTCODE": "", - "ID": "", - 'HASH': '9574c16dfc3cc7b1' + 'type': 'Feature' + 'properties': { + "street": "MAPLE ST", + "unit": "", + "number": "123", + "city": "", + "region": "", + "district": "", + "postcode": "", + "id": "", + 'hash': '9574c16dfc3cc7b1' + }, + 'geometry': { + 'coordinates': (-119.2, 39.3), 'type': 'Point' + } }, r) d = SourceConfig(dict({ @@ -1635,7 +1640,7 @@ def test_row_fxn_constant(self): d = row_fxn_constant(c, d, "region", c["conform"]["region"]) self.assertEqual(e, d) - + "constant - replacing non-empty, non-standard string" c = { "conform": { "region": { From 309da66d01281298796fb2930d8eb16c5a976678 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 12 Sep 2023 14:38:34 +0100 Subject: [PATCH 10/26] Remove zip test as zip mode was removed --- openaddr/conform.py | 18 +++++++--------- openaddr/preview.py | 8 +++---- openaddr/tests/__init__.py | 43 +------------------------------------- openaddr/tests/conform.py | 42 +++++++++++++++++++++---------------- openaddr/tests/preview.py | 26 ----------------------- 5 files changed, 36 insertions(+), 101 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index ff7bf8fe..8c5f38db 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -837,14 +837,12 @@ def row_transform_and_convert(source_config, row): if source_config.layer == "addresses": row = row_round_lat_lon(source_config.data_source, row) + row = row_canonicalize_unit_and_number(source_config.data_source, row) row = row_calculate_hash(cache_fingerprint, row) feat = row_convert_to_out(source_config, row) - if source_config.layer == "addresses": - feat = feat_canonicalize_unit_and_number(source_config.data_source, feat) - return feat def fxn_smash_case(fxn): @@ -1047,14 +1045,14 @@ def row_fxn_constant(sc, row, key, fxn): return row -def feat_canonicalize_unit_and_number(sc, feat): +def row_canonicalize_unit_and_number(sc, row): "Canonicalize address unit and number" - feat["properties"]["unit"] = (feat["properties"]["unit"] or '').strip() - feat["properties"]["number"] = (feat["properties"]["number"] or '').strip() - if feat["properties"]["number"].endswith(".0"): - feat["properties"]["number"] = feat["properties"]["number"][:-2] - feat["properties"]["street"] = (feat["properties"]["street"] or '').strip() - return feat + row["unit"] = (row["unit"] or '').strip() + row["number"] = (row["number"] or '').strip() + if row["number"].endswith(".0"): + row["number"] = row["number"][:-2] + row["street"] = (row["street"] or '').strip() + return row def _round_wgs84_to_7(n): "Round a WGS84 coordinate to 7 decimal points. Input and output both strings." diff --git a/openaddr/preview.py b/openaddr/preview.py index c44eaa20..60bdceda 100644 --- a/openaddr/preview.py +++ b/openaddr/preview.py @@ -88,21 +88,19 @@ def iterate_file_lonlats(filename): ''' Stream (lon, lat) coordinates from an input GeoJSON ''' - print('FILENAME', filename) with open(filename, 'r') as file: - print('FILE', file); for line in file: try: line = json.loads(line) lon, lat, x = ogr.CreateGeometryFromJson(json.dumps(line['geometry'])).PointOnSurface().GetPoint() + + if -180 <= lon <= 180 and -90 <= lat <= 90: + yield (lon, lat) except Exception as e: print('ERROR', e) except: continue - if -180 <= lon <= 180 and -90 <= lat <= 90: - yield (lon, lat) - def get_map_features(xmin, ymin, xmax, ymax, resolution, scale, mapbox_key): ''' ''' diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 3bc9da6c..50401206 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -354,7 +354,6 @@ def test_single_ac_local(self): render_geojsonld.side_effect = return_path_in_second_arg_dir state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) @@ -366,7 +365,6 @@ def test_single_ac_local(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) - self.assertIsNotNone(state['slippymap']) self.assertIsNotNone(state['pmtiles']) self.assertIsNotNone(state['geojsonld']) @@ -404,7 +402,6 @@ def test_single_ac(self): render_geojsonld.side_effect = return_path_in_second_arg_dir state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) @@ -416,7 +413,6 @@ def test_single_ac(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) - self.assertIsNotNone(state['slippymap']) self.assertIsNotNone(state['pmtiles']) output_path = join(dirname(state_path), state['processed']) @@ -453,7 +449,6 @@ def test_single_ac_mixedcase(self): render_geojsonld.side_effect = return_path_in_second_arg_dir state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) @@ -465,7 +460,6 @@ def test_single_ac_mixedcase(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) - self.assertIsNotNone(state['slippymap']) self.assertIsNotNone(state['pmtiles']) output_path = join(dirname(state_path), state['processed']) @@ -499,7 +493,6 @@ def test_single_sf(self): render_geojsonld.side_effect = return_path_in_second_arg_dir state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) @@ -511,7 +504,6 @@ def test_single_sf(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) - self.assertIsNotNone(state['slippymap']) self.assertIsNotNone(state['pmtiles']) output_path = join(dirname(state_path), state['processed']) @@ -549,7 +541,6 @@ def test_single_car(self): render_geojsonld.side_effect = return_path_in_second_arg_dir state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) @@ -562,7 +553,6 @@ def test_single_car(self): self.assertEqual(state['fingerprint'], '23082fe4819682a6934b61443560160c') self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) - self.assertIsNotNone(state['slippymap']) self.assertIsNotNone(state['pmtiles']) with open(join(dirname(state_path), state['processed'])) as file: @@ -592,7 +582,6 @@ def test_single_car_cached(self): self.assertEqual(state['fingerprint'], '1821b2e50a61ed04ac2213fbc7a1984d') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) with open(join(dirname(state_path), state['processed'])) as file: self.assertTrue('555,CARSON ST' in file.read()) @@ -612,7 +601,6 @@ def test_single_car_old_cached(self): self.assertEqual(state['fingerprint'], '1821b2e50a61ed04ac2213fbc7a1984d') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) with open(join(dirname(state_path), state['processed'])) as file: self.assertTrue('555,CARSON ST' in file.read()) @@ -631,7 +619,6 @@ def test_single_tx_runnels(self): self.assertIsNone(state['cache']) self.assertIsNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) # This test data does not contain a working conform object self.assertEqual(state['source problem'], "Missing required ESRI token") @@ -653,7 +640,6 @@ def test_single_oak(self): self.assertEqual(state['source problem'], "Unknown source conform format") self.assertIsNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) def test_single_oak_skip(self): ''' Test complete process_one.process on Oakland sample data. @@ -672,7 +658,6 @@ def test_single_oak_skip(self): self.assertIsNone(state["cache"]) self.assertIsNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) def test_single_berk(self): ''' Test complete process_one.process on Berkeley sample data. @@ -690,7 +675,6 @@ def test_single_berk(self): self.assertEqual(state["source problem"], "Source is missing a conform object") self.assertIsNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) def test_single_berk_404(self): @@ -708,7 +692,6 @@ def test_single_berk_404(self): self.assertIsNone(state["cache"]) self.assertIsNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) def test_single_berk_apn(self): ''' Test complete process_one.process on Berkeley sample data. @@ -724,7 +707,6 @@ def test_single_berk_apn(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -757,7 +739,6 @@ def test_single_pl_ds(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_pl_l(self): ''' Test complete process_one.process on Polish sample data. @@ -773,7 +754,6 @@ def test_single_pl_l(self): self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -803,7 +783,6 @@ def test_single_jp_fukushima2(self): self.assertIsNone(state["source problem"]) self.assertIsNotNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) with open(join(dirname(state_path), state["processed"]), encoding='utf8') as file: rows = list(csv.DictReader(file)) @@ -831,7 +810,6 @@ def test_single_utah(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_iceland(self): ''' Test complete process_one.process. @@ -845,7 +823,6 @@ def test_single_iceland(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['cache']) self.assertIsNotNone(state['geojsonld']) @@ -878,7 +855,6 @@ def test_single_fr_paris(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_fr_lareunion(self): ''' Test complete process_one.process on data that uses non-UTF8 encoding (issue #136) @@ -901,7 +877,6 @@ def test_single_fr_lareunion(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_va_statewide(self): ''' Test complete process_one.process on data with non-OGR .csv filename. @@ -915,7 +890,6 @@ def test_single_va_statewide(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_oh_trumbull(self): @@ -930,7 +904,6 @@ def test_single_oh_trumbull(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_ks_brown(self): @@ -945,7 +918,6 @@ def test_single_ks_brown(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_pa_lancaster(self): @@ -960,7 +932,6 @@ def test_single_pa_lancaster(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -988,7 +959,6 @@ def test_single_ua_kharkiv(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_pa_bucks(self): ''' Test complete process_one.process on data with ESRI multiPolyline geometries. @@ -1002,7 +972,6 @@ def test_single_pa_bucks(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -1030,7 +999,6 @@ def test_single_nm_washington(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -1077,7 +1045,6 @@ def test_single_tx_waco(self): self.assertIsNone(state["source problem"]) self.assertIsNotNone(state["processed"]) self.assertIsNone(state["preview"]) - self.assertIsNone(state["slippymap"]) output_path = join(dirname(state_path), state["processed"]) @@ -1171,7 +1138,6 @@ def test_single_de_berlin(self): self.assertEqual(rows[2]['STREET'], u'Alte Jakobstra\xdfe') self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) def test_single_us_or_portland(self): ''' Test complete process_one.process on data. @@ -1377,7 +1343,6 @@ def test_single_lake_man_gdb(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -1409,7 +1374,6 @@ def test_single_lake_man_gdb_nested(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -1441,7 +1405,6 @@ def test_single_lake_man_gdb_nested_nodir(self): state = dict(zip(*json.load(file))) self.assertIsNone(state['preview']) - self.assertIsNone(state['slippymap']) output_path = join(dirname(state_path), state['processed']) @@ -1487,9 +1450,6 @@ def test_write_state(self): with open(join(self.output_dir, 'preview.png'), 'w') as file: preview_path = file.name - with open(join(self.output_dir, 'slippymap.mbtiles'), 'w') as file: - mbtiles_path = file.name - with open(join(self.output_dir, 'slippymap.pmtiles'), 'w') as file: pmtiles_path = file.name @@ -1513,7 +1473,7 @@ def test_write_state(self): destination=self.output_dir, log_handler=log_handler, cache_result=cache_result, conform_result=conform_result, temp_dir=self.output_dir, preview_path=preview_path, - mbtiles_path=mbtiles_path, pmtiles_path=pmtiles_path, + pmtiles_path=pmtiles_path, geojsonld_path=geojsonld_path, tests_passed=True) @@ -1533,7 +1493,6 @@ def test_write_state(self): self.assertEqual(state1['process time'], '0:00:01') self.assertEqual(state1['output'], 'output.txt') self.assertEqual(state1['preview'], 'preview.png') - self.assertEqual(state1['slippymap'], 'slippymap.mbtiles') self.assertEqual(state1['geojsonld'], 'out.geojson') self.assertEqual(state1['tests passed'], True) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 1574e857..9300b4c3 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -23,7 +23,7 @@ row_fxn_postfixed_unit, row_fxn_remove_prefix, row_fxn_remove_postfix, row_fxn_chain, row_fxn_first_non_empty, row_fxn_constant, - feat_canonicalize_unit_and_number, conform_smash_case, conform_cli, + row_canonicalize_unit_and_number, conform_smash_case, conform_cli, convert_regexp_replace, normalize_ogr_filename_case, is_in, geojson_source_to_csv, check_source_tests ) @@ -72,15 +72,21 @@ def test_row_convert_to_out(self): }) self.assertEqual({ - "GEOM": "POINT (-119.2 39.3)", - "UNIT": "", - "NUMBER": "123", - "STREET": "MAPLE LN", - "CITY": "", - "REGION": "", - "DISTRICT": "", - "POSTCODE": "", - "ID": "" + "type": "Feature", + "properties": { + "unit": "", + "number": "123", + "street": "MAPLE LN", + "city": "", + "region": "", + "district": "", + "postcode": "", + "id": "" + }, + "geometry": { + "type": "Point", + "coordinates": (-119.2, 39.3) + } }, r) def test_row_merge(self): @@ -377,7 +383,7 @@ def test_transform_and_convert(self): r = row_transform_and_convert(d, { "n": "123", "s1": "MAPLE", "s2": "ST", "oa:geom": "POINT (-119.2 39.3)"}) self.assertEqual({ - 'type': 'Feature' + 'type': 'Feature', 'properties': { "street": "MAPLE ST", "unit": "", @@ -455,8 +461,8 @@ def test_transform_and_convert(self): 'HASH': '9574c16dfc3cc7b1' }, r) - def test_feat_canonicalize_unit_and_number(self): - r = feat_canonicalize_unit_and_number({}, {"NUMBER": "324 ", "STREET": " OAK DR.", "UNIT": "1"}) + def test_row_canonicalize_unit_and_number(self): + r = row_canonicalize_unit_and_number({}, {"NUMBER": "324 ", "STREET": " OAK DR.", "UNIT": "1"}) self.assertEqual("324", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("1", r["UNIT"]) @@ -467,17 +473,17 @@ def test_feat_canonicalize_unit_and_number(self): ("3240", "3240"), ("INVALID", "INVALID"), ("324.5", "324.5")): - r = feat_canonicalize_unit_and_number({}, {"NUMBER": a, "STREET": "", "UNIT": ""}) + r = row_canonicalize_unit_and_number({}, {"NUMBER": a, "STREET": "", "UNIT": ""}) self.assertEqual(e, r["NUMBER"]) - def test_feat_canonicalize_street_and_no_number(self): - r = feat_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) + def test_row_canonicalize_street_and_no_number(self): + r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) self.assertEqual("", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("", r["UNIT"]) - def test_feat_canonicalize_street_with_no_unit_number(self): - r = feat_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) + def test_row_canonicalize_street_with_no_unit_number(self): + r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) self.assertEqual("", r["NUMBER"]) self.assertEqual("OAK DR.", r["STREET"]) self.assertEqual("", r["UNIT"]) diff --git a/openaddr/tests/preview.py b/openaddr/tests/preview.py index fdaeac74..1dd021bc 100644 --- a/openaddr/tests/preview.py +++ b/openaddr/tests/preview.py @@ -41,32 +41,6 @@ def test_calculate_bounds(self): bbox = preview.calculate_bounds(points_filename) self.assertEqual(bbox, (-1.04, -1.04, 1.04, 1.04), 'The two outliers are ignored') - def test_render_zip(self): - ''' - ''' - def response_content(url, request): - if url.hostname == 'a.tiles.mapbox.com' and url.path.startswith('/v4/mapbox.mapbox-streets-v7'): - if 'access_token=mapbox-XXXX' not in url.query: - raise ValueError('Missing or wrong API key') - data = b'\x1a\'x\x02\n\x05water(\x80 \x12\x19\x18\x03"\x13\t\xe0\x7f\xff\x1f\x1a\x00\xe0\x9f\x01\xdf\x9f\x01\x00\x00\xdf\x9f\x01\x0f\x08\x00' - return response(200, data, headers={'Content-Type': 'application/vnd.mapbox-vector-tile'}) - raise Exception("Uknown URL") - - zip_filename = join(dirname(__file__), 'outputs', 'alameda_geom.zip') - handle, png_filename = tempfile.mkstemp(prefix='render-', suffix='.png') - os.close(handle) - - try: - with HTTMock(response_content): - preview.render(zip_filename, png_filename, 668, 1, 'mapbox-XXXX') - info = str(subprocess.check_output(('file', png_filename))) - - self.assertTrue('PNG image data' in info) - self.assertTrue('668 x 573' in info) - self.assertTrue('8-bit/color RGB' in info) - finally: - os.remove(png_filename) - def test_render_csv(self): ''' ''' From d9c041c0c79f0372c013aaf10d4cf575295a645a Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 12 Sep 2023 15:19:35 +0100 Subject: [PATCH 11/26] Remove zip test as zip mode was removed --- openaddr/tests/preview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openaddr/tests/preview.py b/openaddr/tests/preview.py index 1dd021bc..a27742d3 100644 --- a/openaddr/tests/preview.py +++ b/openaddr/tests/preview.py @@ -41,7 +41,7 @@ def test_calculate_bounds(self): bbox = preview.calculate_bounds(points_filename) self.assertEqual(bbox, (-1.04, -1.04, 1.04, 1.04), 'The two outliers are ignored') - def test_render_csv(self): + def test_render_geojson(self): ''' ''' def response_content(url, request): From c239467b577419483b16ac7c005d14fa6104e3c5 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 12 Sep 2023 15:37:55 +0100 Subject: [PATCH 12/26] Get Preview Tests working --- openaddr/preview.py | 1 + .../outputs/denver-metro-preview.geojson | 100 ++++++++++++++++++ openaddr/tests/preview.py | 16 +-- 3 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 openaddr/tests/outputs/denver-metro-preview.geojson diff --git a/openaddr/preview.py b/openaddr/preview.py index 60bdceda..2229f661 100644 --- a/openaddr/preview.py +++ b/openaddr/preview.py @@ -92,6 +92,7 @@ def iterate_file_lonlats(filename): for line in file: try: line = json.loads(line) + lon, lat, x = ogr.CreateGeometryFromJson(json.dumps(line['geometry'])).PointOnSurface().GetPoint() if -180 <= lon <= 180 and -90 <= lat <= 90: diff --git a/openaddr/tests/outputs/denver-metro-preview.geojson b/openaddr/tests/outputs/denver-metro-preview.geojson new file mode 100644 index 00000000..6fc5f801 --- /dev/null +++ b/openaddr/tests/outputs/denver-metro-preview.geojson @@ -0,0 +1,100 @@ +{"type": "Feature", "properties": {"number": "7080", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771107963831, 39.8250547879637]}} +{"type": "Feature", "properties": {"number": "7050", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771064688673, 39.8246937966513]}} +{"type": "Feature", "properties": {"number": "18600", "street": "E 71st Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.770112258211, 39.8248690852886]}} +{"type": "Feature", "properties": {"number": "18625", "street": "E 70th Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.770129017272, 39.8239440499257]}} +{"type": "Feature", "properties": {"number": "7010", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771125542895, 39.8239776700479]}} +{"type": "Feature", "properties": {"number": "6900", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771328199631, 39.822680187331]}} +{"type": "Feature", "properties": {"number": "6890", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771174065071, 39.8213672615579]}} +{"type": "Feature", "properties": {"number": "17675", "street": "E 64th Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.782577813529, 39.8164436876585]}} +{"type": "Feature", "properties": {"number": "7001", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.773231931552, 39.8241032365044]}} +{"type": "Feature", "properties": {"number": "6851", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.773174573344, 39.8213960916627]}} +{"type": "Feature", "properties": {"number": "17500", "street": "E 64th Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.781214460572, 39.812545523487]}} +{"type": "Feature", "properties": {"number": "6290", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771137282937, 39.8105315971655]}} +{"type": "Feature", "properties": {"number": "18600", "street": "E 63rd Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.769828865181, 39.810386232637]}} +{"type": "Feature", "properties": {"number": "6210", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.771232571284, 39.809241924477]}} +{"type": "Feature", "properties": {"number": "6250", "street": "N Tower Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.77151758232, 39.8099025458786]}} +{"type": "Feature", "properties": {"number": "4801", "street": "N Picadilly Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.740278348751, 39.785953405813]}} +{"type": "Feature", "properties": {"number": "16100", "street": "E 56th Ave", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.799805662159, 39.7981243450596]}} +{"type": "Feature", "properties": {"number": "5212", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807432620887, 39.7894749696154]}} +{"type": "Feature", "properties": {"number": "5222", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807413255816, 39.7896340791938]}} +{"type": "Feature", "properties": {"number": "5231", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808072529213, 39.789898755156]}} +{"type": "Feature", "properties": {"number": "5221", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807992776923, 39.7896869023858]}} +{"type": "Feature", "properties": {"number": "5211", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807966014826, 39.7895137927565]}} +{"type": "Feature", "properties": {"number": "5201", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80798923891, 39.7893245070684]}} +{"type": "Feature", "properties": {"number": "5210", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808316655101, 39.7893209400554]}} +{"type": "Feature", "properties": {"number": "5220", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808300580042, 39.7895075227337]}} +{"type": "Feature", "properties": {"number": "5230", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808341490191, 39.7896889513933]}} +{"type": "Feature", "properties": {"number": "5240", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808406960429, 39.7899062101831]}} +{"type": "Feature", "properties": {"number": "5250", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808544569929, 39.7900250626152]}} +{"type": "Feature", "properties": {"number": "5260", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808771600755, 39.7900977728796]}} +{"type": "Feature", "properties": {"number": "5270", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809077644867, 39.7900968238761]}} +{"type": "Feature", "properties": {"number": "5265", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809281662609, 39.7899772314413]}} +{"type": "Feature", "properties": {"number": "5255", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809305311695, 39.7897440205935]}} +{"type": "Feature", "properties": {"number": "5245", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809192987287, 39.7895868850222]}} +{"type": "Feature", "properties": {"number": "5235", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80886934011, 39.7895685109555]}} +{"type": "Feature", "properties": {"number": "5225", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808807484885, 39.7893457831457]}} +{"type": "Feature", "properties": {"number": "5215", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809106327972, 39.7893530281721]}} +{"type": "Feature", "properties": {"number": "5205", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809282256611, 39.7891948365969]}} +{"type": "Feature", "properties": {"number": "5195", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809298951672, 39.7889451146891]}} +{"type": "Feature", "properties": {"number": "5185", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809093678926, 39.7888313562756]}} +{"type": "Feature", "properties": {"number": "5175", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808783676799, 39.7888734634286]}} +{"type": "Feature", "properties": {"number": "5165", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808804255873, 39.7885908224011]}} +{"type": "Feature", "properties": {"number": "5155", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809095761933, 39.7886199835071]}} +{"type": "Feature", "properties": {"number": "5145", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809296028661, 39.7885195851421]}} +{"type": "Feature", "properties": {"number": "5135", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809309245709, 39.7882616072043]}} +{"type": "Feature", "properties": {"number": "5125", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809172125211, 39.7880933445925]}} +{"type": "Feature", "properties": {"number": "5115", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808901803228, 39.7880807745469]}} +{"type": "Feature", "properties": {"number": "5105", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808839895003, 39.7878635367571]}} +{"type": "Feature", "properties": {"number": "5085", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809082531885, 39.7877963315128]}} +{"type": "Feature", "properties": {"number": "5075", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80918031024, 39.7876239538861]}} +{"type": "Feature", "properties": {"number": "5065", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80927091257, 39.7874570242793]}} +{"type": "Feature", "properties": {"number": "5055", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809297473666, 39.787289717671]}} +{"type": "Feature", "properties": {"number": "5045", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809291981646, 39.7871249730721]}} +{"type": "Feature", "properties": {"number": "5035", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809289960639, 39.7869684795032]}} +{"type": "Feature", "properties": {"number": "5025", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809298974672, 39.7867763648048]}} +{"type": "Feature", "properties": {"number": "5015", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809115517005, 39.786616066222]}} +{"type": "Feature", "properties": {"number": "5020", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808784794803, 39.7865949091451]}} +{"type": "Feature", "properties": {"number": "5030", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808587771086, 39.7867282676299]}} +{"type": "Feature", "properties": {"number": "5040", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808656499336, 39.7869757425296]}} +{"type": "Feature", "properties": {"number": "5050", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808754527693, 39.7871382901205]}} +{"type": "Feature", "properties": {"number": "5060", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80878129079, 39.7873113987499]}} +{"type": "Feature", "properties": {"number": "5070", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80866585837, 39.7874699443262]}} +{"type": "Feature", "properties": {"number": "5081", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808415283459, 39.7872598325624]}} +{"type": "Feature", "properties": {"number": "5071", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808338924182, 39.7870644718521]}} +{"type": "Feature", "properties": {"number": "5061", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808273049942, 39.786888388212]}} +{"type": "Feature", "properties": {"number": "5051", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808261174899, 39.7866494823435]}} +{"type": "Feature", "properties": {"number": "5041", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807980649879, 39.7865901851279]}} +{"type": "Feature", "properties": {"number": "5052", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807599416493, 39.7869997316168]}} +{"type": "Feature", "properties": {"number": "5062", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807779639148, 39.7871270720797]}} +{"type": "Feature", "properties": {"number": "5072", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807863709454, 39.7872620825706]}} +{"type": "Feature", "properties": {"number": "5082", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807919200656, 39.7874079091007]}} +{"type": "Feature", "properties": {"number": "5092", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807931752701, 39.7875781897197]}} +{"type": "Feature", "properties": {"number": "5093", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807597414486, 39.7875624946627]}} +{"type": "Feature", "properties": {"number": "5083", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807517233194, 39.7873945660522]}} +{"type": "Feature", "properties": {"number": "5073", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807415480825, 39.7872484665211]}} +{"type": "Feature", "properties": {"number": "5063", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807310091441, 39.7871105850198]}} +{"type": "Feature", "properties": {"number": "5044", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.806676868139, 39.7870931329563]}} +{"type": "Feature", "properties": {"number": "5054", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.806653320054, 39.7873153587643]}} +{"type": "Feature", "properties": {"number": "5064", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.806833455709, 39.7874509362571]}} +{"type": "Feature", "properties": {"number": "5084", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.806989017274, 39.7875534256297]}} +{"type": "Feature", "properties": {"number": "5094", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807108580709, 39.7876968851513]}} +{"type": "Feature", "properties": {"number": "5202", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807462848997, 39.7892967059673]}} +{"type": "Feature", "properties": {"number": "5232", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807354751604, 39.7897929607714]}} +{"type": "Feature", "properties": {"number": "5242", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80739210074, 39.7899743684309]}} +{"type": "Feature", "properties": {"number": "5252", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807618723563, 39.7900882548449]}} +{"type": "Feature", "properties": {"number": "5241", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807960376805, 39.7900847728323]}} +{"type": "Feature", "properties": {"number": "5201", "street": "N Idalia St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807342282558, 39.7891931025907]}} +{"type": "Feature", "properties": {"number": "5032", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807610463533, 39.7866017351699]}} +{"type": "Feature", "properties": {"number": "5042", "street": "N Helena St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807420390842, 39.7867516047148]}} +{"type": "Feature", "properties": {"number": "5053", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807151954867, 39.7869092522878]}} +{"type": "Feature", "properties": {"number": "5043", "street": "N Helena Ct", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.806906357974, 39.7869160423126]}} +{"type": "Feature", "properties": {"number": "5180", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.8083163751, 39.7889887638478]}} +{"type": "Feature", "properties": {"number": "5170", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808321613119, 39.7888185882292]}} +{"type": "Feature", "properties": {"number": "5160", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808323103124, 39.7886676056802]}} +{"type": "Feature", "properties": {"number": "5150", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.80834950022, 39.7885167741319]}} +{"type": "Feature", "properties": {"number": "5140", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808400781407, 39.7883688335941]}} +{"type": "Feature", "properties": {"number": "5130", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808427204503, 39.7882152530357]}} +{"type": "Feature", "properties": {"number": "5110", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808396726392, 39.7880585934662]}} +{"type": "Feature", "properties": {"number": "5100", "street": "N Hannibal St", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.808355568242, 39.7879018738965]}} +{"type": "Feature", "properties": {"number": "4878", "street": "N Chambers Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.809028670689, 39.7859000036188]}} +{"type": "Feature", "properties": {"number": "4870", "street": "N Chambers Rd", "unit": "", "city": "", "district": "", "region": "", "postcode": "", "id": ""}, "geometry": {"type": "Point", "coordinates": [-104.807993999928, 39.785894284598]}} diff --git a/openaddr/tests/preview.py b/openaddr/tests/preview.py index a27742d3..db9bc525 100644 --- a/openaddr/tests/preview.py +++ b/openaddr/tests/preview.py @@ -6,7 +6,6 @@ import subprocess from os.path import join, dirname -from zipfile import ZipFile from shutil import rmtree from httmock import HTTMock, response @@ -52,28 +51,23 @@ def response_content(url, request): return response(200, data, headers={'Content-Type': 'application/vnd.mapbox-vector-tile'}) raise Exception("Uknown URL") - zip_filename = join(dirname(__file__), 'outputs', 'portland_metro_geom.zip') handle, png_filename = tempfile.mkstemp(prefix='render-', suffix='.png') os.close(handle) try: - temp_dir = tempfile.mkdtemp(prefix='test_render_csv-') - zipfile = ZipFile(zip_filename) - - with open(join(temp_dir, 'portland.csv'), 'wb') as file: - file.write(zipfile.read('portland_metro/us/or/portland_metro.csv')) - csv_filename = file.name + temp_dir = tempfile.mkdtemp(prefix='test_render_geojson-') with HTTMock(response_content): - preview.render(csv_filename, png_filename, 668, 1, 'mapbox-XXXX') + preview.render(join(dirname(__file__), 'outputs', 'denver-metro-preview.geojson'), png_filename, 668, 1, 'mapbox-XXXX') + + print(png_filename) info = str(subprocess.check_output(('file', png_filename))) self.assertTrue('PNG image data' in info) - self.assertTrue('668 x 289' in info) + self.assertTrue('668 x 493' in info) self.assertTrue('8-bit/color RGB' in info) finally: os.remove(png_filename) - os.remove(csv_filename) os.rmdir(temp_dir) def test_get_map_features(self): From f3da66855d3dc2a8b7c54103db5a1ea73a2d1a04 Mon Sep 17 00:00:00 2001 From: ingalls Date: Thu, 14 Sep 2023 15:25:25 +0200 Subject: [PATCH 13/26] Get Conform Tests Passing --- openaddr/conform.py | 19 +++++++------ openaddr/tests/conform.py | 60 ++++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 8c5f38db..10ec43ce 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -44,9 +44,9 @@ def gdal_error_handler(err_class, err_num, err_msg): # We add columns to the extracted CSV with our own data with these names. GEOM_FIELDNAME = 'OA:GEOM' -ADDRESSES_SCHEMA = [ 'NUMBER', 'STREET', 'UNIT', 'CITY', 'DISTRICT', 'REGION', 'POSTCODE', 'ID' ] -BUILDINGS_SCHEMA = [] -PARCELS_SCHEMA = [ 'PID' ] +ADDRESSES_SCHEMA = [ 'HASH', 'NUMBER', 'STREET', 'UNIT', 'CITY', 'DISTRICT', 'REGION', 'POSTCODE', 'ID' ] +BUILDINGS_SCHEMA = [ 'HASH'] +PARCELS_SCHEMA = [ 'HASH', 'PID' ] RESERVED_SCHEMA = ADDRESSES_SCHEMA + BUILDINGS_SCHEMA + PARCELS_SCHEMA + [ "LAT", "LON" @@ -1047,11 +1047,11 @@ def row_fxn_constant(sc, row, key, fxn): def row_canonicalize_unit_and_number(sc, row): "Canonicalize address unit and number" - row["unit"] = (row["unit"] or '').strip() - row["number"] = (row["number"] or '').strip() - if row["number"].endswith(".0"): - row["number"] = row["number"][:-2] - row["street"] = (row["street"] or '').strip() + row["UNIT"] = (row.get("UNIT", '') or '').strip() + row["NUMBER"] = (row.get("NUMBER", '') or '').strip() + if row["NUMBER"].endswith(".0"): + row["NUMBER"] = row["NUMBER"][:-2] + row["STREET"] = (row.get("STREET", '') or '').strip() return row def _round_wgs84_to_7(n): @@ -1082,7 +1082,7 @@ def row_calculate_hash(cache_fingerprint, row): ''' hash = sha1(cache_fingerprint.encode('utf8')) hash.update(json.dumps(sorted(row.items()), separators=(',', ':')).encode('utf8')) - row.update(HASH=hash.hexdigest()[:16]) + row.update({'oa:hash': hash.hexdigest()[:16]}) return row @@ -1108,6 +1108,7 @@ def row_convert_to_out(source_config, row): else: # Get a native field as specified in the conform object cfield = source_config.data_source['conform'].get(field.lower()) + if cfield: output["properties"][field.lower()] = row.get(cfield.lower()) else: diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 9300b4c3..f15997b8 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -81,6 +81,7 @@ def test_row_convert_to_out(self): "region": "", "district": "", "postcode": "", + "hash": "", "id": "" }, "geometry": { @@ -392,11 +393,12 @@ def test_transform_and_convert(self): "region": "", "district": "", "postcode": "", - "id": "", - 'hash': '9574c16dfc3cc7b1' + 'hash': '406f6b64991cefda', + "id": "" }, 'geometry': { - 'coordinates': (-119.2, 39.3), 'type': 'Point' + 'coordinates': (-119.2, 39.3), + 'type': 'Point' } }, r) @@ -412,16 +414,22 @@ def test_transform_and_convert(self): r = row_transform_and_convert(d, { "n": "123", "s1": "MAPLE", "s2": "ST", GEOM_FIELDNAME: "POINT(-119.2 39.3)"}) self.assertEqual({ - "STREET": "MAPLE ST", - "UNIT": "", - "NUMBER": "123", - "GEOM": "POINT (-119.2 39.3)", - "CITY": "", - "REGION": "", - "DISTRICT": "", - "POSTCODE": "", - "ID": "", - 'HASH': '9574c16dfc3cc7b1' + 'type': 'Feature', + 'properties': { + "street": "MAPLE ST", + "unit": "", + "number": "123", + "city": "", + "region": "", + "district": "", + "postcode": "", + "id": "", + 'hash': '7abe394dfe59bed8' + }, + 'geometry': { + "type": "Point", + "coordinates": (-119.2, 39.3) + } }, r) d = SourceConfig(dict({ @@ -449,16 +457,22 @@ def test_transform_and_convert(self): }), "addresses", "default") r = row_transform_and_convert(d, { "s": "123 MAPLE ST", GEOM_FIELDNAME: "POINT(-119.2 39.3)" }) self.assertEqual({ - "STREET": "MAPLE ST", - "UNIT": "", - "NUMBER": "123", - "GEOM": "POINT (-119.2 39.3)", - "CITY": "", - "REGION": "", - "DISTRICT": "", - "POSTCODE": "", - "ID": "", - 'HASH': '9574c16dfc3cc7b1' + 'type': 'Feature', + 'geometry': { + "type": "Point", + "coordinates": (-119.2, 39.3) + }, + 'properties': { + "street": "MAPLE ST", + "unit": "", + "number": "123", + "city": "", + "region": "", + "district": "", + "postcode": "", + "id": "", + 'hash': 'e388ce063150808f' + } }, r) def test_row_canonicalize_unit_and_number(self): From 7fc28bb0fbc1076d6437430f1652eb9dae8e7ce3 Mon Sep 17 00:00:00 2001 From: ingalls Date: Thu, 14 Sep 2023 15:59:20 +0200 Subject: [PATCH 14/26] Get Conform tests passing --- openaddr/conform.py | 12 +- openaddr/tests/conform.py | 285 ++++++++++++++++++++------------------ 2 files changed, 160 insertions(+), 137 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 10ec43ce..2120f145 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1047,11 +1047,13 @@ def row_fxn_constant(sc, row, key, fxn): def row_canonicalize_unit_and_number(sc, row): "Canonicalize address unit and number" - row["UNIT"] = (row.get("UNIT", '') or '').strip() - row["NUMBER"] = (row.get("NUMBER", '') or '').strip() - if row["NUMBER"].endswith(".0"): - row["NUMBER"] = row["NUMBER"][:-2] - row["STREET"] = (row.get("STREET", '') or '').strip() + row["unit"] = (row.get("unit", '') or '').strip() + row["number"] = (row.get("number", '') or '').strip() + + if row["number"].endswith(".0"): + row["number"] = row["number"][:-2] + + row["street"] = (row.get("street", '') or '').strip() return row def _round_wgs84_to_7(n): diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index f15997b8..e9c73759 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -393,7 +393,7 @@ def test_transform_and_convert(self): "region": "", "district": "", "postcode": "", - 'hash': '406f6b64991cefda', + 'hash': '09212154c33070c2', "id": "" }, 'geometry': { @@ -476,10 +476,10 @@ def test_transform_and_convert(self): }, r) def test_row_canonicalize_unit_and_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": "324 ", "STREET": " OAK DR.", "UNIT": "1"}) - self.assertEqual("324", r["NUMBER"]) - self.assertEqual("OAK DR.", r["STREET"]) - self.assertEqual("1", r["UNIT"]) + r = row_canonicalize_unit_and_number({}, {"number": "324 ", "street": " OAK DR.", "unit": "1"}) + self.assertEqual("324", r["number"]) + self.assertEqual("OAK DR.", r["street"]) + self.assertEqual("1", r["unit"]) # Tests for integer conversion for e, a in (("324", " 324.0 "), @@ -487,20 +487,20 @@ def test_row_canonicalize_unit_and_number(self): ("3240", "3240"), ("INVALID", "INVALID"), ("324.5", "324.5")): - r = row_canonicalize_unit_and_number({}, {"NUMBER": a, "STREET": "", "UNIT": ""}) - self.assertEqual(e, r["NUMBER"]) + r = row_canonicalize_unit_and_number({}, {"number": a, "street": "", "unit": ""}) + self.assertEqual(e, r["number"]) def test_row_canonicalize_street_and_no_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) - self.assertEqual("", r["NUMBER"]) - self.assertEqual("OAK DR.", r["STREET"]) - self.assertEqual("", r["UNIT"]) + r = row_canonicalize_unit_and_number({}, {"number": None, "street": " OAK DR.", "unit": None}) + self.assertEqual("", r["number"]) + self.assertEqual("OAK DR.", r["street"]) + self.assertEqual("", r["unit"]) def test_row_canonicalize_street_with_no_unit_number(self): - r = row_canonicalize_unit_and_number({}, {"NUMBER": None, "STREET": " OAK DR.", "UNIT": None}) - self.assertEqual("", r["NUMBER"]) - self.assertEqual("OAK DR.", r["STREET"]) - self.assertEqual("", r["UNIT"]) + r = row_canonicalize_unit_and_number({}, {"number": None, "street": " OAK DR.", "unit": None}) + self.assertEqual("", r["number"]) + self.assertEqual("OAK DR.", r["street"]) + self.assertEqual("", r["unit"]) def test_row_round_lat_lon(self): r = row_round_lat_lon({}, {"GEOM": "POINT (39.14285717777 -121.20)"}) @@ -1730,136 +1730,138 @@ def test_lake_man(self): self.assertEqual(0, rc) with open(dest_path) as fp: - reader = csv.DictReader(fp) - self.assertEqual([ - 'GEOM', 'HASH', 'NUMBER', 'STREET', 'UNIT', 'CITY', 'DISTRICT', 'REGION', 'POSTCODE', 'ID' - ], reader.fieldnames) + rows = list(map(json.loads, list(fp))) - rows = list(reader) - - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592497 37.8026126)') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) self.assertEqual(6, len(rows)) - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[1]['NUMBER'], '5121') - self.assertEqual(rows[1]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[2]['NUMBER'], '5133') - self.assertEqual(rows[2]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[3]['NUMBER'], '5126') - self.assertEqual(rows[3]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[4]['NUMBER'], '5120') - self.assertEqual(rows[4]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[5]['NUMBER'], '5115') - self.assertEqual(rows[5]['STREET'], 'OLD MILL RD') + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') def test_lake_man_gdb(self): rc, dest_path = self._run_conform_on_source('lake-man-gdb', 'gdb') self.assertEqual(0, rc) with open(dest_path) as fp: - reader = csv.DictReader(fp) - self.assertEqual([ - 'GEOM', 'HASH', 'NUMBER', 'STREET', 'UNIT', 'CITY', 'DISTRICT', 'REGION', 'POSTCODE', 'ID' - ], reader.fieldnames) - - rows = list(reader) + rows = list(map(json.loads, list(fp))) - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592497 37.8026126)') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) self.assertEqual(6, len(rows)) - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[1]['NUMBER'], '5121') - self.assertEqual(rows[1]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[2]['NUMBER'], '5133') - self.assertEqual(rows[2]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[3]['NUMBER'], '5126') - self.assertEqual(rows[3]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[4]['NUMBER'], '5120') - self.assertEqual(rows[4]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[5]['NUMBER'], '5115') - self.assertEqual(rows[5]['STREET'], 'OLD MILL RD') + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') def test_lake_man_split(self): rc, dest_path = self._run_conform_on_source('lake-man-split', 'shp') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['NUMBER'], '915') - self.assertEqual(rows[0]['STREET'], 'EDWARD AVE') - self.assertEqual(rows[1]['NUMBER'], '3273') - self.assertEqual(rows[1]['STREET'], 'PETER ST') - self.assertEqual(rows[2]['NUMBER'], '976') - self.assertEqual(rows[2]['STREET'], 'FORD BLVD') - self.assertEqual(rows[3]['NUMBER'], '7055') - self.assertEqual(rows[3]['STREET'], 'ST ROSE AVE') - self.assertEqual(rows[4]['NUMBER'], '534') - self.assertEqual(rows[4]['STREET'], 'WALLACE AVE') - self.assertEqual(rows[5]['NUMBER'], '531') - self.assertEqual(rows[5]['STREET'], 'SCOFIELD AVE') + rows = list(map(json.loads, list(fp))) + + self.assertEqual(rows[0]['properties']['number'], '915') + self.assertEqual(rows[0]['properties']['street'], 'EDWARD AVE') + self.assertEqual(rows[1]['properties']['number'], '3273') + self.assertEqual(rows[1]['properties']['street'], 'PETER ST') + self.assertEqual(rows[2]['properties']['number'], '976') + self.assertEqual(rows[2]['properties']['street'], 'FORD BLVD') + self.assertEqual(rows[3]['properties']['number'], '7055') + self.assertEqual(rows[3]['properties']['street'], 'ST ROSE AVE') + self.assertEqual(rows[4]['properties']['number'], '534') + self.assertEqual(rows[4]['properties']['street'], 'WALLACE AVE') + self.assertEqual(rows[5]['properties']['number'], '531') + self.assertEqual(rows[5]['properties']['street'], 'SCOFIELD AVE') def test_lake_man_merge_postcode(self): rc, dest_path = self._run_conform_on_source('lake-man-merge-postcode', 'shp') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['NUMBER'], '35845') - self.assertEqual(rows[0]['STREET'], 'EKLUTNA LAKE RD') - self.assertEqual(rows[1]['NUMBER'], '35850') - self.assertEqual(rows[1]['STREET'], 'EKLUTNA LAKE RD') - self.assertEqual(rows[2]['NUMBER'], '35900') - self.assertEqual(rows[2]['STREET'], 'EKLUTNA LAKE RD') - self.assertEqual(rows[3]['NUMBER'], '35870') - self.assertEqual(rows[3]['STREET'], 'EKLUTNA LAKE RD') - self.assertEqual(rows[4]['NUMBER'], '32551') - self.assertEqual(rows[4]['STREET'], 'EKLUTNA LAKE RD') - self.assertEqual(rows[5]['NUMBER'], '31401') - self.assertEqual(rows[5]['STREET'], 'EKLUTNA LAKE RD') + rows = list(map(json.loads, list(fp))) + + self.assertEqual(rows[0]['properties']['number'], '35845') + self.assertEqual(rows[0]['properties']['street'], 'EKLUTNA LAKE RD') + self.assertEqual(rows[1]['properties']['number'], '35850') + self.assertEqual(rows[1]['properties']['street'], 'EKLUTNA LAKE RD') + self.assertEqual(rows[2]['properties']['number'], '35900') + self.assertEqual(rows[2]['properties']['street'], 'EKLUTNA LAKE RD') + self.assertEqual(rows[3]['properties']['number'], '35870') + self.assertEqual(rows[3]['properties']['street'], 'EKLUTNA LAKE RD') + self.assertEqual(rows[4]['properties']['number'], '32551') + self.assertEqual(rows[4]['properties']['street'], 'EKLUTNA LAKE RD') + self.assertEqual(rows[5]['properties']['number'], '31401') + self.assertEqual(rows[5]['properties']['street'], 'EKLUTNA LAKE RD') def test_lake_man_merge_postcode2(self): rc, dest_path = self._run_conform_on_source('lake-man-merge-postcode2', 'shp') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['NUMBER'], '85') - self.assertEqual(rows[0]['STREET'], 'MAITLAND DR') - self.assertEqual(rows[1]['NUMBER'], '81') - self.assertEqual(rows[1]['STREET'], 'MAITLAND DR') - self.assertEqual(rows[2]['NUMBER'], '92') - self.assertEqual(rows[2]['STREET'], 'MAITLAND DR') - self.assertEqual(rows[3]['NUMBER'], '92') - self.assertEqual(rows[3]['STREET'], 'MAITLAND DR') - self.assertEqual(rows[4]['NUMBER'], '92') - self.assertEqual(rows[4]['STREET'], 'MAITLAND DR') - self.assertEqual(rows[5]['NUMBER'], '92') - self.assertEqual(rows[5]['STREET'], 'MAITLAND DR') + rows = list(map(json.loads, list(fp))) + + self.assertEqual(rows[0]['properties']['number'], '85') + self.assertEqual(rows[0]['properties']['street'], 'MAITLAND DR') + self.assertEqual(rows[1]['properties']['number'], '81') + self.assertEqual(rows[1]['properties']['street'], 'MAITLAND DR') + self.assertEqual(rows[2]['properties']['number'], '92') + self.assertEqual(rows[2]['properties']['street'], 'MAITLAND DR') + self.assertEqual(rows[3]['properties']['number'], '92') + self.assertEqual(rows[3]['properties']['street'], 'MAITLAND DR') + self.assertEqual(rows[4]['properties']['number'], '92') + self.assertEqual(rows[4]['properties']['street'], 'MAITLAND DR') + self.assertEqual(rows[5]['properties']['number'], '92') + self.assertEqual(rows[5]['properties']['street'], 'MAITLAND DR') def test_lake_man_shp_utf8(self): rc, dest_path = self._run_conform_on_source('lake-man-utf8', 'shp') self.assertEqual(0, rc) with open(dest_path, encoding='utf-8') as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['STREET'], u'PZ ESPA\u00d1A') + rows = list(map(json.loads, list(fp))) + + self.assertEqual(rows[0]['properties']['street'], u'PZ ESPA\u00d1A') def test_lake_man_shp_epsg26943(self): rc, dest_path = self._run_conform_on_source('lake-man-epsg26943', 'shp') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592497 37.8026126)') + rows = list(map(json.loads, list(fp))) + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) def test_lake_man_shp_noprj_epsg26943(self): rc, dest_path = self._run_conform_on_source('lake-man-epsg26943-noprj', 'shp') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592497 37.8026126)') + rows = list(map(json.loads, list(fp))) + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) # TODO: add tests for non-ESRI GeoJSON sources @@ -1869,62 +1871,81 @@ def test_lake_man_split2(self): self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['NUMBER'], '1') - self.assertEqual(rows[0]['STREET'], 'Spectrum Pointe Dr #320') - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.25925 37.802613)') - self.assertEqual(rows[1]['NUMBER'], '') - self.assertEqual(rows[1]['STREET'], '') - self.assertEqual(rows[1]['GEOM'], 'POINT (-122.256718 37.802528)') - self.assertEqual(rows[2]['NUMBER'], '300') - self.assertEqual(rows[2]['STREET'], 'E Chapman Ave') - self.assertEqual(rows[2]['GEOM'], 'POINT (-122.257941 37.802969)') - self.assertEqual(rows[3]['NUMBER'], '1') - self.assertEqual(rows[3]['STREET'], 'Spectrum Pointe Dr #320') - self.assertEqual(rows[3]['GEOM'], 'POINT (-122.258971 37.800748)') - self.assertEqual(rows[4]['NUMBER'], '1') - self.assertEqual(rows[4]['STREET'], 'Spectrum Pointe Dr #320') - self.assertEqual(rows[4]['GEOM'], 'POINT (-122.256954 37.800714)') - self.assertEqual(rows[5]['NUMBER'], '1') - self.assertEqual(rows[5]['STREET'], 'Spectrum Pointe Dr #320') - self.assertEqual(rows[5]['GEOM'], 'POINT (-122.25764 37.804359)') + rows = list(map(json.loads, list(fp))) + self.assertEqual(rows[0]['properties']['number'], '1') + self.assertEqual(rows[0]['properties']['street'], 'Spectrum Pointe Dr #320') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(rows[1]['properties']['number'], '') + self.assertEqual(rows[1]['properties']['street'], '') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(rows[2]['properties']['number'], '300') + self.assertEqual(rows[2]['properties']['street'], 'E Chapman Ave') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(rows[3]['properties']['number'], '1') + self.assertEqual(rows[3]['properties']['street'], 'Spectrum Pointe Dr #320') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(rows[4]['properties']['number'], '1') + self.assertEqual(rows[4]['properties']['street'], 'Spectrum Pointe Dr #320') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(rows[5]['properties']['number'], '1') + self.assertEqual(rows[5]['properties']['street'], 'Spectrum Pointe Dr #320') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.25925, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.802613, rows[0]['geometry']['coordinates'][1], places=4) def test_nara_jp(self): "Test case from jp-nara.json" rc, dest_path = self._run_conform_on_source('jp-nara', 'csv') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['NUMBER'], '2543-6') - self.assertEqual(rows[0]['GEOM'], 'POINT (135.955104 34.607832)') - self.assertEqual(rows[0]['STREET'], u'\u91dd\u753a') - self.assertEqual(rows[1]['NUMBER'], '202-6') + rows = list(map(json.loads, list(fp))) + self.assertEqual(rows[0]['properties']['number'], '2543-6') + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(135.955104, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(34.607832, rows[0]['geometry']['coordinates'][1], places=4) + self.assertEqual(rows[0]['properties']['street'], u'\u91dd\u753a') + self.assertEqual(rows[1]['properties']['number'], '202-6') def test_lake_man_3740(self): "CSV in an oddball SRS" rc, dest_path = self._run_conform_on_source('lake-man-3740', 'csv') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) - x,y= wkt_pt(rows[0]['GEOM']) + rows = list(map(json.loads, list(fp))) # POINT (-122.2592495 37.8026123) - self.assertAlmostEqual(-122.2592495, x, places=4) - self.assertAlmostEqual(37.8026123, y, places=4) + self.assertAlmostEqual(-122.2592495, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026123, rows[0]['geometry']['coordinates'][1], places=4) - self.assertEqual(rows[0]['NUMBER'], '5') - self.assertEqual(rows[0]['STREET'], u'PZ ESPA\u00d1A') + self.assertEqual(rows[0]['properties']['number'], '5') + self.assertEqual(rows[0]['properties']['street'], u'PZ ESPA\u00d1A') def test_lake_man_gml(self): "GML XML files" rc, dest_path = self._run_conform_on_source('lake-man-gml', 'gml') self.assertEqual(0, rc) with open(dest_path) as fp: - rows = list(csv.DictReader(fp)) + rows = list(map(json.loads, list(fp))) self.assertEqual(6, len(rows)) - self.assertEqual(rows[0]['GEOM'], 'POINT (37.8026126 -122.2592497)') - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][1], places=4) + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') class TestConformMisc(unittest.TestCase): From 99d66c7c658a3e2b9f6dd3ed63f413233b54aa84 Mon Sep 17 00:00:00 2001 From: ingalls Date: Thu, 14 Sep 2023 21:58:30 +0200 Subject: [PATCH 15/26] Fix TestState tests --- openaddr/tests/__init__.py | 50 ++++++++------------------------------ openaddr/tests/conform.py | 4 +-- 2 files changed, 12 insertions(+), 42 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 50401206..6ecb8091 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -347,17 +347,14 @@ def test_single_ac_local(self): with HTTMock(self.response_content), \ mock.patch('openaddr.preview.render') as preview_ren, \ - mock.patch('openaddr.slippymap.generate') as slippymap_gen, \ - mock.patch('openaddr.process_one.render_geojsonld') as render_geojsonld: + mock.patch('openaddr.slippymap.generate') as slippymap_gen: preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file - render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) - self.assertTrue(render_geojsonld.mock_calls[0][1][0].endswith('.csv')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -366,7 +363,6 @@ def test_single_ac_local(self): self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) self.assertIsNotNone(state['pmtiles']) - self.assertIsNotNone(state['geojsonld']) output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: @@ -395,17 +391,14 @@ def test_single_ac(self): with HTTMock(self.response_content), \ mock.patch('openaddr.preview.render') as preview_ren, \ - mock.patch('openaddr.slippymap.generate') as slippymap_gen, \ - mock.patch('openaddr.process_one.render_geojsonld') as render_geojsonld: + mock.patch('openaddr.slippymap.generate') as slippymap_gen: preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file - render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) - self.assertTrue(render_geojsonld.mock_calls[0][1][0].endswith('.csv')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -442,17 +435,14 @@ def test_single_ac_mixedcase(self): with HTTMock(self.response_content), \ mock.patch('openaddr.preview.render') as preview_ren, \ - mock.patch('openaddr.slippymap.generate') as slippymap_gen, \ - mock.patch('openaddr.process_one.render_geojsonld') as render_geojsonld: + mock.patch('openaddr.slippymap.generate') as slippymap_gen: preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file - render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) - self.assertTrue(render_geojsonld.mock_calls[0][1][0].endswith('.csv')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -486,17 +476,14 @@ def test_single_sf(self): with HTTMock(self.response_content), \ mock.patch('openaddr.preview.render') as preview_ren, \ - mock.patch('openaddr.slippymap.generate') as slippymap_gen, \ - mock.patch('openaddr.process_one.render_geojsonld') as render_geojsonld: + mock.patch('openaddr.slippymap.generate') as slippymap_gen: preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file - render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) - self.assertTrue(render_geojsonld.mock_calls[0][1][0].endswith('.csv')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -534,17 +521,14 @@ def test_single_car(self): with HTTMock(self.response_content), \ mock.patch('openaddr.preview.render') as preview_ren, \ - mock.patch('openaddr.slippymap.generate') as slippymap_gen, \ - mock.patch('openaddr.process_one.render_geojsonld') as render_geojsonld: + mock.patch('openaddr.slippymap.generate') as slippymap_gen: preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file - render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) - self.assertTrue(render_geojsonld.mock_calls[0][1][0].endswith('.csv')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -825,17 +809,8 @@ def test_single_iceland(self): self.assertIsNone(state['preview']) self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['cache']) - self.assertIsNotNone(state['geojsonld']) with open(join(dirname(state_path), state['processed']), encoding='utf8') as file: - rows = list(csv.DictReader(file)) - - self.assertEqual(len(rows), 15) - self.assertEqual(rows[0]['STREET'], u'2.Gata v/Rauðavatn') - self.assertEqual(rows[2]['GEOM'], 'POINT (-21.7684622 64.110974)') - self.assertEqual(rows[3]['GEOM'], 'POINT (-21.7665982 64.1100444)') - - with open(join(dirname(state_path), state['geojsonld']), encoding='utf8') as file: features = [json.loads(line) for line in file] self.assertEqual(len(features), 15) @@ -1453,9 +1428,6 @@ def test_write_state(self): with open(join(self.output_dir, 'slippymap.pmtiles'), 'w') as file: pmtiles_path = file.name - with open(join(self.output_dir, 'out.geojsonld'), 'w') as file: - geojsonld_path = file.name - conform_result = ConformResult(processed=None, feat_count=999, path=processed_path, @@ -1474,7 +1446,6 @@ def test_write_state(self): cache_result=cache_result, conform_result=conform_result, temp_dir=self.output_dir, preview_path=preview_path, pmtiles_path=pmtiles_path, - geojsonld_path=geojsonld_path, tests_passed=True) path1 = process_one.write_state(**args) @@ -1493,7 +1464,6 @@ def test_write_state(self): self.assertEqual(state1['process time'], '0:00:01') self.assertEqual(state1['output'], 'output.txt') self.assertEqual(state1['preview'], 'preview.png') - self.assertEqual(state1['geojsonld'], 'out.geojson') self.assertEqual(state1['tests passed'], True) # diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index e9c73759..34e2cf08 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -424,7 +424,7 @@ def test_transform_and_convert(self): "district": "", "postcode": "", "id": "", - 'hash': '7abe394dfe59bed8' + 'hash': '3372445616223f37' }, 'geometry': { "type": "Point", @@ -471,7 +471,7 @@ def test_transform_and_convert(self): "district": "", "postcode": "", "id": "", - 'hash': 'e388ce063150808f' + 'hash': '7bfd895bef404ea4' } }, r) From 84d33870052ff0bc3193973d03b27fec6abb91b9 Mon Sep 17 00:00:00 2001 From: ingalls Date: Fri, 15 Sep 2023 16:06:43 +0200 Subject: [PATCH 16/26] Continue to update tests --- openaddr/__init__.py | 1 - openaddr/tests/__init__.py | 66 ++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/openaddr/__init__.py b/openaddr/__init__.py index dcd0e68b..b5427d3d 100644 --- a/openaddr/__init__.py +++ b/openaddr/__init__.py @@ -146,7 +146,6 @@ def conform(source_config, destdir, extras): _L.warning("Error doing conform; skipping", exc_info=True) out_path, feat_count = None, 0 - out_path = None if out_path is not None and exists(out_path): move(out_path, join(destdir, 'out.geojson')) out_path = realpath(join(destdir, 'out.geojson')) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 6ecb8091..6f3069e7 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -366,7 +366,7 @@ def test_single_ac_local(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['ID'], '') self.assertEqual(rows[10]['ID'], '') self.assertEqual(rows[100]['ID'], '') @@ -410,7 +410,7 @@ def test_single_ac(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['ID'], '') self.assertEqual(rows[10]['ID'], '') self.assertEqual(rows[100]['ID'], '') @@ -455,7 +455,7 @@ def test_single_ac_mixedcase(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['ID'], '') self.assertEqual(rows[10]['ID'], '') self.assertEqual(rows[100]['ID'], '') @@ -496,7 +496,7 @@ def test_single_sf(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['ID'], '') self.assertEqual(rows[10]['ID'], '') self.assertEqual(rows[100]['ID'], '') @@ -695,7 +695,7 @@ def test_single_berk_apn(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['ID'], '055 188300600') self.assertEqual(rows[10]['ID'], '055 189504000') self.assertEqual(rows[100]['ID'], '055 188700100') @@ -742,7 +742,7 @@ def test_single_pl_l(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['NUMBER'], u'5') self.assertEqual(rows[10]['NUMBER'], u'8') self.assertEqual(rows[100]['NUMBER'], u'5a') @@ -769,7 +769,7 @@ def test_single_jp_fukushima2(self): self.assertIsNone(state["preview"]) with open(join(dirname(state_path), state["processed"]), encoding='utf8') as file: - rows = list(csv.DictReader(file)) + rows = list(map(json.loads, list(file))) self.assertEqual(len(rows), 6) self.assertEqual(rows[0]['NUMBER'], u'24-9') @@ -911,7 +911,7 @@ def test_single_pa_lancaster(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['UNIT'], u'2') self.assertEqual(rows[11]['UNIT'], u'11') self.assertEqual(rows[21]['UNIT'], u'') @@ -951,7 +951,7 @@ def test_single_pa_bucks(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['UNIT'], u'') self.assertEqual(rows[10]['UNIT'], u'') self.assertEqual(rows[20]['UNIT'], u'') @@ -978,7 +978,7 @@ def test_single_nm_washington(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[1]['UNIT'], u'') self.assertEqual(rows[5]['UNIT'], u'') self.assertEqual(rows[9]['UNIT'], u'') @@ -1024,7 +1024,7 @@ def test_single_tx_waco(self): output_path = join(dirname(state_path), state["processed"]) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[0]['REGION'], u'TX') self.assertEqual(rows[0]['ID'], u'') self.assertEqual(rows[0]['NUMBER'], u'308') @@ -1052,16 +1052,18 @@ def test_single_wy_park(self): output_path = join(dirname(state_path), state["processed"]) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) - self.assertEqual(rows[0]['ID'], u'') - self.assertEqual(rows[0]['NUMBER'], u'162') - self.assertEqual(rows[0]['HASH'], u'fa774c4d6e199cb1') - self.assertEqual(rows[0]['CITY'], u'') - self.assertEqual(rows[0]['GEOM'], u'POINT (-108.7563613 44.7538737)') - self.assertEqual(rows[0]['STREET'], u'N CLARK ST') - self.assertEqual(rows[0]['POSTCODE'], u'') - self.assertEqual(rows[0]['UNIT'], u'') - self.assertEqual(rows[0]['DISTRICT'], u'') + rows = list(map(json.loads, list(input))) + self.assertEqual(rows[0]['properties']['id'], u'') + self.assertEqual(rows[0]['properties']['number'], u'162') + self.assertEqual(rows[0]['properties']['hash'], u'd666aba81e7f9b01') + self.assertEqual(rows[0]['properties']['CITY'], u'') + self.assertEqual(rows[0]['geometry']['type'], 'Point'); + self.assertAlmostEqual(rows[0]['geometry']['coordinates'][0], -108.7563613); + self.assertAlmostEqual(rows[0]['geometry']['coordinates'][1], 44.7538737); + self.assertEqual(rows[0]['properties']['street'], u'N CLARK ST') + self.assertEqual(rows[0]['properties']['postcode'], u'') + self.assertEqual(rows[0]['properties']['unit'], u'') + self.assertEqual(rows[0]['properties']['district'], u'') def test_single_ny_orange(self): ''' Test complete process_one.process on data NaN values in ESRI response. @@ -1079,7 +1081,7 @@ def test_single_ny_orange(self): output_path = join(dirname(state_path), state["processed"]) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[0]['ID'], u'') self.assertEqual(rows[0]['NUMBER'], u'434') self.assertEqual(rows[0]['HASH'], u'58a4d4fbbf126d86') @@ -1104,7 +1106,7 @@ def test_single_de_berlin(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(rows[0]['NUMBER'], u'72') self.assertEqual(rows[1]['NUMBER'], u'3') self.assertEqual(rows[2]['NUMBER'], u'75') @@ -1128,7 +1130,7 @@ def test_single_us_or_portland(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 12) self.assertEqual(rows[2]['NUMBER'], u'1') self.assertEqual(rows[3]['NUMBER'], u'10') @@ -1157,7 +1159,7 @@ def test_single_nl_countrywide(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 8) self.assertEqual(rows[0]['NUMBER'], u'34x') self.assertEqual(rows[1]['NUMBER'], u'65-x') @@ -1182,8 +1184,8 @@ def test_single_be_wa_brussels(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) - self.assertEqual(len(rows), 666) + rows = list(map(json.loads, list(input))) + self.assertEqual(len(rows), 665) self.assertEqual(rows[0]['NUMBER'], u'2') self.assertEqual(rows[0]['STREET'], u'Rue de la Victoire') self.assertEqual(rows[1]['NUMBER'], u'16') @@ -1223,7 +1225,7 @@ def test_single_it_52_statewide(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 19) self.assertEqual(rows[0]['NUMBER'], u'33') self.assertEqual(rows[0]['STREET'], u'VIA CARLO CARRÀ') @@ -1249,7 +1251,7 @@ def test_single_us_nj_statewide(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 1045) self.assertEqual(rows[0]['NUMBER'], u'7') self.assertEqual(rows[0]['STREET'], u'Sagamore Avenue') @@ -1322,7 +1324,7 @@ def test_single_lake_man_gdb(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) self.assertEqual(rows[0]['NUMBER'], '5115') self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') @@ -1353,7 +1355,7 @@ def test_single_lake_man_gdb_nested(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) self.assertEqual(rows[0]['NUMBER'], '5115') self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') @@ -1384,7 +1386,7 @@ def test_single_lake_man_gdb_nested_nodir(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: - rows = list(csv.DictReader(input)) + rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) self.assertEqual(rows[0]['NUMBER'], '5115') self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') From 964f363af0a54c46d475f47000651d5c9c4ed36d Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 06:00:27 +0200 Subject: [PATCH 17/26] Continue massaging tests into final form --- openaddr/conform.py | 39 ++- openaddr/tests/__init__.py | 502 ++++++++++++++++++------------------- openaddr/tests/conform.py | 32 +-- 3 files changed, 268 insertions(+), 305 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 2120f145..0784c26e 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -835,14 +835,16 @@ def row_transform_and_convert(source_config, row): # Make up a random fingerprint if none exists cache_fingerprint = source_config.data_source.get('fingerprint', str(uuid4())) - if source_config.layer == "addresses": - row = row_round_lat_lon(source_config.data_source, row) - row = row_canonicalize_unit_and_number(source_config.data_source, row) - row = row_calculate_hash(cache_fingerprint, row) feat = row_convert_to_out(source_config, row) + if source_config.layer == "addresses": + feat['properties'] = row_canonicalize_unit_and_number(source_config.data_source, feat['properties']) + + if feat['geometry'] and len(feat['geometry']['coordinates']) > 0: + feat['geometry']['coordinates'] = set_precision(feat['geometry']['coordinates'], 7) + return feat def fxn_smash_case(fxn): @@ -1049,33 +1051,22 @@ def row_canonicalize_unit_and_number(sc, row): "Canonicalize address unit and number" row["unit"] = (row.get("unit", '') or '').strip() row["number"] = (row.get("number", '') or '').strip() + row["street"] = (row.get("street", '') or '').strip() - if row["number"].endswith(".0"): + if row.get("number", '').endswith('.0'): row["number"] = row["number"][:-2] - row["street"] = (row.get("street", '') or '').strip() return row -def _round_wgs84_to_7(n): - "Round a WGS84 coordinate to 7 decimal points. Input and output both strings." +def set_precision(coords, precision): + result = [] try: - return "%.12g" % round(float(n), 7) - except: - return n - -def row_round_lat_lon(sc, row): - "Round WGS84 coordinates to 1cm precision" - if row.get('GEOM') is not None and 'POINT' in row['GEOM']: - try: - geom = ogr.CreateGeometryFromWkt(row['GEOM']) - x = _round_wgs84_to_7(geom.GetX()) - y = _round_wgs84_to_7(geom.GetY()) + return round(coords, int(precision)) + except TypeError: + for coord in coords: + result.append(set_precision(coord, precision)) - row['GEOM'] = ogr.CreateGeometryFromWkt('POINT ({} {})'.format(x, y)).ExportToWkt() - except Exception: - pass - - return row + return result def row_calculate_hash(cache_fingerprint, row): ''' Calculate row hash based on content and existing fingerprint. diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 6f3069e7..0d8a089f 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -367,22 +367,22 @@ def test_single_ac_local(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['ID'], '') - self.assertEqual(rows[10]['ID'], '') - self.assertEqual(rows[100]['ID'], '') - self.assertEqual(rows[1000]['ID'], '') - self.assertEqual(rows[1]['NUMBER'], '2147') - self.assertEqual(rows[10]['NUMBER'], '605') - self.assertEqual(rows[100]['NUMBER'], '167') - self.assertEqual(rows[1000]['NUMBER'], '322') - self.assertEqual(rows[1]['STREET'], 'BROADWAY') - self.assertEqual(rows[10]['STREET'], 'HILLSBOROUGH ST') - self.assertEqual(rows[100]['STREET'], '8TH ST') - self.assertEqual(rows[1000]['STREET'], 'HANOVER AV') - self.assertEqual(rows[1]['UNIT'], '') - self.assertEqual(rows[10]['UNIT'], '') - self.assertEqual(rows[100]['UNIT'], '') - self.assertEqual(rows[1000]['UNIT'], '') + self.assertEqual(rows[1]['properties']['id'], '') + self.assertEqual(rows[10]['properties']['id'], '') + self.assertEqual(rows[100]['properties']['id'], '') + self.assertEqual(rows[1000]['properties']['id'], '') + self.assertEqual(rows[1]['properties']['number'], '2147') + self.assertEqual(rows[10]['properties']['number'], '605') + self.assertEqual(rows[100]['properties']['number'], '167') + self.assertEqual(rows[1000]['properties']['number'], '322') + self.assertEqual(rows[1]['properties']['street'], 'BROADWAY') + self.assertEqual(rows[10]['properties']['street'], 'HILLSBOROUGH ST') + self.assertEqual(rows[100]['properties']['street'], '8TH ST') + self.assertEqual(rows[1000]['properties']['street'], 'HANOVER AV') + self.assertEqual(rows[1]['properties']['unit'], '') + self.assertEqual(rows[10]['properties']['unit'], '') + self.assertEqual(rows[100]['properties']['unit'], '') + self.assertEqual(rows[1000]['properties']['unit'], '') def test_single_ac(self): ''' Test complete process_one.process on Alameda County sample data. @@ -411,22 +411,22 @@ def test_single_ac(self): output_path = join(dirname(state_path), state['processed']) with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['ID'], '') - self.assertEqual(rows[10]['ID'], '') - self.assertEqual(rows[100]['ID'], '') - self.assertEqual(rows[1000]['ID'], '') - self.assertEqual(rows[1]['NUMBER'], '2147') - self.assertEqual(rows[10]['NUMBER'], '605') - self.assertEqual(rows[100]['NUMBER'], '167') - self.assertEqual(rows[1000]['NUMBER'], '322') - self.assertEqual(rows[1]['STREET'], 'BROADWAY') - self.assertEqual(rows[10]['STREET'], 'HILLSBOROUGH ST') - self.assertEqual(rows[100]['STREET'], '8TH ST') - self.assertEqual(rows[1000]['STREET'], 'HANOVER AV') - self.assertEqual(rows[1]['UNIT'], '') - self.assertEqual(rows[10]['UNIT'], '') - self.assertEqual(rows[100]['UNIT'], '') - self.assertEqual(rows[1000]['UNIT'], '') + self.assertEqual(rows[1]['properties']['id'], '') + self.assertEqual(rows[10]['properties']['id'], '') + self.assertEqual(rows[100]['properties']['id'], '') + self.assertEqual(rows[1000]['properties']['id'], '') + self.assertEqual(rows[1]['properties']['number'], '2147') + self.assertEqual(rows[10]['properties']['number'], '605') + self.assertEqual(rows[100]['properties']['number'], '167') + self.assertEqual(rows[1000]['properties']['number'], '322') + self.assertEqual(rows[1]['properties']['street'], 'BROADWAY') + self.assertEqual(rows[10]['properties']['street'], 'HILLSBOROUGH ST') + self.assertEqual(rows[100]['properties']['street'], '8TH ST') + self.assertEqual(rows[1000]['properties']['street'], 'HANOVER AV') + self.assertEqual(rows[1]['properties']['unit'], '') + self.assertEqual(rows[10]['properties']['unit'], '') + self.assertEqual(rows[100]['properties']['unit'], '') + self.assertEqual(rows[1000]['properties']['unit'], '') def test_single_ac_mixedcase(self): ''' Test complete process_one.process on Alameda County sample data. @@ -456,18 +456,18 @@ def test_single_ac_mixedcase(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['ID'], '') - self.assertEqual(rows[10]['ID'], '') - self.assertEqual(rows[100]['ID'], '') - self.assertEqual(rows[1000]['ID'], '') - self.assertEqual(rows[1]['NUMBER'], '2147') - self.assertEqual(rows[10]['NUMBER'], '605') - self.assertEqual(rows[100]['NUMBER'], '167') - self.assertEqual(rows[1000]['NUMBER'], '322') - self.assertEqual(rows[1]['STREET'], 'BROADWAY') - self.assertEqual(rows[10]['STREET'], 'HILLSBOROUGH ST') - self.assertEqual(rows[100]['STREET'], '8TH ST') - self.assertEqual(rows[1000]['STREET'], 'HANOVER AV') + self.assertEqual(rows[1]['properties']['id'], '') + self.assertEqual(rows[10]['properties']['id'], '') + self.assertEqual(rows[100]['properties']['id'], '') + self.assertEqual(rows[1000]['properties']['id'], '') + self.assertEqual(rows[1]['properties']['number'], '2147') + self.assertEqual(rows[10]['properties']['number'], '605') + self.assertEqual(rows[100]['properties']['number'], '167') + self.assertEqual(rows[1000]['properties']['number'], '322') + self.assertEqual(rows[1]['properties']['street'], 'BROADWAY') + self.assertEqual(rows[10]['properties']['street'], 'HILLSBOROUGH ST') + self.assertEqual(rows[100]['properties']['street'], '8TH ST') + self.assertEqual(rows[1000]['properties']['street'], 'HANOVER AV') def test_single_sf(self): ''' Test complete process_one.process on San Francisco sample data. @@ -497,22 +497,22 @@ def test_single_sf(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['ID'], '') - self.assertEqual(rows[10]['ID'], '') - self.assertEqual(rows[100]['ID'], '') - self.assertEqual(rows[1000]['ID'], '') - self.assertEqual(rows[1]['NUMBER'], '27') - self.assertEqual(rows[10]['NUMBER'], '42') - self.assertEqual(rows[100]['NUMBER'], '209') - self.assertEqual(rows[1000]['NUMBER'], '1415') - self.assertEqual(rows[1]['STREET'], 'OCTAVIA ST') - self.assertEqual(rows[10]['STREET'], 'GOLDEN GATE AVE') - self.assertEqual(rows[100]['STREET'], 'OCTAVIA ST') - self.assertEqual(rows[1000]['STREET'], 'FOLSOM ST') - self.assertEqual(rows[1]['UNIT'], '') - self.assertEqual(rows[10]['UNIT'], '') - self.assertEqual(rows[100]['UNIT'], '') - self.assertEqual(rows[1000]['UNIT'], '') + self.assertEqual(rows[1]['properties']['id'], '') + self.assertEqual(rows[10]['properties']['id'], '') + self.assertEqual(rows[100]['properties']['id'], '') + self.assertEqual(rows[1000]['properties']['id'], '') + self.assertEqual(rows[1]['properties']['number'], '27') + self.assertEqual(rows[10]['properties']['number'], '42') + self.assertEqual(rows[100]['properties']['number'], '209') + self.assertEqual(rows[1000]['properties']['number'], '1415') + self.assertEqual(rows[1]['properties']['street'], 'OCTAVIA ST') + self.assertEqual(rows[10]['properties']['street'], 'GOLDEN GATE AVE') + self.assertEqual(rows[100]['properties']['street'], 'OCTAVIA ST') + self.assertEqual(rows[1000]['properties']['street'], 'FOLSOM ST') + self.assertEqual(rows[1]['properties']['unit'], '') + self.assertEqual(rows[10]['properties']['unit'], '') + self.assertEqual(rows[100]['properties']['unit'], '') + self.assertEqual(rows[1000]['properties']['unit'], '') def test_single_car(self): ''' Test complete process_one.process on Carson sample data. @@ -542,14 +542,14 @@ def test_single_car(self): with open(join(dirname(state_path), state['processed'])) as file: rows = list(DictReader(file, dialect='excel')) self.assertEqual(5, len(rows)) - self.assertEqual(rows[0]['NUMBER'], '555') - self.assertEqual(rows[0]['STREET'], 'CARSON ST') - self.assertEqual(rows[0]['UNIT'], '') - self.assertEqual(rows[0]['CITY'], 'CARSON, CA') - self.assertEqual(rows[0]['POSTCODE'], '90745') - self.assertEqual(rows[0]['DISTRICT'], '') - self.assertEqual(rows[0]['REGION'], '') - self.assertEqual(rows[0]['ID'], '') + self.assertEqual(rows[0]['properties']['number'], '555') + self.assertEqual(rows[0]['properties']['street'], 'CARSON ST') + self.assertEqual(rows[0]['properties']['unit'], '') + self.assertEqual(rows[0]['properties']['city'], 'CARSON, CA') + self.assertEqual(rows[0]['properties']['postcode'], '90745') + self.assertEqual(rows[0]['properties']['district'], '') + self.assertEqual(rows[0]['properties']['region'], '') + self.assertEqual(rows[0]['properties']['id'], '') def test_single_car_cached(self): ''' Test complete process_one.process on Carson sample data. @@ -568,7 +568,7 @@ def test_single_car_cached(self): self.assertIsNone(state['preview']) with open(join(dirname(state_path), state['processed'])) as file: - self.assertTrue('555,CARSON ST' in file.read()) + self.assertTrue('555' in file.read()) def test_single_car_old_cached(self): ''' Test complete process_one.process on Carson sample data. @@ -587,7 +587,7 @@ def test_single_car_old_cached(self): self.assertIsNone(state['preview']) with open(join(dirname(state_path), state['processed'])) as file: - self.assertTrue('555,CARSON ST' in file.read()) + self.assertTrue('555' in file.read()) def test_single_tx_runnels(self): ''' Test complete process_one.process on Oakland sample data. @@ -696,18 +696,18 @@ def test_single_berk_apn(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['ID'], '055 188300600') - self.assertEqual(rows[10]['ID'], '055 189504000') - self.assertEqual(rows[100]['ID'], '055 188700100') - self.assertEqual(rows[1]['NUMBER'], '2418') - self.assertEqual(rows[10]['NUMBER'], '2029') - self.assertEqual(rows[100]['NUMBER'], '2298') - self.assertEqual(rows[1]['STREET'], 'DANA ST') - self.assertEqual(rows[10]['STREET'], 'CHANNING WAY') - self.assertEqual(rows[100]['STREET'], 'DURANT AVE') - self.assertEqual(rows[1]['UNIT'], u'') - self.assertEqual(rows[10]['UNIT'], u'') - self.assertEqual(rows[100]['UNIT'], u'') + self.assertEqual(rows[1]['properties']['id'], '055 188300600') + self.assertEqual(rows[10]['properties']['id'], '055 189504000') + self.assertEqual(rows[100]['properties']['id'], '055 188700100') + self.assertEqual(rows[1]['properties']['number'], '2418') + self.assertEqual(rows[10]['properties']['number'], '2029') + self.assertEqual(rows[100]['properties']['number'], '2298') + self.assertEqual(rows[1]['properties']['street'], 'DANA ST') + self.assertEqual(rows[10]['properties']['street'], 'CHANNING WAY') + self.assertEqual(rows[100]['properties']['street'], 'DURANT AVE') + self.assertEqual(rows[1]['properties']['unit'], u'') + self.assertEqual(rows[10]['properties']['unit'], u'') + self.assertEqual(rows[100]['properties']['unit'], u'') def test_single_pl_ds(self): ''' Test complete process_one.process on Polish sample data. @@ -743,15 +743,15 @@ def test_single_pl_l(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['NUMBER'], u'5') - self.assertEqual(rows[10]['NUMBER'], u'8') - self.assertEqual(rows[100]['NUMBER'], u'5a') - self.assertEqual(rows[1]['STREET'], u'Ulica Dolnych Wa\u0142\xf3w Gliwice') - self.assertEqual(rows[10]['STREET'], u'Ulica Dolnych Wa\u0142\xf3w Gliwice') - self.assertEqual(rows[100]['STREET'], u'Plac pl. Inwalid\xf3w Wojennych Gliwice') - self.assertEqual(rows[1]['UNIT'], u'') - self.assertEqual(rows[10]['UNIT'], u'') - self.assertEqual(rows[100]['UNIT'], u'') + self.assertEqual(rows[1]['properties']['number'], u'5') + self.assertEqual(rows[10]['properties']['number'], u'8') + self.assertEqual(rows[100]['properties']['number'], u'5a') + self.assertEqual(rows[1]['properties']['street'], u'Ulica Dolnych Wa\u0142\xf3w Gliwice') + self.assertEqual(rows[10]['properties']['street'], u'Ulica Dolnych Wa\u0142\xf3w Gliwice') + self.assertEqual(rows[100]['properties']['street'], u'Plac pl. Inwalid\xf3w Wojennych Gliwice') + self.assertEqual(rows[1]['properties']['unit'], u'') + self.assertEqual(rows[10]['properties']['unit'], u'') + self.assertEqual(rows[100]['properties']['unit'], u'') def test_single_jp_fukushima2(self): ''' Test complete process_one.process on Japanese sample data. @@ -772,15 +772,15 @@ def test_single_jp_fukushima2(self): rows = list(map(json.loads, list(file))) self.assertEqual(len(rows), 6) - self.assertEqual(rows[0]['NUMBER'], u'24-9') - self.assertEqual(rows[0]['STREET'], u'田沢字姥懐') - self.assertEqual(rows[1]['NUMBER'], u'16-9') - self.assertEqual(rows[1]['STREET'], u'田沢字躑躅ケ森') - self.assertEqual(rows[2]['NUMBER'], u'22-9') - self.assertEqual(rows[2]['STREET'], u'小田字正夫田') - self.assertEqual(rows[0]['GEOM'], 'POINT (140.480007 37.706391)') - self.assertEqual(rows[1]['GEOM'], 'POINT (140.486267 37.707664)') - self.assertEqual(rows[2]['GEOM'], 'POINT (140.41875 37.710239)') + self.assertEqual(rows[0]['properties']['number'], u'24-9') + self.assertEqual(rows[0]['properties']['street'], u'田沢字姥懐') + self.assertEqual(rows[1]['properties']['number'], u'16-9') + self.assertEqual(rows[1]['properties']['street'], u'田沢字躑躅ケ森') + self.assertEqual(rows[2]['properties']['number'], u'22-9') + self.assertEqual(rows[2]['properties']['street'], u'小田字正夫田') + self.assertEqual(rows[0]['geometry']['coordinates'], [140.480007, 37.706391]) + self.assertEqual(rows[1]['geometry']['coordinates'], [140.486267, 37.707664]) + self.assertEqual(rows[2]['geometry']['coordinates'], [140.41875, 37.710239]) def test_single_utah(self): ''' Test complete process_one.process on data that uses file selection with mixed case (issue #104) @@ -912,15 +912,15 @@ def test_single_pa_lancaster(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['UNIT'], u'2') - self.assertEqual(rows[11]['UNIT'], u'11') - self.assertEqual(rows[21]['UNIT'], u'') - self.assertEqual(rows[1]['NUMBER'], u'423') - self.assertEqual(rows[11]['NUMBER'], u'423') - self.assertEqual(rows[21]['NUMBER'], u'7') - self.assertEqual(rows[1]['STREET'], u'W 28TH DIVISION HWY') - self.assertEqual(rows[11]['STREET'], u'W 28TH DIVISION HWY') - self.assertEqual(rows[21]['STREET'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[1]['unit'], u'2') + self.assertEqual(rows[11]['unit'], u'11') + self.assertEqual(rows[21]['unit'], u'') + self.assertEqual(rows[1]['number'], u'423') + self.assertEqual(rows[11]['number'], u'423') + self.assertEqual(rows[21]['number'], u'7') + self.assertEqual(rows[1]['street'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[11]['street'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[21]['street'], u'W 28TH DIVISION HWY') def test_single_ua_kharkiv(self): ''' Test complete process_one.process on data with ESRI multiPolyline geometries. @@ -952,15 +952,15 @@ def test_single_pa_bucks(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['UNIT'], u'') - self.assertEqual(rows[10]['UNIT'], u'') - self.assertEqual(rows[20]['UNIT'], u'') - self.assertEqual(rows[1]['NUMBER'], u'') - self.assertEqual(rows[10]['NUMBER'], u'') - self.assertEqual(rows[20]['NUMBER'], u'429') - self.assertEqual(rows[1]['STREET'], u'STATE RD') - self.assertEqual(rows[10]['STREET'], u'STATE RD') - self.assertEqual(rows[20]['STREET'], u'WALNUT AVE E') + self.assertEqual(rows[1]['properties']['unit'], u'') + self.assertEqual(rows[10]['properties']['unit'], u'') + self.assertEqual(rows[20]['properties']['unit'], u'') + self.assertEqual(rows[1]['properties']['number'], u'') + self.assertEqual(rows[10]['properties']['number'], u'') + self.assertEqual(rows[20]['properties']['number'], u'429') + self.assertEqual(rows[1]['properties']['street'], u'STATE RD') + self.assertEqual(rows[10]['properties']['street'], u'STATE RD') + self.assertEqual(rows[20]['properties']['street'], u'WALNUT AVE E') def test_single_nm_washington(self): ''' Test complete process_one.process on data without ESRI support for resultRecordCount. @@ -979,15 +979,15 @@ def test_single_nm_washington(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['UNIT'], u'') - self.assertEqual(rows[5]['UNIT'], u'') - self.assertEqual(rows[9]['UNIT'], u'') - self.assertEqual(rows[1]['NUMBER'], u'9884') - self.assertEqual(rows[5]['NUMBER'], u'3842') - self.assertEqual(rows[9]['NUMBER'], u'') - self.assertEqual(rows[1]['STREET'], u'5TH STREET LN N') - self.assertEqual(rows[5]['STREET'], u'ABERCROMBIE LN') - self.assertEqual(rows[9]['STREET'], u'') + self.assertEqual(rows[1]['properties']['unit'], u'') + self.assertEqual(rows[5]['properties']['unit'], u'') + self.assertEqual(rows[9]['properties']['unit'], u'') + self.assertEqual(rows[1]['properties']['number'], u'9884') + self.assertEqual(rows[5]['properties']['number'], u'3842') + self.assertEqual(rows[9]['properties']['number'], u'') + self.assertEqual(rows[1]['properties']['street'], u'5TH STREET LN N') + self.assertEqual(rows[5]['properties']['street'], u'ABERCROMBIE LN') + self.assertEqual(rows[9]['properties']['street'], u'') def test_single_tx_waco(self): ''' Test complete process_one.process on data without ESRI support for resultRecordCount. @@ -1025,16 +1025,16 @@ def test_single_tx_waco(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[0]['REGION'], u'TX') - self.assertEqual(rows[0]['ID'], u'') - self.assertEqual(rows[0]['NUMBER'], u'308') - self.assertEqual(rows[0]['HASH'], u'c782a47f7d52d99f') - self.assertEqual(rows[0]['CITY'], u'Mcgregor') - self.assertEqual(rows[0]['GEOM'], u'POINT (-97.3961768 31.4432706)') - self.assertEqual(rows[0]['STREET'], u'PULLEN ST') - self.assertEqual(rows[0]['POSTCODE'], u'76657') - self.assertEqual(rows[0]['UNIT'], u'') - self.assertEqual(rows[0]['DISTRICT'], u'') + self.assertEqual(rows[0]['properties']['region'], u'TX') + self.assertEqual(rows[0]['properties']['id'], u'') + self.assertEqual(rows[0]['properties']['number'], u'308') + self.assertEqual(rows[0]['properties']['hash'], u'c782a47f7d52d99f') + self.assertEqual(rows[0]['properties']['city'], u'Mcgregor') + self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]), + self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST') + self.assertEqual(rows[0]['properties']['postcode'], u'76657') + self.assertEqual(rows[0]['properties']['unit'], u'') + self.assertEqual(rows[0]['properties']['district'], u'') def test_single_wy_park(self): ''' Test complete process_one.process on data without ESRI support for resultRecordCount. @@ -1055,8 +1055,8 @@ def test_single_wy_park(self): rows = list(map(json.loads, list(input))) self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'162') - self.assertEqual(rows[0]['properties']['hash'], u'd666aba81e7f9b01') - self.assertEqual(rows[0]['properties']['CITY'], u'') + self.assertEqual(rows[0]['properties']['hash'], u'730e5ad1893108e4') + self.assertEqual(rows[0]['properties']['city'], u'') self.assertEqual(rows[0]['geometry']['type'], 'Point'); self.assertAlmostEqual(rows[0]['geometry']['coordinates'][0], -108.7563613); self.assertAlmostEqual(rows[0]['geometry']['coordinates'][1], 44.7538737); @@ -1082,15 +1082,15 @@ def test_single_ny_orange(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[0]['ID'], u'') - self.assertEqual(rows[0]['NUMBER'], u'434') - self.assertEqual(rows[0]['HASH'], u'58a4d4fbbf126d86') - self.assertEqual(rows[0]['CITY'], u'MONROE') - self.assertEqual(rows[0]['GEOM'], u'POINT (-74.1926686 41.3187728)') - self.assertEqual(rows[0]['STREET'], u'') - self.assertEqual(rows[0]['POSTCODE'], u'10950') - self.assertEqual(rows[0]['UNIT'], u'') - self.assertEqual(rows[0]['DISTRICT'], u'') + self.assertEqual(rows[0]['properties']['id'], u'') + self.assertEqual(rows[0]['properties']['number'], u'434') + self.assertEqual(rows[0]['properties']['hash'], u'58a4d4fbbf126d86') + self.assertEqual(rows[0]['properties']['city'], u'MONROE') + self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728]) + self.assertEqual(rows[0]['properties']['street'], u'') + self.assertEqual(rows[0]['properties']['postcode'], u'10950') + self.assertEqual(rows[0]['properties']['unit'], u'') + self.assertEqual(rows[0]['properties']['district'], u'') def test_single_de_berlin(self): ''' Test complete process_one.process on data. @@ -1107,12 +1107,12 @@ def test_single_de_berlin(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[0]['NUMBER'], u'72') - self.assertEqual(rows[1]['NUMBER'], u'3') - self.assertEqual(rows[2]['NUMBER'], u'75') - self.assertEqual(rows[0]['STREET'], u'Otto-Braun-Stra\xdfe') - self.assertEqual(rows[1]['STREET'], u'Dorotheenstra\xdfe') - self.assertEqual(rows[2]['STREET'], u'Alte Jakobstra\xdfe') + self.assertEqual(rows[0]['properties']['number'], u'72') + self.assertEqual(rows[1]['properties']['number'], u'3') + self.assertEqual(rows[2]['properties']['number'], u'75') + self.assertEqual(rows[0]['properties']['street'], u'Otto-Braun-Stra\xdfe') + self.assertEqual(rows[1]['properties']['street'], u'Dorotheenstra\xdfe') + self.assertEqual(rows[2]['properties']['street'], u'Alte Jakobstra\xdfe') self.assertIsNone(state['preview']) @@ -1132,18 +1132,18 @@ def test_single_us_or_portland(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 12) - self.assertEqual(rows[2]['NUMBER'], u'1') - self.assertEqual(rows[3]['NUMBER'], u'10') - self.assertEqual(rows[-2]['NUMBER'], u'2211') - self.assertEqual(rows[-1]['NUMBER'], u'2211') - self.assertEqual(rows[2]['STREET'], u'SW RICHARDSON ST') - self.assertEqual(rows[3]['STREET'], u'SW PORTER ST') - self.assertEqual(rows[-2]['STREET'], u'SE OCHOCO ST') - self.assertEqual(rows[-1]['STREET'], u'SE OCHOCO ST') - self.assertTrue(bool(rows[2]['GEOM'])) - self.assertTrue(bool(rows[3]['GEOM'])) - self.assertFalse(bool(rows[-2]['GEOM'])) - self.assertTrue(bool(rows[-1]['GEOM'])) + self.assertEqual(rows[2]['properties']['number'], u'1') + self.assertEqual(rows[3]['properties']['number'], u'10') + self.assertEqual(rows[-2]['properties']['number'], u'2211') + self.assertEqual(rows[-1]['properties']['number'], u'2211') + self.assertEqual(rows[2]['properties']['street'], u'SW RICHARDSON ST') + self.assertEqual(rows[3]['properties']['street'], u'SW PORTER ST') + self.assertEqual(rows[-2]['properties']['street'], u'SE OCHOCO ST') + self.assertEqual(rows[-1]['properties']['street'], u'SE OCHOCO ST') + self.assertTrue(bool(rows[2]['geometry'])) + self.assertTrue(bool(rows[3]['geometry'])) + self.assertFalse(bool(rows[-2]['geometry'])) + self.assertTrue(bool(rows[-1]['geometry'])) def test_single_nl_countrywide(self): ''' Test complete process_one.process on data. @@ -1161,14 +1161,14 @@ def test_single_nl_countrywide(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 8) - self.assertEqual(rows[0]['NUMBER'], u'34x') - self.assertEqual(rows[1]['NUMBER'], u'65-x') - self.assertEqual(rows[2]['NUMBER'], u'147x-x') - self.assertEqual(rows[3]['NUMBER'], u'6') - self.assertEqual(rows[4]['NUMBER'], u'279b') - self.assertEqual(rows[5]['NUMBER'], u'10') - self.assertEqual(rows[6]['NUMBER'], u'601') - self.assertEqual(rows[7]['NUMBER'], u'2') + self.assertEqual(rows[0]['properties']['number'], u'34x') + self.assertEqual(rows[1]['properties']['number'], u'65-x') + self.assertEqual(rows[2]['properties']['number'], u'147x-x') + self.assertEqual(rows[3]['properties']['number'], u'6') + self.assertEqual(rows[4]['properties']['number'], u'279b') + self.assertEqual(rows[5]['properties']['number'], u'10') + self.assertEqual(rows[6]['properties']['number'], u'601') + self.assertEqual(rows[7]['properties']['number'], u'2') def test_single_be_wa_brussels(self): ''' Test complete process_one.process on data. @@ -1185,31 +1185,27 @@ def test_single_be_wa_brussels(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(len(rows), 665) - self.assertEqual(rows[0]['NUMBER'], u'2') - self.assertEqual(rows[0]['STREET'], u'Rue de la Victoire') - self.assertEqual(rows[1]['NUMBER'], u'16') - self.assertEqual(rows[1]['STREET'], u'Rue Fontainas') - self.assertEqual(rows[2]['NUMBER'], u'23C') - self.assertEqual(rows[2]['STREET'], u'Rue Fontainas') - self.assertEqual(rows[3]['NUMBER'], u'2') - self.assertEqual(rows[3]['STREET'], u"Rue de l'Eglise Saint-Gilles") - - x,y = wkt_pt(rows[0]['GEOM']) - self.assertAlmostEqual(4.3458216, x, places=4) - self.assertAlmostEqual(50.8324706, y, places=4) - - x,y = wkt_pt(rows[1]['GEOM']) - self.assertAlmostEqual(4.3412631, x, places=4) - self.assertAlmostEqual(50.8330868, y, places=4) - - x,y = wkt_pt(rows[2]['GEOM']) - self.assertAlmostEqual(4.3410663, x, places=4) - self.assertAlmostEqual(50.8334315, y, places=4) - - x,y = wkt_pt(rows[3]['GEOM']) - self.assertAlmostEqual(4.3421632, x, places=4) - self.assertAlmostEqual(50.8322201, y, places=4) + self.assertEqual(len(rows), 666) + self.assertEqual(rows[0]['properties']['number'], u'2') + self.assertEqual(rows[0]['properties']['street'], u'Rue de la Victoire') + self.assertEqual(rows[1]['properties']['number'], u'16') + self.assertEqual(rows[1]['properties']['street'], u'Rue Fontainas') + self.assertEqual(rows[2]['properties']['number'], u'23C') + self.assertEqual(rows[2]['properties']['street'], u'Rue Fontainas') + self.assertEqual(rows[3]['properties']['number'], u'2') + self.assertEqual(rows[3]['properties']['street'], u"Rue de l'Eglise Saint-Gilles") + + self.assertAlmostEqual(4.3458216, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(50.8324706, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertAlmostEqual(4.3412631, rows[1]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(50.8330868, rows[1]['geometry']['coordinates'][1], places=4) + + self.assertAlmostEqual(4.3410663, rows[2]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(50.8334315, rows[2]['geometry']['coordinates'][1], places=4) + + self.assertAlmostEqual(4.3421632, rows[3]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(50.8322201, rows[3]['geometry']['coordinates'][1], places=4) def test_single_it_52_statewide(self): ''' Test complete process_one.process on data. @@ -1227,15 +1223,15 @@ def test_single_it_52_statewide(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 19) - self.assertEqual(rows[0]['NUMBER'], u'33') - self.assertEqual(rows[0]['STREET'], u'VIA CARLO CARRÀ') - self.assertEqual(rows[1]['NUMBER'], u'23') - self.assertEqual(rows[1]['STREET'], u'VIA CARLO CARRÀ') - self.assertEqual(rows[2]['NUMBER'], u'2') - self.assertEqual(rows[2]['STREET'], u'VIA MARINO MARINI') - self.assertEqual(rows[0]['GEOM'], 'POINT (10.1863188 43.9562646)') - self.assertEqual(rows[1]['GEOM'], 'POINT (10.1856048 43.9558156)') - self.assertEqual(rows[2]['GEOM'], 'POINT (10.1860548 43.9553626)') + self.assertEqual(rows[0]['properties']['number'], u'33') + self.assertEqual(rows[0]['properties']['street'], u'VIA CARLO CARRÀ') + self.assertEqual(rows[1]['properties']['number'], u'23') + self.assertEqual(rows[1]['properties']['street'], u'VIA CARLO CARRÀ') + self.assertEqual(rows[2]['properties']['number'], u'2') + self.assertEqual(rows[2]['properties']['street'], u'VIA MARINO MARINI') + self.assertEqual(rows[0]['geometry']['coordinates'], [10.1863188, 43.9562646]) + self.assertEqual(rows[1]['geometry']['coordinates'], [10.1856048, 43.9558156]) + self.assertEqual(rows[2]['geometry']['coordinates'], [10.1860548, 43.9553626]) def test_single_us_nj_statewide(self): ''' Test complete process_one.process on data. @@ -1253,15 +1249,15 @@ def test_single_us_nj_statewide(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 1045) - self.assertEqual(rows[0]['NUMBER'], u'7') - self.assertEqual(rows[0]['STREET'], u'Sagamore Avenue') - self.assertEqual(rows[1]['NUMBER'], u'29') - self.assertEqual(rows[1]['STREET'], u'Sagamore Avenue') - self.assertEqual(rows[2]['NUMBER'], u'47') - self.assertEqual(rows[2]['STREET'], u'Seneca Place') - self.assertEqual(rows[0]['GEOM'], 'POINT (-74.0012016 40.3201199)') - self.assertEqual(rows[1]['GEOM'], 'POINT (-74.0027904 40.3203365)') - self.assertEqual(rows[2]['GEOM'], 'POINT (-74.0011386 40.3166497)') + self.assertEqual(rows[0]['properties']['number'], u'7') + self.assertEqual(rows[0]['properties']['street'], u'Sagamore Avenue') + self.assertEqual(rows[1]['properties']['number'], u'29') + self.assertEqual(rows[1]['properties']['street'], u'Sagamore Avenue') + self.assertEqual(rows[2]['properties']['number'], u'47') + self.assertEqual(rows[2]['properties']['street'], u'Seneca Place') + self.assertEqual(rows[0]['geometry']['coordinates'], [-74.0012016, 40.3201199]), + self.assertEqual(rows[1]['geometry']['coordinates'], [-74.0027904, 40.3203365]) + self.assertEqual(rows[2]['geometry']['coordinates'], [-74.0011386, 40.3166497]) def test_single_cz_countrywide(self): ''' Test complete process_one.process on data. @@ -1326,18 +1322,18 @@ def test_single_lake_man_gdb(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[1]['NUMBER'], '5121') - self.assertEqual(rows[1]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[2]['NUMBER'], '5133') - self.assertEqual(rows[2]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[3]['NUMBER'], '5126') - self.assertEqual(rows[3]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[4]['NUMBER'], '5120') - self.assertEqual(rows[4]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[5]['NUMBER'], '5115') - self.assertEqual(rows[5]['STREET'], 'OLD MILL RD') + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') def test_single_lake_man_gdb_nested(self): ''' Test complete process_one.process on data. @@ -1357,18 +1353,18 @@ def test_single_lake_man_gdb_nested(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[1]['NUMBER'], '5121') - self.assertEqual(rows[1]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[2]['NUMBER'], '5133') - self.assertEqual(rows[2]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[3]['NUMBER'], '5126') - self.assertEqual(rows[3]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[4]['NUMBER'], '5120') - self.assertEqual(rows[4]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[5]['NUMBER'], '5115') - self.assertEqual(rows[5]['STREET'], 'OLD MILL RD') + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') def test_single_lake_man_gdb_nested_nodir(self): ''' Test complete process_one.process on data. @@ -1388,18 +1384,18 @@ def test_single_lake_man_gdb_nested_nodir(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) self.assertEqual(len(rows), 6) - self.assertEqual(rows[0]['NUMBER'], '5115') - self.assertEqual(rows[0]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[1]['NUMBER'], '5121') - self.assertEqual(rows[1]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[2]['NUMBER'], '5133') - self.assertEqual(rows[2]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[3]['NUMBER'], '5126') - self.assertEqual(rows[3]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[4]['NUMBER'], '5120') - self.assertEqual(rows[4]['STREET'], 'FRUITED PLAINS LN') - self.assertEqual(rows[5]['NUMBER'], '5115') - self.assertEqual(rows[5]['STREET'], 'OLD MILL RD') + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') class TestState (unittest.TestCase): diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 34e2cf08..21ba7898 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -17,7 +17,7 @@ from ..conform import ( GEOM_FIELDNAME, csv_source_to_csv, find_source_path, row_transform_and_convert, - row_fxn_regexp, row_smash_case, row_round_lat_lon, row_merge, + row_fxn_regexp, row_smash_case, row_merge, row_extract_and_reproject, row_convert_to_out, row_fxn_join, row_fxn_format, row_fxn_prefixed_number, row_fxn_postfixed_street, row_fxn_postfixed_unit, @@ -397,7 +397,7 @@ def test_transform_and_convert(self): "id": "" }, 'geometry': { - 'coordinates': (-119.2, 39.3), + 'coordinates': [-119.2, 39.3], 'type': 'Point' } }, r) @@ -428,7 +428,7 @@ def test_transform_and_convert(self): }, 'geometry': { "type": "Point", - "coordinates": (-119.2, 39.3) + "coordinates": [-119.2, 39.3] } }, r) @@ -460,7 +460,7 @@ def test_transform_and_convert(self): 'type': 'Feature', 'geometry': { "type": "Point", - "coordinates": (-119.2, 39.3) + "coordinates": [-119.2, 39.3] }, 'properties': { "street": "MAPLE ST", @@ -502,30 +502,6 @@ def test_row_canonicalize_street_with_no_unit_number(self): self.assertEqual("OAK DR.", r["street"]) self.assertEqual("", r["unit"]) - def test_row_round_lat_lon(self): - r = row_round_lat_lon({}, {"GEOM": "POINT (39.14285717777 -121.20)"}) - self.assertEqual({"GEOM": "POINT (39.1428572 -121.2)"}, r) - for e, a in (( "" , ""), - ( "39.3" , "39.3"), - ( "39.3" , "39.3000000"), - ( "-39.3" , "-39.3000"), - ( "39.1428571", "39.142857143"), - ( "139.1428572", "139.142857153"), - ( "39.1428572", "39.142857153"), - ( "3.1428572", "3.142857153"), - ( "0.1428572", "0.142857153"), - ("-139.1428572","-139.142857153"), - ( "-39.1428572", "-39.142857153"), - ( "-3.1428572", "-3.142857153"), - ( "-0.1428572", "-0.142857153"), - ( "39.1428572", "39.142857153"), - ( "0" , " 0.00"), - ( "0" , "-0.00"), - ( "180" , "180.0"), - ("-180" , "-180")): - r = row_round_lat_lon({}, {"GEOM": "POINT ({} {})".format(a, a)}) - self.assertEqual("POINT ({} {})".format(e, e), r["GEOM"]) - def test_row_extract_and_reproject(self): # CSV lat/lon column names d = SourceConfig(dict({ From 8d5bb3841a78f7f7a3db87b9727d6bd2770f7f28 Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 06:05:41 +0200 Subject: [PATCH 18/26] Update slippymap mock calls --- openaddr/tests/__init__.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 0d8a089f..9d10dadf 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -352,9 +352,8 @@ def test_single_ac_local(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) - self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -396,9 +395,8 @@ def test_single_ac(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) - self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -440,9 +438,8 @@ def test_single_ac_mixedcase(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) - self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -481,9 +478,8 @@ def test_single_sf(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) - self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -526,9 +522,8 @@ def test_single_car(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][1][0].endswith('.pmtiles')) - self.assertTrue(slippymap_gen.mock_calls[1][1][1].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) + self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) with open(state_path) as file: state = dict(zip(*json.load(file))) From 4c436846a9f19a0345ae86e953e477829fcc146f Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 12:00:46 +0200 Subject: [PATCH 19/26] Continue to work on getting tests passing --- CHANGELOG | 6 ++++ openaddr/VERSION | 2 +- openaddr/conform.py | 6 +++- openaddr/tests/__init__.py | 65 +++++++++++++++++++------------------- 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 01855f42..e9184bba 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +2023-09-17 v9.0.0 +- Remove GeoJSONLD Render Functions +- Remove MBTiles Render Functions +- Rewrite confrom to natively output GeoJSON+LD +- Rewrite Preview module to expect GeoJSON+LD + 2023-09-06 v8.3.0 - Update to next major alpine version - Remove mock library in favour of unittest.mock diff --git a/openaddr/VERSION b/openaddr/VERSION index 1365b923..f7ee0669 100644 --- a/openaddr/VERSION +++ b/openaddr/VERSION @@ -1 +1 @@ -8.2.3 +9.0.0 diff --git a/openaddr/conform.py b/openaddr/conform.py index 0784c26e..23db91ed 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1082,10 +1082,14 @@ def row_calculate_hash(cache_fingerprint, row): def row_convert_to_out(source_config, row): "Convert a row from the source schema to OpenAddresses output schema" + geom = row.get(GEOM_FIELDNAME.lower(), None) + if geom == "POINT EMPTY": + geom = None + output = { "type": "Feature", "properties": {}, - "geometry": row.get(GEOM_FIELDNAME.lower(), None), + "geometry": geom } if output["geometry"] is not None: diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 9d10dadf..6083ec1c 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -25,7 +25,6 @@ import logging from os import close, environ, mkdir, remove from io import BytesIO -from csv import DictReader from itertools import cycle from zipfile import ZipFile from datetime import datetime, timedelta @@ -352,8 +351,8 @@ def test_single_ac_local(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][1].endswith('.geojson')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -395,8 +394,8 @@ def test_single_ac(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][1].endswith('.geojson')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -438,8 +437,8 @@ def test_single_ac_mixedcase(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][1].endswith('.geojson')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -478,8 +477,8 @@ def test_single_sf(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][1].endswith('.geojson')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -522,8 +521,8 @@ def test_single_car(self): slippymap_gen.side_effect = touch_first_arg_file state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, mapbox_key='mapbox-XXXX') - self.assertTrue(slippymap_gen.mock_calls[0][0].endswith('.geojson')) - self.assertTrue(slippymap_gen.mock_calls[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][0].endswith('.pmtiles')) + self.assertTrue(list(slippymap_gen.mock_calls[0])[1][1].endswith('.geojson')) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -535,7 +534,7 @@ def test_single_car(self): self.assertIsNotNone(state['pmtiles']) with open(join(dirname(state_path), state['processed'])) as file: - rows = list(DictReader(file, dialect='excel')) + rows = list(map(json.loads, list(file))) self.assertEqual(5, len(rows)) self.assertEqual(rows[0]['properties']['number'], '555') self.assertEqual(rows[0]['properties']['street'], 'CARSON ST') @@ -552,7 +551,7 @@ def test_single_car_cached(self): source = join(self.src_dir, 'us-ca-carson-cached.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -571,7 +570,7 @@ def test_single_car_old_cached(self): source = join(self.src_dir, 'us-ca-carson-old-cached.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -590,7 +589,7 @@ def test_single_tx_runnels(self): source = join(self.src_dir, 'us/tx/runnels.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -608,7 +607,7 @@ def test_single_oak(self): source = join(self.src_dir, 'us-ca-oakland.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -626,7 +625,7 @@ def test_single_oak_skip(self): source = join(self.src_dir, 'us-ca-oakland-skip.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -644,7 +643,7 @@ def test_single_berk(self): source = join(self.src_dir, 'us-ca-berkeley.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -662,7 +661,7 @@ def test_single_berk_404(self): source = join(self.src_dir, 'us-ca-berkeley-404.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -678,7 +677,7 @@ def test_single_berk_apn(self): source = join(self.src_dir, 'us-ca-berkeley-apn.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -710,7 +709,7 @@ def test_single_pl_ds(self): source = join(self.src_dir, 'pl-dolnoslaskie.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -725,7 +724,7 @@ def test_single_pl_l(self): source = join(self.src_dir, 'pl-lodzkie.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -907,15 +906,15 @@ def test_single_pa_lancaster(self): with open(output_path, encoding='utf8') as input: rows = list(map(json.loads, list(input))) - self.assertEqual(rows[1]['unit'], u'2') - self.assertEqual(rows[11]['unit'], u'11') - self.assertEqual(rows[21]['unit'], u'') - self.assertEqual(rows[1]['number'], u'423') - self.assertEqual(rows[11]['number'], u'423') - self.assertEqual(rows[21]['number'], u'7') - self.assertEqual(rows[1]['street'], u'W 28TH DIVISION HWY') - self.assertEqual(rows[11]['street'], u'W 28TH DIVISION HWY') - self.assertEqual(rows[21]['street'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[1]['properties']['unit'], u'2') + self.assertEqual(rows[11]['properties']['unit'], u'11') + self.assertEqual(rows[21]['properties']['unit'], u'') + self.assertEqual(rows[1]['properties']['number'], u'423') + self.assertEqual(rows[11]['properties']['number'], u'423') + self.assertEqual(rows[21]['properties']['number'], u'7') + self.assertEqual(rows[1]['properties']['street'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[11]['properties']['street'], u'W 28TH DIVISION HWY') + self.assertEqual(rows[21]['properties']['street'], u'W 28TH DIVISION HWY') def test_single_ua_kharkiv(self): ''' Test complete process_one.process on data with ESRI multiPolyline geometries. @@ -1006,7 +1005,7 @@ def test_single_tx_waco(self): with HTTMock(self.response_content): ofs = csv.field_size_limit() csv.field_size_limit(sys.maxsize) - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) csv.field_size_limit(ofs) with open(state_path) as file: @@ -1079,7 +1078,7 @@ def test_single_ny_orange(self): rows = list(map(json.loads, list(input))) self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'434') - self.assertEqual(rows[0]['properties']['hash'], u'58a4d4fbbf126d86') + self.assertEqual(rows[0]['properties']['hash'], u'8cb84b9e793a4986') self.assertEqual(rows[0]['properties']['city'], u'MONROE') self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728]) self.assertEqual(rows[0]['properties']['street'], u'') From 3fd47c68e294daf74d431501a1a232ac46b23443 Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 12:03:13 +0200 Subject: [PATCH 20/26] Continue to improve tests --- openaddr/tests/conform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 21ba7898..fa0dbb7a 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -393,7 +393,7 @@ def test_transform_and_convert(self): "region": "", "district": "", "postcode": "", - 'hash': '09212154c33070c2', + 'hash': 'b3af08e447c7ed16', "id": "" }, 'geometry': { @@ -424,7 +424,7 @@ def test_transform_and_convert(self): "district": "", "postcode": "", "id": "", - 'hash': '3372445616223f37' + 'hash': 'd4681f7e1d34e6ed' }, 'geometry': { "type": "Point", @@ -471,7 +471,7 @@ def test_transform_and_convert(self): "district": "", "postcode": "", "id": "", - 'hash': '7bfd895bef404ea4' + 'hash': '591d7970b5753b0d' } }, r) From 42125f4bbc19988a1f290bc46981741f4907ef33 Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 12:18:16 +0200 Subject: [PATCH 21/26] Correct more process() args --- openaddr/conform.py | 3 ++- openaddr/tests/__init__.py | 52 +++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 23db91ed..5b240079 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1210,7 +1210,8 @@ def check_source_tests(source_config): for (index, test) in enumerate(acceptance_tests): input = row_smash_case(source_config.data_source, test['inputs']) output = row_smash_case(source_config.data_source, row_transform_and_convert(source_config, input)) - actual = {k: v for (k, v) in output.items() if k in test['expected']} + + actual = {k: v for (k, v) in output['properties'].items() if k in test['expected']} expected = row_smash_case(source_config.data_source, test['expected']) if actual != expected: diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 6083ec1c..e6c30ef2 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -753,7 +753,7 @@ def test_single_jp_fukushima2(self): source = join(self.src_dir, 'jp-fukushima2.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -782,7 +782,7 @@ def test_single_utah(self): source = join(self.src_dir, 'us-ut.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -795,7 +795,7 @@ def test_single_iceland(self): source = join(self.src_dir, 'iceland.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -818,7 +818,7 @@ def test_single_fr_paris(self): source = join(self.src_dir, 'fr-paris.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -840,7 +840,7 @@ def test_single_fr_lareunion(self): raise Exception('Could not find a usable fr/la-réunion.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -853,7 +853,7 @@ def test_single_va_statewide(self): source = join(self.src_dir, 'us/va/statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -867,7 +867,7 @@ def test_single_oh_trumbull(self): source = join(self.src_dir, 'us/oh/trumbull.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -881,7 +881,7 @@ def test_single_ks_brown(self): source = join(self.src_dir, 'us/ks/brown_county.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -895,7 +895,7 @@ def test_single_pa_lancaster(self): source = join(self.src_dir, 'us/pa/lancaster.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -922,7 +922,7 @@ def test_single_ua_kharkiv(self): source = join(self.src_dir, 'ua-63-city_of_kharkiv.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -935,7 +935,7 @@ def test_single_pa_bucks(self): source = join(self.src_dir, 'us/pa/bucks.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -962,7 +962,7 @@ def test_single_nm_washington(self): source = join(self.src_dir, 'us/nm/washington.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -991,7 +991,7 @@ def test_single_tx_waco(self): with HTTMock(self.response_content): ofs = csv.field_size_limit() csv.field_size_limit(1) - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) csv.field_size_limit(ofs) with open(state_path) as file: @@ -1065,7 +1065,7 @@ def test_single_ny_orange(self): source = join(self.src_dir, 'us-ny-orange.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1092,7 +1092,7 @@ def test_single_de_berlin(self): source = join(self.src_dir, 'de/berlin.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1116,7 +1116,7 @@ def test_single_us_or_portland(self): source = join(self.src_dir, 'us/or/portland.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1145,7 +1145,7 @@ def test_single_nl_countrywide(self): source = join(self.src_dir, 'nl/countrywide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1170,7 +1170,7 @@ def test_single_be_wa_brussels(self): source = join(self.src_dir, 'be/wa/brussels-fr.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1207,7 +1207,7 @@ def test_single_it_52_statewide(self): source = join(self.src_dir, 'it-52-statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1233,7 +1233,7 @@ def test_single_us_nj_statewide(self): source = join(self.src_dir, 'us/nj/statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1259,7 +1259,7 @@ def test_single_cz_countrywide(self): source = join(self.src_dir, 'cz-countrywide-bad-tests.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1274,7 +1274,7 @@ def test_single_or_curry(self): source = join(self.src_dir, 'us-or-curry.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1289,7 +1289,7 @@ def test_single_mi_grand_traverse(self): source = join(self.src_dir, 'us-mi-grand_traverse.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1304,7 +1304,7 @@ def test_single_lake_man_gdb(self): source = join(self.src_dir, 'lake-man-gdb.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1335,7 +1335,7 @@ def test_single_lake_man_gdb_nested(self): source = join(self.src_dir, 'lake-man-gdb-nested.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1366,7 +1366,7 @@ def test_single_lake_man_gdb_nested_nodir(self): source = join(self.src_dir, 'lake-man-gdb-nested-nodir.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) From a6ab61c0ea17ea438aa2180d0113a2b6f0ba371c Mon Sep 17 00:00:00 2001 From: ingalls Date: Sun, 17 Sep 2023 12:23:35 +0200 Subject: [PATCH 22/26] Remove Debug --- openaddr/tests/__init__.py | 1 - openaddr/tests/preview.py | 1 - 2 files changed, 2 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index e6c30ef2..2455e18c 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -1295,7 +1295,6 @@ def test_single_mi_grand_traverse(self): state = dict(zip(*json.load(file))) self.assertIsNone(state["processed"]) - print(state) self.assertEqual(state["source problem"], "Found no features in source data") def test_single_lake_man_gdb(self): diff --git a/openaddr/tests/preview.py b/openaddr/tests/preview.py index db9bc525..849a776b 100644 --- a/openaddr/tests/preview.py +++ b/openaddr/tests/preview.py @@ -60,7 +60,6 @@ def response_content(url, request): with HTTMock(response_content): preview.render(join(dirname(__file__), 'outputs', 'denver-metro-preview.geojson'), png_filename, 668, 1, 'mapbox-XXXX') - print(png_filename) info = str(subprocess.check_output(('file', png_filename))) self.assertTrue('PNG image data' in info) From facf91a7563789ed27fe3217cd98ca21db6fbc3e Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 09:19:02 +0100 Subject: [PATCH 23/26] Remove field limit --- openaddr/tests/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 2455e18c..7a61260c 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -989,10 +989,7 @@ def test_single_tx_waco(self): source = join(self.src_dir, 'us/tx/city_of_waco.json') with HTTMock(self.response_content): - ofs = csv.field_size_limit() - csv.field_size_limit(1) state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) - csv.field_size_limit(ofs) with open(state_path) as file: state = dict(zip(*json.load(file))) From d11c6b069ff1bab6d295bd8023992723aa452a28 Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 09:26:44 +0100 Subject: [PATCH 24/26] Fix Empty Geom Error --- openaddr/conform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 5b240079..b2b3240c 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1083,7 +1083,7 @@ def row_convert_to_out(source_config, row): "Convert a row from the source schema to OpenAddresses output schema" geom = row.get(GEOM_FIELDNAME.lower(), None) - if geom == "POINT EMPTY": + if geom == "POINT EMPTY" or geom == '': geom = None output = { From 8775cc14c1d2c5ecff42c1ac5c7994d15a27cb0d Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 11:43:16 +0100 Subject: [PATCH 25/26] Finish getting tests passing --- openaddr/conform.py | 2 +- openaddr/tests/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index b2b3240c..0b084140 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -388,7 +388,7 @@ def convert(self, source_config, source_paths, workdir): rc = conform_cli(source_config, source_path, dest_path) if rc == 0: with open(dest_path) as file: - addr_count = sum(1 for line in file) - 1 + addr_count = sum(1 for line in file) # Success! Return the path of the output CSV return dest_path, addr_count diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 7a61260c..be71dbca 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -994,8 +994,8 @@ def test_single_tx_waco(self): with open(state_path) as file: state = dict(zip(*json.load(file))) - self.assertEqual(state["source problem"], "Could not conform source data") - self.assertIsNone(state["processed"]) + self.assertIsNone(state["source problem"]) + self.assertEqual(state["processed"], 'out.geojson') source = join(self.src_dir, 'us/tx/city_of_waco.json') @@ -1019,7 +1019,7 @@ def test_single_tx_waco(self): self.assertEqual(rows[0]['properties']['region'], u'TX') self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'308') - self.assertEqual(rows[0]['properties']['hash'], u'c782a47f7d52d99f') + self.assertEqual(rows[0]['properties']['hash'], u'431f816eebac0000') self.assertEqual(rows[0]['properties']['city'], u'Mcgregor') self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]), self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST') From 20325d6817a2a37f86bceade124c086e02bcc92c Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 10:40:34 -0500 Subject: [PATCH 26/26] Fix CHANGELOG --- CHANGELOG | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d606f7b1..3f51f62b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,19 +1,9 @@ -<<<<<<< HEAD 2023-09-17 v9.0.0 - Remove GeoJSONLD Render Functions - Remove MBTiles Render Functions - Rewrite confrom to natively output GeoJSON+LD - Rewrite Preview module to expect GeoJSON+LD -2023-09-06 v8.3.0 -- Update to next major alpine version -- Remove mock library in favour of unittest.mock -- Update all python deps to latest versions - -2023-09-05 v8.2.4 -- Update TippeCanoe to latest version -- Make GeoJSON+LD output a default -======= 2023-09-09 v8.3.1 - Use the file extension from the content-type header when it disagrees with content-disposition in https://github.com/openaddresses/batch-machine/pull/54 @@ -21,8 +11,10 @@ - Update Mapbox Vector Tile dependency in https://github.com/openaddresses/batch-machine/pull/52 - Make GeoJSON+LD output a default in https://github.com/openaddresses/batch-machine/pull/51 - Update TippeCanoe to latest version in https://github.com/openaddresses/batch-machine/pull/50 ->>>>>>> 4f57e3e6be53695ef0a4a8db60139154de09b6cf +2023-09-05 v8.2.4 +- Update TippeCanoe to latest version +- Make GeoJSON+LD output a default 2023-09-02 v8.2.3 - Fix properties in geojson-ld features in https://github.com/openaddresses/batch-machine/pull/49 - Remove extra whitespace in geojson-ld output in https://github.com/openaddresses/batch-machine/pull/48