diff --git a/.gitignore b/.gitignore index b0a89a5..f540a38 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,18 @@ -*.swp -*.pyc +*-secret.ini +*.db +*.db-journal +*.egg *.egg-info -*.lprof *.log +*.lprof +*.py[co] +*.sw[op] *.tmp +*~ +.coverage .ipynb_checkpoints -ConvNet__* +build +data_* +dist +launch*.ini +sdist diff --git a/CHANGES.md b/CHANGES.md index e69de29..55cce72 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -0,0 +1,2 @@ +0.0 +--- diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..59bced1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include *.ini *.cfg *.md +recursive-include count_buildings *.mako *.css *.js *.ico *.png diff --git a/README.md b/README.md index b311a72..3436020 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,36 @@ Count buildings in satellite image ================================== -Estimate the number of buildings in a satellite image. +Reveal population density. + + +Run scripts +----------- get_tiles_from_image.py get_points_from_tiles.py get_examples_from_points.py get_dataset_from_examples.py + get_batches_from_datasets.py get_marker_from_dataset.py + get_array_shape_from_batches.py + get_index_from_batches.py + get_arrays_from_image.py + get_batches_from_arrays.py get_predictions_from_arrays.py get_counts_from_predictions.py + + +Launch interface +---------------- + + ENV=~/.virtualenvs/crosscompute + virtualenv $ENV + source $ENV/bin/activate + pip install -U crosscompute + + cd ~/Projects/count-buildings + python setup.py develop + pserve development.ini diff --git a/TODO.goals b/TODO.goals index af2c7a0..615b3ac 100644 --- a/TODO.goals +++ b/TODO.goals @@ -1,8 +1,6 @@ -# US/Pacific 7/23/2014 +# US/Pacific 7/24/2014 = Deploy web app - = Write dummy web app for uploading a request and downloading the response - Write dummy processor that takes a request and returns a response - Replace dummy proccesor with real processor + = Replace dummy proccesor with real processor Feature Modi Research Group logo and link on tool webpage Improve development classifier Consider optimizing get_counts by searching maximums locally @@ -17,4 +15,4 @@ Make scripts easier to use Send access credentials to all tools, valid with no membership fee for one year Add payment Mask paths in run.pkl -Evaluate classifiers on cost-effective lower resolution satellite images \ No newline at end of file +Evaluate classifiers on cost-effective lower resolution satellite images diff --git a/TODO.log b/TODO.log index d8cb001..5686f3c 100644 --- a/TODO.log +++ b/TODO.log @@ -1,4 +1,8 @@ -# UTC 07/23/2014 +# UTC 07/24/2014 ++ Write dummy web app for uploading a request and downloading the response [07/24/2014] ++ Update description [07/24/2014] ++ Let user paste image URL [07/24/2014] ++ Trigger rqworker that downloads and gets file size [07/24/2014] + Investigate why the positive count is 7027 instead of 7028 [07/23/2014] + Prepare small test geotiff [07/23/2014] + Finish crosscompute-scaffolds [07/23/2014] diff --git a/count_buildings/__init__.py b/count_buildings/__init__.py index e69de29..72cc8e6 100644 --- a/count_buildings/__init__.py +++ b/count_buildings/__init__.py @@ -0,0 +1,24 @@ +from pyramid.config import Configurator + +from . import view + + +def describe(): + return dict( + route=view.ROUTE_NAME, + title=view.PACKAGE_TITLE) + + +def includeme(config): + config.scan() + config.include(view) + config.add_static_view( + view.ROUTE_NAME + '/_', + view.PACKAGE_NAME + ':assets', cache_max_age=3600) + + +def main(global_config, **settings): + config = Configurator(settings=settings) + config.include('crosscompute') + includeme(config) + return config.make_wsgi_app() diff --git a/count_buildings/assets/main.js b/count_buildings/assets/main.js new file mode 100644 index 0000000..90079ba --- /dev/null +++ b/count_buildings/assets/main.js @@ -0,0 +1,41 @@ +require.config({ + paths: { + common: static_url + 'common' + } +}); +require(['common'], function(common) { + require(['cc'], function(cc) { + + $('#classifier_name_question').show().find('select').click(function() { + $('#image_url_question, #run').reveal(); + }); + + $('#run').click(function() { + + if (!window.user_id) { + window.location = login_url; + return false; + } + + $('#run').prop('disabled', true); + + var $target = $('#target_table').html('Counting buildings...').reveal(); + $('html').animate({scrollTop: $target.offset().top}, 500); + + cc.post(run_url, { + classifier_name: $('#classifier_name').val(), + image_url: $('#image_url').val() + }, { + end: true + }, function(result) { + var summary = result.summary; + var columns = summary.columns, rows = summary.rows; + $target + .data('result_id', result.id) + .fill_table(columns, rows); + $('#run').prop('disabled', false); + }); + }); + + }); +}); diff --git a/count_buildings/run.py b/count_buildings/run.py new file mode 100644 index 0000000..e50f57a --- /dev/null +++ b/count_buildings/run.py @@ -0,0 +1,50 @@ +import sys +from os import makedirs +from os.path import basename, join +from tempfile import mkstemp +from urllib2 import urlopen + +from count_buildings.scripts.get_tiles_from_image import save_image_properties +from crosscompute.libraries import script +from crosscompute.libraries import queue + + +CLASSIFIER_FOLDER = '/tmp/classifiers' +DOWNLOAD_FOLDER = '/tmp/downloads' +try: + makedirs(DOWNLOAD_FOLDER) +except OSError: + pass + + +def start(argv=sys.argv): + with script.Starter(run, argv) as starter: + starter.add_argument( + '--classifier_name', metavar='NAME', required=True) + starter.add_argument( + '--image_url', metavar='NAME', required=True) + + +def schedule(target_result_id, classifier_name, image_url): + target_folder = script.get_target_folder(target_result_id) + summary = run(target_folder, classifier_name, image_url) + queue.save(target_result_id, summary) + + +def run(target_folder, classifier_name, image_url): + # classifier_path = join(CLASSIFIER_FOLDER, classifier_name) + image_path = download(image_url) + image_name = basename(image_url) + image_properties = save_image_properties(image_path) + return dict( + columns=['Dimensions', 'Bands'], + rows=[[ + image_name, + '%ix%im' % tuple(image_properties['image_dimensions']), + image_properties['image_band_count']]]) + + +def download(url): + path = mkstemp(dir=DOWNLOAD_FOLDER)[1] + open(path, 'wb').write(urlopen(url).read()) + return path diff --git a/count_buildings/scripts/get_arrays_from_image.py b/count_buildings/scripts/get_arrays_from_image.py index f7c6338..7e5508d 100644 --- a/count_buildings/scripts/get_arrays_from_image.py +++ b/count_buildings/scripts/get_arrays_from_image.py @@ -5,7 +5,7 @@ from crosscompute.libraries import script from .get_examples_from_points import get_pixel_centers -from .get_tiles_from_image import save_image_dimensions +from .get_tiles_from_image import save_image_properties from ..libraries.kdtree import KDTree from ..libraries.satellite_image import ImageScope from ..libraries.satellite_image import SatelliteImage @@ -43,7 +43,7 @@ def run( tile_dimensions, overlap_dimensions, included_pixel_bounds): if tile_dimensions is None and included_pixel_bounds is None: - return save_image_dimensions(image_path) + return save_image_properties(image_path) elif tile_dimensions is None: return save_pixel_bounds( target_folder, image_path, included_pixel_bounds) diff --git a/count_buildings/scripts/get_marker_from_dataset.py b/count_buildings/scripts/get_marker_from_dataset.py index b641016..e3c5456 100644 --- a/count_buildings/scripts/get_marker_from_dataset.py +++ b/count_buildings/scripts/get_marker_from_dataset.py @@ -2,8 +2,8 @@ import sys from crosscompute.libraries import script +from .get_dataset_from_examples import DATASET_NAME from ..libraries.markers import initialize_marker -from ..libraries.dataset import DATASET_NAME def start(argv=sys.argv): diff --git a/count_buildings/scripts/get_pixel_bounds_from_image.py b/count_buildings/scripts/get_pixel_bounds_from_image.py deleted file mode 100644 index fc78542..0000000 --- a/count_buildings/scripts/get_pixel_bounds_from_image.py +++ /dev/null @@ -1,10 +0,0 @@ - -def start(argv=sys.argv): - with script.Starter(run, argv) as starter: - starter.add_argument( - '--image_path', metavar='PATH', required=True, - help='satellite image') - starter.add_argument( - '--tile_dimensions', metavar='WIDTH,HEIGHT', - type=script.parse_dimensions, - help='dimensions of extracted tile in geographic units') diff --git a/count_buildings/scripts/get_tiles_from_image.py b/count_buildings/scripts/get_tiles_from_image.py index d8e7288..9554a13 100644 --- a/count_buildings/scripts/get_tiles_from_image.py +++ b/count_buildings/scripts/get_tiles_from_image.py @@ -36,7 +36,7 @@ def run( tile_dimensions, overlap_dimensions, tile_indices, included_pixel_bounds, list_pixel_bounds): if tile_dimensions is None and included_pixel_bounds is None: - return save_image_dimensions(image_path) + return save_image_properties(image_path) elif tile_dimensions is None: return save_pixel_bounds( target_folder, image_path, included_pixel_bounds) @@ -50,7 +50,7 @@ def run( included_pixel_bounds, list_pixel_bounds) -def save_image_dimensions(image_path): +def save_image_properties(image_path): image = satellite_image.SatelliteImage(image_path) return dict( image_dimensions=image.to_dimensions(image.pixel_dimensions), diff --git a/count_buildings/show.mako b/count_buildings/show.mako new file mode 100644 index 0000000..392cf50 --- /dev/null +++ b/count_buildings/show.mako @@ -0,0 +1,42 @@ +<%! script_url = '/count-buildings/_/main' %> +<%inherit file='crosscompute:templates/base.mako'/> + +

${title}

+

Reveal regional population density.

+ +
+ +
+ + +
+ +
+ + +
+ + + +
+ +
+
+ +
+ +
+Thanks to +Shaky Sherpa, +Vijay Modi, +Alex Krizhevsky, +Daniel Nouri and +Ilya Sutskever. +
+ + diff --git a/count_buildings/test.py b/count_buildings/test.py new file mode 100644 index 0000000..c16daba --- /dev/null +++ b/count_buildings/test.py @@ -0,0 +1,117 @@ +import geometryIO +from mock import patch +from pyramid.testing import DummyRequest +from unittest import TestCase + +from . import view +from crosscompute import models as m +from crosscompute.libraries import validation as v + + +process = view.count_buildings_ +result = m.Result( + id=1, + name=u'whee', + summary={'columns': ['a', 'b']}) + + +class ViewTest(TestCase): + + def test_no_parameters_given(self): + request = DummyRequest() + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.REQUIRED in errors['source_table']) + self.assert_(v.REQUIRED in errors['column_x_index']) + self.assert_(v.REQUIRED in errors['column_y_index']) + self.assert_(v.REQUIRED in errors['source_proj4']) + self.assert_(v.REQUIRED in errors['target_proj4']) + + @patch('crosscompute.models.Result.get') + def test_source_table_not_found(self, get): + get.side_effect = v.ResultIndexError + request = DummyRequest({'source_table': 'xxx'}) + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.INVALID in errors['source_table']) + + @patch('crosscompute.models.Result.get') + def test_source_table_denied(self, get): + get.side_effect = v.ResultAccessError + request = DummyRequest({'source_table': '10'}) + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.DENIED in errors['source_table']) + + @patch('crosscompute.models.Result.get') + def test_column_index_invalid_number(self, get): + get.return_value = result + request = DummyRequest({ + 'source_table': '1', + 'column_x_index': 'xxx', + 'column_y_index': 'yyy', + 'source_proj4': geometryIO.proj4LL, + 'target_proj4': geometryIO.proj4SM, + }) + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.INVALID in errors['column_x_index']) + self.assert_(v.INVALID in errors['column_y_index']) + + @patch('crosscompute.models.Result.get') + def test_column_index_invalid_index(self, get): + get.return_value = result + request = DummyRequest({ + 'source_table': '1', + 'column_x_index': '100', + 'column_y_index': '101', + 'source_proj4': geometryIO.proj4LL, + 'target_proj4': geometryIO.proj4SM, + }) + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.INVALID in errors['column_x_index']) + self.assert_(v.INVALID in errors['column_y_index']) + + @patch('crosscompute.models.Result.get') + def test_proj4_invalid(self, get): + get.return_value = result + request = DummyRequest({ + 'source_table': '1', + 'column_x_index': '1', + 'column_y_index': '2', + 'source_proj4': 'xxx', + 'target_proj4': 'yyy', + }) + data = process(request) + errors = data['errors'] + self.assertEqual(400, request.response.status_code) + self.assert_(v.INVALID in errors['source_proj4']) + self.assert_(v.INVALID in errors['target_proj4']) + + @patch('crosscompute.libraries.queue.schedule') + @patch('crosscompute.models.Result.get') + def test_success(self, get, schedule): + get.return_value = result + request = DummyRequest({ + 'source_table': '1', + 'column_x_index': '0', + 'column_y_index': '1', + 'source_proj4': unicode(geometryIO.proj4LL), + 'target_proj4': unicode(geometryIO.proj4SM), + }) + process(request) + self.assertEqual(200, request.response.status_code) + columns = result.summary['columns'] + params = request.params + schedule.assert_called_once_with( + request, view.schedule.start, result.name, result.id, + columns[int(params['column_x_index'])], + columns[int(params['column_y_index'])], + params['source_proj4'], + params['target_proj4']) diff --git a/count_buildings/view.py b/count_buildings/view.py new file mode 100644 index 0000000..1db2a6e --- /dev/null +++ b/count_buildings/view.py @@ -0,0 +1,49 @@ +from os.path import basename, dirname +from pyramid.view import view_config +from voluptuous import Schema, MultipleInvalid + +from . import run +from crosscompute.libraries import queue +from crosscompute.libraries import validation as v + + +PACKAGE_TITLE = 'Count buildings in satellite images' +PACKAGE_NAME = basename(dirname(__file__)) +ROUTE_NAME = PACKAGE_NAME.replace('_', '-') +DEFAULT_STRFTIME = '%Y-%m-%d %H:%M' + + +def includeme(config): + config.add_route( + ROUTE_NAME, '/%s' % ROUTE_NAME) + + +@view_config( + renderer=PACKAGE_NAME + ':show.mako', + request_method='GET', + route_name=ROUTE_NAME) +def count_buildings(request): + return dict( + title=PACKAGE_TITLE) + + +@view_config( + permission='run', + renderer='json', + request_method='POST', + route_name=ROUTE_NAME) +def count_buildings_(request): + try: + params = Schema({ + v.Required('classifier_name'): unicode, + v.Required('image_url'): unicode, + }, extra=True)(dict(request.params)) + except MultipleInvalid as exception: + return {'errors': v.render_errors(request, exception.errors)} + classifier_name = params['classifier_name'] + image_url = params['image_url'] + image_name = basename(image_url) + target_name = '%s-%s' % (classifier_name, image_name) + return queue.schedule( + request, 'gpu', run.schedule, target_name, + classifier_name, image_url) diff --git a/development.ini b/development.ini new file mode 100644 index 0000000..07abfed --- /dev/null +++ b/development.ini @@ -0,0 +1,47 @@ +[app:main] +use = egg:count-buildings + +data.folder = %(here)s/data_development +sqlalchemy.url = sqlite:///%(data.folder)s/db.sqlite + +pyramid.reload_templates = true +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en +pyramid.includes = + pyramid_debugtoolbar + +[server:main] +use = egg:waitress#main +host = 0.0.0.0 +port = 6543 + +[pshell] +m = crosscompute.models +db = crosscompute.models.db + +[loggers] +keys = root, count_buildings, sqlalchemy +[handlers] +keys = console +[formatters] +keys = generic +[logger_root] +level = INFO +handlers = console +[logger_count_buildings] +level = DEBUG +handlers = +qualname = count_buildings +[logger_sqlalchemy] +level = INFO +handlers = +qualname = sqlalchemy.engine +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/production.ini b/production.ini new file mode 100644 index 0000000..e571be2 --- /dev/null +++ b/production.ini @@ -0,0 +1,45 @@ +[app:main] +use = egg:count-buildings + +data.folder = %(here)s/data_production +sqlalchemy.url = sqlite:///%(data.folder)s/db.sqlite + +pyramid.reload_templates = false +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en + +[server:main] +use = egg:waitress#main +host = 0.0.0.0 +port = 6543 + +[pshell] +m = crosscompute.models +db = crosscompute.models.db + +[loggers] +keys = root, count_buildings, sqlalchemy +[handlers] +keys = console +[formatters] +keys = generic +[logger_root] +level = WARN +handlers = console +[logger_count_buildings] +level = WARN +handlers = +qualname = count_buildings +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..58bf978 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,27 @@ +[nosetests] +match = ^test +nocapture = 1 +cover-package = count_buildings +with-coverage = 1 +cover-erase = 1 + +[compile_catalog] +directory = count_buildings/locale +domain = count_buildings +statistics = true + +[extract_messages] +add_comments = TRANSLATORS: +output_file = count_buildings/locale/count_buildings.pot +width = 80 + +[init_catalog] +domain = count_buildings +input_file = count_buildings/locale/count_buildings.pot +output_dir = count_buildings/locale + +[update_catalog] +domain = count_buildings +input_file = count_buildings/locale/count_buildings.pot +output_dir = count_buildings/locale +previous = true diff --git a/setup.py b/setup.py index c9b3aed..8d4f933 100644 --- a/setup.py +++ b/setup.py @@ -11,35 +11,42 @@ 'crosscompute', ] ENTRY_POINTS = """\ +[paste.app_factory] +main = count_buildings:main [console_scripts] +count_buildings = \ + count_buildings.run:start get_tiles_from_image =\ count_buildings.scripts.get_tiles_from_image:start get_examples_from_points =\ count_buildings.scripts.get_examples_from_points:start get_dataset_from_examples =\ count_buildings.scripts.get_dataset_from_examples:start +get_batches_from_datasets =\ + count_buildings.scripts.get_batches_from_datasets:start get_marker_from_dataset =\ count_buildings.scripts.get_marker_from_dataset:start +get_array_shape_from_batches =\ + count_buildings.scripts.get_array_shape_from_batches:start +get_index_from_batches =\ + count_buildings.scripts.get_index_from_batches:start get_arrays_from_image =\ count_buildings.scripts.get_arrays_from_image:start -get_batches_from_datasets =\ - count_buildings.scripts.get_batches_from_datasets:start get_batches_from_arrays =\ count_buildings.scripts.get_batches_from_arrays:start -get_index_from_batches =\ - count_buildings.scripts.get_index_from_batches:start -get_array_shape_from_batches =\ - count_buildings.scripts.get_array_shape_from_batches:start get_predictions_from_arrays =\ count_buildings.scripts.get_predictions_from_arrays:start get_counts_from_probabilities =\ count_buildings.scripts.get_counts_from_probabilities:start +[crosscompute.tools] +describe = count_buildings:describe +include = count_buildings:includeme """ setup( name='count_buildings', - version='0.0.2', + version='0.1', description='count_buildings', long_description=DESCRIPTION, classifiers=[