Fix bugs in sync data

Author: micahjsmith

Makefile

@@ -55,7 +55,7 @@ sync_data_to_s3:
 
 ## Download Data from S3
 sync_data_from_s3:
-	$(PYTHON_INTERPRETER) -m dengue_prediction.data.sync_data upload
+	$(PYTHON_INTERPRETER) -m dengue_prediction.data.sync_data download
 
 ## Set up python interpreter environment
 create_environment:

config.yml

@@ -7,6 +7,7 @@ problem:
 
 data:
   s3_bucket: mit-dai-micahs-misc
+  s3_bucket_region: us-east-1
   train: data/raw
   entities_table_name: dengue_features
   target_table_name: dengue_labels

dengue_prediction/data/sync_data.py

@@ -20,24 +20,28 @@ def get_s3_base_url():
 def run_aws_s3_sync(src, dst, credentials=True, profile=None):
     cmd = ['aws', 's3', 'sync']
     if not credentials:
-        cmd.append('--no-sign-request')
-        region = 'us-east-1'
+        cmd.insert(1, '--no-sign-request')
+        region = cg('data', 's3_bucket_region')
         cmd.append('--region={}'.format(region))
+        cmd.append('--source-region={}'.format(region))
     cmd.append(src)
     cmd.append(dst)
     if profile is not None:
         cmd.append('--profile')
         cmd.append(profile)
     try:
-        output = subprocess.check_output(cmd, universal_newlines=True)
+        logger.debug('Executing command: {}'.format(' '.join(cmd)))
+        output = subprocess.check_output(
+            cmd, universal_newlines=True, stderr=subprocess.STDOUT)
         if output:
             logger.info(output)
         return output
     except subprocess.CalledProcessError as e:
-        if 'Unable to locate credentials' in e.output:
-            if credentials:
-                return run_aws_s3_sync(
-                    src, dst, credentials=False, profile=profile)
+        if 'Unable to locate credentials' in e.output and credentials:
+            return run_aws_s3_sync(
+                src, dst, credentials=False, profile=profile)
+        else:
+            raise
 
 
 def upload(profile=None):