Fix: Race between layer and Lambda update (#5927)

Author: dsotirho-ucsc

Makefile

@@ -99,6 +99,7 @@ $(1)terraform: lambdas
 
 .PHONY: $(1)deploy
 $(1)deploy: check_python $(1)terraform
+	python $(project_root)/scripts/delete_lambda_versions.py
 	python $(project_root)/scripts/post_deploy_tdr.py
 endef
 

scripts/delete_lambda_versions.py

@@ -0,0 +1,41 @@
+"""
+Delete all published AWS Lambda versions except for "$LATEST" in the current
+deployment.
+"""
+
+import logging
+
+from azul.deployment import (
+    aws,
+)
+from azul.lambdas import (
+    Lambdas,
+)
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    lambdas = Lambdas().list_lambdas()
+    for lambda_ in lambdas:
+        response = aws.lambda_.list_versions_by_function(
+            FunctionName=lambda_.name
+        )
+        for version in response['Versions']:
+            if version['Version'] == '$LATEST':
+                pass
+            else:
+                version_number = version['Version']
+                log.info('Deleting published version %s of %s', version_number, lambda_.name)
+                aws.lambda_.delete_function(
+                    FunctionName=lambda_.name,
+                    Qualifier=version_number
+                )
+
+
+if __name__ == '__main__':
+    configure_script_logging(log)
+    main()

src/azul/terraform.py

@@ -708,6 +708,10 @@ def tf_config(self, app_name):
         for resource in resources['aws_lambda_function'].values():
             assert 'layers' not in resource
             resource['layers'] = ['${aws_lambda_layer_version.dependencies.arn}']
+            # Publishing the Lambda function as a new version prevents possible
+            # race conditions when an update to the function's configuration and
+            # code rely on the update of each other in order to work correctly.
+            resource['publish'] = True
             env = config.es_endpoint_env(
                 es_endpoint=(
                     aws.es_endpoint