Fix: Race between layer and Lambda update (#5927)

Author: dsotirho-ucsc

scripts/delete_stale_function_versions.py

@@ -0,0 +1,31 @@
+"""
+Delete all but the latest published version of every AWS Lambda function in the
+current deployment.
+"""
+
+import logging
+
+from azul import (
+    config,
+    require,
+)
+from azul.lambdas import (
+    Lambdas,
+)
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    require(config.terraform_component == '',
+            'This script cannot be run with a Terraform component selected',
+            config.terraform_component)
+    Lambdas().delete_stale_function_versions()
+
+
+if __name__ == '__main__':
+    configure_script_logging(log)
+    main()

src/azul/lambdas.py

@@ -202,3 +202,18 @@ def reset_lambda_roles(self):
                             time.sleep(1)
                         else:
                             break
+
+    def delete_stale_function_versions(self):
+        """
+        Delete all but the latest published version of every AWS Lambda function
+        in the current deployment.
+        """
+        log.info('Deleting stale versions of AWS Lambda functions')
+        for function in self.list_lambdas(deployment=config.deployment_stage,
+                                          all_versions=True):
+            if function.version == '$LATEST':
+                log.info('Skipping latest version %r', function.name)
+            else:
+                log.info('Deleting version %r of %r', function.version, function.name)
+                self._lambda.delete_function(FunctionName=function.name,
+                                             Qualifier=function.version)

src/azul/terraform.py

@@ -708,6 +708,10 @@ def tf_config(self, app_name):
         for resource in resources['aws_lambda_function'].values():
             assert 'layers' not in resource
             resource['layers'] = ['${aws_lambda_layer_version.dependencies.arn}']
+            # Publishing the Lambda function as a new version prevents a race
+            # condition when there's a dependency between updates to the
+            # function's configuration and its code.
+            resource['publish'] = True
             env = config.es_endpoint_env(
                 es_endpoint=(
                     aws.es_endpoint

terraform/Makefile

@@ -43,8 +43,12 @@ import_resources: rename_resources
 plan: import_resources
 	terraform plan
 
+.PHONY: delete_stale_function_versions
+delete_stale_function_versions: import_resources check_python
+	python $(project_root)/scripts/delete_stale_function_versions.py
+
 .PHONY: apply
-apply: import_resources
+apply: delete_stale_function_versions
 ifeq ($(AZUL_PRIVATE_API),1)
 	# For private API we need the VPC endpoints to be created first so that the
 	# aws_lb_target_group_attachment can iterate over the network_interface_ids.
@@ -53,7 +57,7 @@ endif
 	terraform apply
 
 .PHONY: auto_apply
-auto_apply: import_resources
+auto_apply: delete_stale_function_versions
 ifeq ($(AZUL_PRIVATE_API),1)
 	# See `apply` above
 	terraform apply -auto-approve -target aws_vpc_endpoint.indexer -target aws_vpc_endpoint.service