Skip to content

Commit

Permalink
Added support for EMR Security Configurations and Kerberos Attributes. (
Browse files Browse the repository at this point in the history 
getmoto#3456)

* Added support for EMR Security Configurations and Kerberos Attributes.

* Revised exception-raising test to work with pytest api.

* Added htmlcov to .gitignore; upgrading botocore to 1.18.17, per commit d29475e.

Co-authored-by: Joseph Weitekamp <[email protected]>
  • Loading branch information
@jweite @jweite-amazon
jweite and jweite-amazon authored Nov 17, 2020
1 parent f045af7 commit 5fe921c
Show file tree
Hide file tree
Showing 6 changed files with 217 additions and 10 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ tests/file.tmp
.mypy_cache/
*.tmp
.venv/
htmlcov/
9 changes: 8 additions & 1 deletion moto/emr/exceptions.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
from __future__ import unicode_literals

from moto.core.exceptions import RESTError
from moto.core.exceptions import RESTError, JsonRESTError


class EmrError(RESTError):
code = 400


class InvalidRequestException(JsonRESTError):
def __init__(self, message, **kwargs):
super(InvalidRequestException, self).__init__(
"InvalidRequestException", message, **kwargs
)
47 changes: 46 additions & 1 deletion moto/emr/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from boto3 import Session
from dateutil.parser import parse as dtparse
from moto.core import BaseBackend, BaseModel
from moto.emr.exceptions import EmrError
from moto.emr.exceptions import EmrError, InvalidRequestException
from .utils import (
random_instance_group_id,
random_cluster_id,
Expand Down Expand Up @@ -147,6 +147,8 @@ def __init__(
running_ami_version=None,
custom_ami_id=None,
step_concurrency_level=1,
security_configuration=None,
kerberos_attributes=None,
):
self.id = cluster_id or random_cluster_id()
emr_backend.clusters[self.id] = self
Expand Down Expand Up @@ -249,6 +251,10 @@ def __init__(
self.run_bootstrap_actions()
if self.steps:
self.steps[0].start()
self.security_configuration = (
security_configuration # ToDo: Raise if doesn't already exist.
)
self.kerberos_attributes = kerberos_attributes

@property
def instance_groups(self):
Expand Down Expand Up @@ -337,12 +343,20 @@ def set_visibility(self, visibility):
self.visible_to_all_users = visibility


class FakeSecurityConfiguration(BaseModel):
def __init__(self, name, security_configuration):
self.name = name
self.security_configuration = security_configuration
self.creation_date_time = datetime.now(pytz.utc)


class ElasticMapReduceBackend(BaseBackend):
def __init__(self, region_name):
super(ElasticMapReduceBackend, self).__init__()
self.region_name = region_name
self.clusters = {}
self.instance_groups = {}
self.security_configurations = {}

def reset(self):
region_name = self.region_name
Expand Down Expand Up @@ -527,6 +541,37 @@ def remove_auto_scaling_policy(self, cluster_id, instance_group_id):
instance_group = instance_groups[0]
instance_group.auto_scaling_policy = None

def create_security_configuration(self, name, security_configuration):
if name in self.security_configurations:
raise InvalidRequestException(
message="SecurityConfiguration with name '{}' already exists.".format(
name
)
)
security_configuration = FakeSecurityConfiguration(
name=name, security_configuration=security_configuration
)
self.security_configurations[name] = security_configuration
return security_configuration

def get_security_configuration(self, name):
if name not in self.security_configurations:
raise InvalidRequestException(
message="Security configuration with name '{}' does not exist.".format(
name
)
)
return self.security_configurations[name]

def delete_security_configuration(self, name):
if name not in self.security_configurations:
raise InvalidRequestException(
message="Security configuration with name '{}' does not exist.".format(
name
)
)
del self.security_configurations[name]


emr_backends = {}
for region in Session().get_available_regions("emr"):
Expand Down
106 changes: 100 additions & 6 deletions moto/emr/responses.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,29 @@ def add_tags(self):
def cancel_steps(self):
raise NotImplementedError

@generate_boto3_response("CreateSecurityConfiguration")
def create_security_configuration(self):
raise NotImplementedError
name = self._get_param("Name")
security_configuration = self._get_param("SecurityConfiguration")
resp = self.backend.create_security_configuration(
name=name, security_configuration=security_configuration
)
template = self.response_template(CREATE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render(name=name, creation_date_time=resp.creation_date_time)

@generate_boto3_response("DescribeSecurityConfiguration")
def describe_security_configuration(self):
name = self._get_param("Name")
security_configuration = self.backend.get_security_configuration(name=name)
template = self.response_template(DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render(security_configuration=security_configuration)

@generate_boto3_response("DeleteSecurityConfiguration")
def delete_security_configuration(self):
raise NotImplementedError
name = self._get_param("Name")
self.backend.delete_security_configuration(name=name)
template = self.response_template(DELETE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render()

@generate_boto3_response("DescribeCluster")
def describe_cluster(self):
Expand Down Expand Up @@ -190,9 +208,6 @@ def modify_cluster(self):
template = self.response_template(MODIFY_CLUSTER_TEMPLATE)
return template.render(cluster=cluster)

def describe_security_configuration(self):
raise NotImplementedError

@generate_boto3_response("ModifyInstanceGroups")
def modify_instance_groups(self):
instance_groups = self._get_list_prefix("InstanceGroups.member")
Expand Down Expand Up @@ -327,6 +342,39 @@ def run_job_flow(self):
if step_concurrency_level:
kwargs["step_concurrency_level"] = step_concurrency_level

security_configuration = self._get_param("SecurityConfiguration")
if security_configuration:
kwargs["security_configuration"] = security_configuration

kerberos_attributes = {}
kwargs["kerberos_attributes"] = kerberos_attributes

realm = self._get_param("KerberosAttributes.Realm")
if realm:
kerberos_attributes["Realm"] = realm

kdc_admin_password = self._get_param("KerberosAttributes.KdcAdminPassword")
if kdc_admin_password:
kerberos_attributes["KdcAdminPassword"] = kdc_admin_password

cross_realm_principal_password = self._get_param(
"KerberosAttributes.CrossRealmTrustPrincipalPassword"
)
if cross_realm_principal_password:
kerberos_attributes[
"CrossRealmTrustPrincipalPassword"
] = cross_realm_principal_password

ad_domain_join_user = self._get_param("KerberosAttributes.ADDomainJoinUser")
if ad_domain_join_user:
kerberos_attributes["ADDomainJoinUser"] = ad_domain_join_user

ad_domain_join_password = self._get_param(
"KerberosAttributes.ADDomainJoinPassword"
)
if ad_domain_join_password:
kerberos_attributes["ADDomainJoinPassword"] = ad_domain_join_password

cluster = self.backend.run_job_flow(**kwargs)

applications = self._get_list_prefix("Applications.member")
Expand Down Expand Up @@ -560,6 +608,23 @@ def remove_auto_scaling_policy(self):
<ServiceAccessSecurityGroup>{{ cluster.service_access_security_group }}</ServiceAccessSecurityGroup>
</Ec2InstanceAttributes>
<Id>{{ cluster.id }}</Id>
<KerberosAttributes>
{% if 'Realm' in cluster.kerberos_attributes%}
<Realm>{{ cluster.kerberos_attributes['Realm'] }}</Realm>
{% endif %}
{% if 'KdcAdminPassword' in cluster.kerberos_attributes%}
<KdcAdminPassword>{{ cluster.kerberos_attributes['KdcAdminPassword'] }}</KdcAdminPassword>
{% endif %}
{% if 'CrossRealmTrustPrincipalPassword' in cluster.kerberos_attributes%}
<CrossRealmTrustPrincipalPassword>{{ cluster.kerberos_attributes['CrossRealmTrustPrincipalPassword'] }}</CrossRealmTrustPrincipalPassword>
{% endif %}
{% if 'ADDomainJoinUser' in cluster.kerberos_attributes%}
<ADDomainJoinUser>{{ cluster.kerberos_attributes['ADDomainJoinUser'] }}</ADDomainJoinUser>
{% endif %}
{% if 'ADDomainJoinPassword' in cluster.kerberos_attributes%}
<ADDomainJoinPassword>{{ cluster.kerberos_attributes['ADDomainJoinPassword'] }}</ADDomainJoinPassword>
{% endif %}
</KerberosAttributes>
<LogUri>{{ cluster.log_uri }}</LogUri>
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
<Name>{{ cluster.name }}</Name>
Expand All @@ -573,7 +638,9 @@ def remove_auto_scaling_policy(self):
{% if cluster.running_ami_version is not none %}
<RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
{% endif %}
<SecurityConfiguration/>
{% if cluster.security_configuration is not none %}
<SecurityConfiguration>{{ cluster.security_configuration }}</SecurityConfiguration>
{% endif %}
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
<Status>
<State>{{ cluster.state }}</State>
Expand Down Expand Up @@ -1253,3 +1320,30 @@ def remove_auto_scaling_policy(self):
<RequestId>c04a1042-5340-4c0a-a7b5-7779725ce4f7</RequestId>
</ResponseMetadata>
</RemoveAutoScalingPolicyResponse>"""

CREATE_SECURITY_CONFIGURATION_TEMPLATE = """<CreateSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<CreateSecurityConfigurationResult>
<Name>{{name}}</Name>
<CreationDateTime>{{creation_date_time}}</CreationDateTime>
</CreateSecurityConfigurationResult>
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</CreateSecurityConfigurationResponse>"""

DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE = """<DescribeSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<DescribeSecurityConfigurationResult>
<Name>{{security_configuration['name']}}</Name>
<SecurityConfiguration>{{security_configuration['security_configuration']}}</SecurityConfiguration>
<CreationDateTime>{{security_configuration['creation_date_time']}}</CreationDateTime>
</DescribeSecurityConfigurationResult>
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</DescribeSecurityConfigurationResponse>"""

DELETE_SECURITY_CONFIGURATION_TEMPLATE = """<DeleteSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</DeleteSecurityConfigurationResponse>"""
2 changes: 1 addition & 1 deletion requirements-dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ flask
flask-cors
boto>=2.45.0
boto3>=1.4.4
botocore>=1.15.13
botocore>=1.18.17
six>=1.9
prompt-toolkit==2.0.10 # 3.x is not available with python2
click==6.7
Expand Down
62 changes: 61 additions & 1 deletion tests/test_emr/test_emr_boto3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,15 @@ def test_describe_cluster():
args["Instances"]["EmrManagedSlaveSecurityGroup"] = "slave-security-group"
args["Instances"]["KeepJobFlowAliveWhenNoSteps"] = False
args["Instances"]["ServiceAccessSecurityGroup"] = "service-access-security-group"
args["KerberosAttributes"] = {
"Realm": "MY-REALM.COM",
"KdcAdminPassword": "SuperSecretPassword2",
"CrossRealmTrustPrincipalPassword": "SuperSecretPassword3",
"ADDomainJoinUser": "Bob",
"ADDomainJoinPassword": "SuperSecretPassword4",
}
args["Tags"] = [{"Key": "tag1", "Value": "val1"}, {"Key": "tag2", "Value": "val2"}]
args["SecurityConfiguration"] = "my-security-configuration"

cluster_id = client.run_job_flow(**args)["JobFlowId"]

Expand Down Expand Up @@ -145,14 +153,16 @@ def test_describe_cluster():
args["Instances"]["ServiceAccessSecurityGroup"]
)
cl["Id"].should.equal(cluster_id)
cl["KerberosAttributes"].should.equal(args["KerberosAttributes"])
cl["LogUri"].should.equal(args["LogUri"])
cl["MasterPublicDnsName"].should.be.a(six.string_types)
cl["Name"].should.equal(args["Name"])
cl["NormalizedInstanceHours"].should.equal(0)
# cl['ReleaseLabel'].should.equal('emr-5.0.0')
cl.shouldnt.have.key("RequestedAmiVersion")
cl["RunningAmiVersion"].should.equal("1.0.0")
# cl['SecurityConfiguration'].should.be.a(six.string_types)
cl["SecurityConfiguration"].should.be.a(six.string_types)
cl["SecurityConfiguration"].should.equal(args["SecurityConfiguration"])
cl["ServiceRole"].should.equal(args["ServiceRole"])

status = cl["Status"]
Expand Down Expand Up @@ -985,3 +995,53 @@ def test_tags():
client.remove_tags(ResourceId=cluster_id, TagKeys=[t["Key"] for t in input_tags])
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
resp["Tags"].should.equal([])


@mock_emr
def test_security_configurations():

client = boto3.client("emr", region_name="us-east-1")

security_configuration_name = "MySecurityConfiguration"

security_configuration = """
{
"EncryptionConfiguration": {
"AtRestEncryptionConfiguration": {
"S3EncryptionConfiguration": {
"EncryptionMode": "SSE-S3"
}
},
"EnableInTransitEncryption": false,
"EnableAtRestEncryption": true
}
}
""".strip()

resp = client.create_security_configuration(
Name=security_configuration_name, SecurityConfiguration=security_configuration
)

resp["Name"].should.equal(security_configuration_name)
resp["CreationDateTime"].should.be.a("datetime.datetime")

resp = client.describe_security_configuration(Name=security_configuration_name)
resp["Name"].should.equal(security_configuration_name)
resp["SecurityConfiguration"].should.equal(security_configuration)
resp["CreationDateTime"].should.be.a("datetime.datetime")

client.delete_security_configuration(Name=security_configuration_name)

with pytest.raises(ClientError) as ex:
client.describe_security_configuration(Name=security_configuration_name)
ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
ex.value.response["Error"]["Message"].should.match(
r"Security configuration with name .* does not exist."
)

with pytest.raises(ClientError) as ex:
client.delete_security_configuration(Name=security_configuration_name)
ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
ex.value.response["Error"]["Message"].should.match(
r"Security configuration with name .* does not exist."
)

0 comments on commit 5fe921c

Please sign in to comment.