Skip to content

Commit

Permalink
make AWS resource tagging optional via a config flag
Browse files Browse the repository at this point in the history
  • Loading branch information
haugenj committed Jul 7, 2020
1 parent 153e22f commit 991f956
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 40 deletions.
1 change: 1 addition & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func setupCloudProvider(nodegroups []controller.NodeGroupOptions) cloudprovider.
FleetInstanceReadyTimeout: n.AWS.FleetInstanceReadyTimeoutDuration(),
Lifecycle: n.AWS.Lifecycle,
InstanceTypeOverrides: n.AWS.InstanceTypeOverrides,
ResourceTagging: n.AWS.ResourceTagging,
},
})
}
Expand Down
7 changes: 7 additions & 0 deletions docs/configuration/nodegroup.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ node_groups:
launch_template_id: "1"
lifecycle: on-demand
instance_type_overrides: ["t2.large", "t3.large"]
resource_tagging: false
```
## Options
Expand Down Expand Up @@ -242,3 +243,9 @@ Dependent on Launch Template ID being specified.
An optional list of instance types to override the instance type within the launch template. Providing multiple instance
types here increases the likelihood of a Spot request being successful. If omitted the instance type to request will
be taken from the launch template.

### `aws.resource_tagging`

Tag ASG and Fleet Request resources used by Escalator with the metatdata key-value pair
`k8s.io/atlassian-escalator/enabled`:`true`. Tagging doesn't alter the functionality of Escalator. Read more about
tagging your AWS resources [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html).
91 changes: 51 additions & 40 deletions pkg/cloudprovider/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ const (
LifecycleSpot = "spot"
// The AttachInstances API only supports adding 20 instances at a time
batchSize = 20
// tagKey is the key for the tag applied to the ASG
// tagKey is the key for the tag applied to ASGs and Fleet requests
tagKey = "k8s.io/atlassian-escalator/enabled"
// tagValue is the value for the tag applied to the ASG
// tagValue is the value for the tag applied to ASGs and Fleet requests
tagValue = "true"
)

Expand Down Expand Up @@ -96,33 +96,7 @@ func (c *CloudProvider) RegisterNodeGroups(groups ...cloudprovider.NodeGroupConf
continue
}

// Search the asg for tagKey, then add it if it's not present
hasTag := false
tags := group.Tags
for _, tag := range tags {
if *tag.Key == tagKey {
hasTag = true
break
}
}
if !hasTag {
tagInput := &autoscaling.CreateOrUpdateTagsInput{
Tags: []*autoscaling.Tag{
{
Key: awsapi.String(tagKey),
PropagateAtLaunch: awsapi.Bool(true),
ResourceId: awsapi.String(id),
ResourceType: awsapi.String("auto-scaling-group"),
Value: awsapi.String(tagValue),
},
},
}
log.WithField("asg", id).Infof("creating auto scaling tag")
_, err := c.service.CreateOrUpdateTags(tagInput)
if err != nil {
log.Errorf("failed to create auto scaling tag for ASG %v", id)
}
}
addASGTags(configs[id], group, c)

c.nodeGroups[id] = NewNodeGroup(configs[id], group, c)
}
Expand Down Expand Up @@ -517,17 +491,6 @@ func createFleetInput(n NodeGroup, addCount int64) (*ec2.CreateFleetInput, error
Overrides: launchTemplateOverrides,
},
},
TagSpecifications: []*ec2.TagSpecification{
{
ResourceType: awsapi.String(ec2.ResourceTypeFleet),
Tags: []*ec2.Tag{
{
Key: awsapi.String(tagKey),
Value: awsapi.String(tagValue),
},
},
},
},
}

if lifecycle == LifecycleOnDemand {
Expand All @@ -542,6 +505,20 @@ func createFleetInput(n NodeGroup, addCount int64) (*ec2.CreateFleetInput, error
}
}

if n.config.AWSConfig.ResourceTagging {
fleetInput.TagSpecifications = []*ec2.TagSpecification{
{
ResourceType: awsapi.String(ec2.ResourceTypeFleet),
Tags: []*ec2.Tag{
{
Key: awsapi.String(tagKey),
Value: awsapi.String(tagValue),
},
},
},
}
}

return fleetInput, nil
}

Expand Down Expand Up @@ -590,3 +567,37 @@ func createTemplateOverrides(n NodeGroup) ([]*ec2.FleetLaunchTemplateOverridesRe

return launchTemplateOverrides, nil
}

// addASGTags will search an ASG for the tagKey and add the tag if it's not found
func addASGTags(config *cloudprovider.NodeGroupConfig, asg *autoscaling.Group, provider *CloudProvider) {
if !config.AWSConfig.ResourceTagging {
return
}

tags := asg.Tags
for _, tag := range tags {
if *tag.Key == tagKey {
return
}
}

id := awsapi.StringValue(asg.AutoScalingGroupName)

tagInput := &autoscaling.CreateOrUpdateTagsInput{
Tags: []*autoscaling.Tag{
{
Key: awsapi.String(tagKey),
PropagateAtLaunch: awsapi.Bool(true),
ResourceId: awsapi.String(id),
ResourceType: awsapi.String("auto-scaling-group"),
Value: awsapi.String(tagValue),
},
},
}

log.WithField("asg", id).Infof("creating auto scaling tag")
_, err := provider.service.CreateOrUpdateTags(tagInput)
if err != nil {
log.Errorf("failed to create auto scaling tag for ASG %v", id)
}
}
88 changes: 88 additions & 0 deletions pkg/cloudprovider/aws/aws_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func setupAWSMocks() {
MaxSize: aws.Int64(int64(25)),
DesiredCapacity: aws.Int64(int64(1)),
VPCZoneIdentifier: aws.String("subnetID-1,subnetID-2"),
Tags: []*autoscaling.TagDescription{},
}

mockAWSConfig = cloudprovider.AWSNodeGroupConfig{
Expand All @@ -36,6 +37,7 @@ func setupAWSMocks() {
FleetInstanceReadyTimeout: tickerTimeout,
Lifecycle: LifecycleOnDemand,
InstanceTypeOverrides: []string{"instance-1", "instance-2"},
ResourceTagging: false,
}

mockNodeGroup = NodeGroup{
Expand Down Expand Up @@ -128,6 +130,29 @@ func TestCreateFleetInput(t *testing.T) {
}
}

func TestCreateFleetInput_WithResourceTagging(t *testing.T) {
setupAWSMocks()
autoScalingGroups := []*autoscaling.Group{&mockASG}
nodeGroups := map[string]*NodeGroup{mockNodeGroup.id: &mockNodeGroup}
addCount := int64(2)

awsCloudProvider, _ := newMockCloudProviderUsingInjection(
nodeGroups,
&test.MockAutoscalingService{
DescribeAutoScalingGroupsOutput: &autoscaling.DescribeAutoScalingGroupsOutput{
AutoScalingGroups: autoScalingGroups,
},
},
&test.MockEc2Service{},
)
mockNodeGroup.provider = awsCloudProvider
mockAWSConfig.ResourceTagging = true
mockNodeGroupConfig.AWSConfig = mockAWSConfig

_, err := createFleetInput(mockNodeGroup, addCount)
assert.Nil(t, err, "Expected no error from createFleetInput")
}

func TestCreateTemplateOverrides_FailedCall(t *testing.T) {
setupAWSMocks()
expectedError := errors.New("call failed")
Expand Down Expand Up @@ -234,3 +259,66 @@ func TestCreateTemplateOverrides_NoInstanceTypeOverrides_Success(t *testing.T) {
_, err := createTemplateOverrides(mockNodeGroup)
assert.Nil(t, err, "Expected no error from createTemplateOverrides")
}
func TestAddASGTags_ResourceTaggingFalse(t *testing.T) {
setupAWSMocks()
mockNodeGroupConfig.AWSConfig.ResourceTagging = false
awsCloudProvider, _ := newMockCloudProviderUsingInjection(
nil,
&test.MockAutoscalingService{},
&test.MockEc2Service{},
)
addASGTags(&mockNodeGroupConfig, &mockASG, awsCloudProvider)
}

func TestAddASGTags_ResourceTaggingTrue(t *testing.T) {
setupAWSMocks()
mockNodeGroupConfig.AWSConfig.ResourceTagging = true

// Mock service call
awsCloudProvider, _ := newMockCloudProviderUsingInjection(
nil,
&test.MockAutoscalingService{
CreateOrUpdateTagsOutput: &autoscaling.CreateOrUpdateTagsOutput{},
},
&test.MockEc2Service{},
)
addASGTags(&mockNodeGroupConfig, &mockASG, awsCloudProvider)
}

func TestAddASGTags_ASGAlreadyTagged(t *testing.T) {
setupAWSMocks()
mockNodeGroupConfig.AWSConfig.ResourceTagging = true

// Mock existing tags
key := tagKey
asgTag := autoscaling.TagDescription{
Key: &key,
}
mockASG.Tags = append(mockASG.Tags, &asgTag)

// Mock service call
awsCloudProvider, _ := newMockCloudProviderUsingInjection(
nil,
&test.MockAutoscalingService{
CreateOrUpdateTagsOutput: &autoscaling.CreateOrUpdateTagsOutput{},
},
&test.MockEc2Service{},
)
addASGTags(&mockNodeGroupConfig, &mockASG, awsCloudProvider)
}

func TestAddASGTags_WithErrorResponse(t *testing.T) {
setupAWSMocks()
mockNodeGroupConfig.AWSConfig.ResourceTagging = true

// Mock service call and error
awsCloudProvider, _ := newMockCloudProviderUsingInjection(
nil,
&test.MockAutoscalingService{
CreateOrUpdateTagsOutput: &autoscaling.CreateOrUpdateTagsOutput{},
CreateOrUpdateTagsErr: errors.New("unauthorized"),
},
&test.MockEc2Service{},
)
addASGTags(&mockNodeGroupConfig, &mockASG, awsCloudProvider)
}
1 change: 1 addition & 0 deletions pkg/cloudprovider/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,4 +117,5 @@ type AWSNodeGroupConfig struct {
FleetInstanceReadyTimeout time.Duration
Lifecycle string
InstanceTypeOverrides []string
ResourceTagging bool
}
1 change: 1 addition & 0 deletions pkg/controller/node_group.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ type AWSNodeGroupOptions struct {
FleetInstanceReadyTimeout string `json:"fleet_instance_ready_timeout,omitempty" yaml:"fleet_instance_ready_timeout,omitempty"`
Lifecycle string `json:"lifecycle,omitempty" yaml:"lifecycle,omitempty"`
InstanceTypeOverrides []string `json:"instance_type_overrides,omitempty" yaml:"instance_type_overrides,omitempty"`
ResourceTagging bool `json:"resource_tagging,omitempty" yaml:"resource_tagging,omitempty"`

// Private variables for storing the parsed duration from the string
fleetInstanceReadyTimeout time.Duration
Expand Down

0 comments on commit 991f956

Please sign in to comment.