Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial Idem Support #2460

Open
wants to merge 43 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
230fd14
Initial Idem Support
vreutova May 10, 2023
074811c
Added idem examples for testing.
May 11, 2023
6c891dd
Add support for CloudWatch resources
vreutova May 11, 2023
ba9857c
Merge pull request #1 from praveenkalluri/master
vreutova May 11, 2023
40414e6
Merge pull request #2 from vreutova/cloud_watch
vreutova May 11, 2023
887287a
Added support for elb resource
May 11, 2023
9a07c82
Merge pull request #3 from Ankush-Sethi/master
vreutova May 11, 2023
f3ed8f0
Added tests file for Idem
May 11, 2023
3dd3ff2
added acm_cert and cloudwatch_loggroup resources
May 12, 2023
7ad6103
conflict fixes
May 12, 2023
708c8a3
Added support for EC2 Transit gateway vpc attachment
May 12, 2023
cdae578
Added support for config-rule configuration-recorder
May 12, 2023
aad6e88
added eip resource
May 12, 2023
c3dc05f
Merge pull request #6 from praveenkalluri/ec2_resources
vreutova May 12, 2023
5855ee5
Merge remote-tracking branch 'upstream/master'
May 12, 2023
c2fe7bf
added test cases for acm and eip
May 12, 2023
7b5eb82
added support for flow_log
May 12, 2023
1ce6281
Merge remote-tracking branch 'upstream/master'
May 12, 2023
fb731ea
Fix parser
vreutova May 12, 2023
4c94d69
Merge pull request #5 from charugargtillster/master
vreutova May 12, 2023
4b0366e
Added support for KMS and Added Test for ELB, KMS, config_rule, confi…
May 12, 2023
98f392b
Added Support for eks_fargate_profile
May 13, 2023
157946e
Added support for RDS cluster and cluster instance
May 14, 2023
c763ddc
Added Support for efs_file_system
May 14, 2023
b1027f4
Added support for wafv2_web_acl
May 15, 2023
6fe86e2
Merge pull request #7 from Ankush-Sethi/master
vreutova May 15, 2023
f265954
Merge pull request #8 from vreutova/parser
vreutova May 15, 2023
62fed95
Merge pull request #9 from Ankush-Sethi/eks_fargate_profile
vreutova May 15, 2023
2f6f19f
Merge pull request #10 from praveenkalluri/rds_cluster
vreutova May 15, 2023
e1c001e
added support for ebs snapshot and volume
May 15, 2023
e501ff7
Fixed idem/instance.go logic
vreutova May 15, 2023
d145408
Merge pull request #13 from charugargtillster/master
vreutova May 15, 2023
f551622
Merge pull request #14 from vreutova/iinstance_ebs_fixes
vreutova May 15, 2023
2bd30e6
Removed typo and Corrected the message for Seller fees
May 16, 2023
f3a1d28
Commented the lifecyle_policy , Currently it is not supported by Idem
May 16, 2023
ccf1ec1
Merge pull request #12 from Ankush-Sethi/web_acl_wafv2
vreutova May 16, 2023
e9b6861
Added Support for Cloudfront_distribution
May 16, 2023
6f16802
Merge pull request #15 from Ankush-Sethi/cloudfront_distribution
vreutova May 16, 2023
85f9d33
Added Support for S3 bucket lifecycle configuration
May 17, 2023
3824fd6
Merge pull request #16 from Ankush-Sethi/s3_bucket_lifecycle_configur…
vreutova May 17, 2023
ac6659e
Added GetEIPRegistryItem to resolve the merge conflict with master
May 17, 2023
f85a720
Merge branch 'master' into efs_file_system
vreutova May 18, 2023
00204e5
Merge pull request #11 from Ankush-Sethi/efs_file_system
vreutova May 18, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
577 changes: 577 additions & 0 deletions examples/idem/idem_plan.json

Large diffs are not rendered by default.

6 changes: 5 additions & 1 deletion internal/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,10 @@ func hasSupportedTerraformProvider(rType string) bool {
return strings.HasPrefix(rType, "aws_") || strings.HasPrefix(rType, "google_") || strings.HasPrefix(rType, "azurerm_")
}

func hasSupportedIdemProvider(rType string) bool {
return strings.HasPrefix(rType, "states.aws.") || strings.HasPrefix(rType, "states.gcp.") || strings.HasPrefix(rType, "states.azure.")
}

func BuildSummary(resources []*schema.Resource, opts SummaryOptions) (*Summary, error) {
s := &Summary{}

Expand All @@ -733,7 +737,7 @@ func BuildSummary(resources []*schema.Resource, opts SummaryOptions) (*Summary,
}

for _, r := range resources {
if !opts.IncludeUnsupportedProviders && !hasSupportedTerraformProvider(r.ResourceType) {
if !opts.IncludeUnsupportedProviders && !hasSupportedTerraformProvider(r.ResourceType) && !hasSupportedIdemProvider(r.ResourceType) {
continue
}

Expand Down
30 changes: 30 additions & 0 deletions internal/providers/detect.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"archive/zip"
"encoding/json"
"fmt"
"github.com/infracost/infracost/internal/providers/idem"
"os"
"path/filepath"
"sync"
Expand Down Expand Up @@ -88,6 +89,9 @@ func Detect(ctx *config.ProjectContext, includePastResources bool) (schema.Provi
return terraform.NewStateJSONProvider(ctx, includePastResources), nil
case "cloudformation":
return cloudformation.NewTemplateProvider(ctx, includePastResources), nil
case "idem":
return idem.NewTemplateProvider(ctx, includePastResources), nil

}

return nil, fmt.Errorf("could not detect path type for '%s'", path)
Expand Down Expand Up @@ -116,6 +120,10 @@ func validateProjectForHCL(ctx *config.ProjectContext) error {
}

func DetectProjectType(path string, forceCLI bool) string {
if isIdemJson(path) {
return "idem"
}

if isCloudFormationTemplate(path) {
return "cloudformation"
}
Expand Down Expand Up @@ -170,6 +178,28 @@ func isTerraformPlanJSON(path string) bool {
return jsonFormat.FormatVersion != "" && jsonFormat.PlannedValues != nil
}

func isIdemJson(path string) bool {
b, err := os.ReadFile(path)
if err != nil {
return false
}

type jsonFormat struct {
Tag string `json:"tag"`
Name string `json:"name"`
NewState interface{} `json:"new_state"`
OldState interface{} `json:"old_state"`
}

var jsonMap map[string]jsonFormat

err = json.Unmarshal(b, &jsonMap)
if err != nil {
return false
}
return true
}

func isTerraformStateJSON(path string) bool {
b, err := os.ReadFile(path)
if err != nil {
Expand Down
23 changes: 23 additions & 0 deletions internal/providers/idem/aws/acm_certificate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package aws

import (
"github.com/infracost/infracost/internal/resources/aws"
"github.com/infracost/infracost/internal/schema"
)

func GetACMCertificate() *schema.RegistryItem {
return &schema.RegistryItem{
Name: "states.aws.acm.certificate_manager.present",
RFunc: NewACMCertificate,
}
}
func NewACMCertificate(d *schema.ResourceData, u *schema.UsageData) *schema.Resource {
r := &aws.ACMCertificate{
Address: d.Address,
Region: d.Get("region").String(),
CertificateAuthorityARN: d.Get("certificate_authority_arn").String(),
}

r.PopulateUsage(u)
return r.BuildResource()
}
16 changes: 16 additions & 0 deletions internal/providers/idem/aws/acm_certificate_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package aws_test

import (
"github.com/infracost/infracost/internal/providers/idem/idemtest"
"testing"
)

func TestACMCertificateGoldenFile(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("skipping test in short mode")
}
opts := idemtest.DefaultGoldenFileOptions()
opts.CaptureLogs = true
idemtest.GoldenFileResourceTestsWithOpts(t, "acm_certificate_test", opts)
}
36 changes: 36 additions & 0 deletions internal/providers/idem/aws/app_autoscalling_target.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package aws

import (
"github.com/infracost/infracost/internal/resources/aws"
"github.com/infracost/infracost/internal/schema"
)

func getAppAutoscalingTargetRegistryItem() *schema.RegistryItem {
return &schema.RegistryItem{
Name: "state.aws.application_autoscaling.scalable_target.present",
RFunc: NewAppAutoscalingTargetResource,
// This reference is used by other resources (e.g. DynamoDBTable) to generate
// a reverse reference
ReferenceAttributes: []string{"resource_id"},
}
}

func NewAppAutoscalingTargetResource(d *schema.ResourceData, u *schema.UsageData) *schema.Resource {
r := newAppAutoscalingTarget(d, u)
return r.BuildResource()
}

func newAppAutoscalingTarget(d *schema.ResourceData, u *schema.UsageData) *aws.AppAutoscalingTarget {
r := &aws.AppAutoscalingTarget{
Address: d.Address,
Region: d.Get("region").String(),
ResourceID: d.Get("resource_id").String(),
ScalableDimension: d.Get("scalable_dimension").String(),
MinCapacity: d.Get("min_capacity").Int(),
MaxCapacity: d.Get("max_capacity").Int(),
}

r.PopulateUsage(u)

return r
}
203 changes: 203 additions & 0 deletions internal/providers/idem/aws/autoscaling_group.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
package aws

import (
"fmt"
"strings"

"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"github.com/tidwall/gjson"

"github.com/infracost/infracost/internal/resources/aws"

"github.com/infracost/infracost/internal/schema"
)

func GetAutoscalingGroupRegistryItem() *schema.RegistryItem {
return &schema.RegistryItem{
Name: "states.aws.autoscaling.auto_scaling_group.present",
RFunc: NewAutoscalingGroup,
ReferenceAttributes: []string{
"states.aws.autoscaling.auto_scaling_group.present:launch_configuration_name",
"states.aws.ec2.launch_template.present:launch_template.LaunchTemplateId",
"states.aws.ec2.launch_template.present:mixed_instances_policy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateId",
},
}
}

func NewAutoscalingGroup(d *schema.ResourceData, u *schema.UsageData) *schema.Resource {
a := &aws.AutoscalingGroup{
Address: d.Address,
Region: d.Get("region").String(),
Name: d.Get("name").String(),
}

var instanceCount int64

if !d.IsEmpty("desired_capacity") {
instanceCount = d.Get("desired_capacity").Int()
} else {
instanceCount = d.Get("min_size").Int()
if instanceCount == 0 {
log.Debugf("Using instance count 1 for %s since no desired_capacity or non-zero min_size is set. To override this set the instance_count attribute for this resource in the Infracost usage file.", a.Address)
instanceCount = 1
}
}

// The Autoscaling Group resource has either a Launch Configuration or Launch Template sub-resource.
// So we create generic resources for these and add them as a subresource of the Autoscaling Group resource.
launchConfigurationRef := d.References("states.aws.autoscaling.auto_scaling_group.present:launch_configuration_name")
launchTemplateRef := d.References("states.aws.ec2.launch_template.present:launch_template.LaunchTemplateId")
mixedInstanceLaunchTemplateRef := d.References("states.aws.ec2.launch_template.present:mixed_instances_policy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateId")

if len(launchConfigurationRef) > 0 {
data := launchConfigurationRef[0]
a.LaunchConfiguration = newLaunchConfiguration(data, u, a.Region, instanceCount)
} else if len(launchTemplateRef) > 0 {
data := launchTemplateRef[0]

onDemandPercentageAboveBaseCount := int64(100)
if strings.ToLower(data.Get("InstanceMarketOptions.MarketType").String()) == "spot" {
onDemandPercentageAboveBaseCount = int64(0)
}

a.LaunchTemplate = newLaunchTemplate(data, a.Region, instanceCount, int64(0), onDemandPercentageAboveBaseCount)
} else if len(mixedInstanceLaunchTemplateRef) > 0 {
data := mixedInstanceLaunchTemplateRef[0]
a.LaunchTemplate = newMixedInstancesLaunchTemplate(data, u, a.Region, instanceCount, d.Get("mixed_instances_policy"))
}

a.PopulateUsage(u)

return a.BuildResource()
}

func newLaunchConfiguration(d *schema.ResourceData, u *schema.UsageData, region string, instanceCount int64) *aws.LaunchConfiguration {
purchaseOption := "on_demand"
if d.Get("spot_price").String() != "" {
purchaseOption = "spot"
}

a := &aws.LaunchConfiguration{
Address: d.Address,
Region: region,
AMI: d.Get("image_id").String(),
InstanceCount: intPtr(instanceCount),
Tenancy: d.Get("placement_tenancy").String(),
PurchaseOption: purchaseOption,
InstanceType: d.Get("instance_type").String(),
EBSOptimized: d.Get("ebs_optimized").Bool(),
EnableMonitoring: d.GetBoolOrDefault("instance_monitoring.Enabled", true),
// TODO
//CPUCredits: d.Get("launch_template_data.CreditSpecification.CpuCredits").String(),
}

/*
// TODO: What is root?
a.RootBlockDevice = &aws.EBSVolume{
Address: "root_block_device",
Region: region,
Type: d.Get("launch_template_data.root_block_device.BlockDeviceMappings.0.Ebs.VolumeType").String(),
IOPS: d.Get("launch_template_data.root_block_device.BlockDeviceMappings.0.Ebs.Iops").Int(),
}

if d.Get("launch_template_data.root_block_device.BlockDeviceMappings.0.Ebs.VolumeSize").Type != gjson.Null {
a.RootBlockDevice.Size = intPtr(d.Get("launch_template_data.root_block_device.BlockDeviceMappings.0.Ebs.VolumeSize").Int())
}*/

for i, data := range d.Get("launch_template_data.BlockDeviceMappings").Array() {
ebsBlockDevice := &aws.EBSVolume{
Address: fmt.Sprintf("ebs_block_device[%d]", i),
Region: region,
Type: data.Get("VolumeSize").String(),
IOPS: data.Get("Iops").Int(),
}

if data.Get("VolumeSize").Type != gjson.Null {
ebsBlockDevice.Size = intPtr(data.Get("VolumeSize").Int())
}

a.EBSBlockDevices = append(a.EBSBlockDevices, ebsBlockDevice)
}

return a
}

func newLaunchTemplate(d *schema.ResourceData, region string, instanceCount, onDemandBaseCount, onDemandPercentageAboveBaseCount int64) *aws.LaunchTemplate {
a := &aws.LaunchTemplate{
Address: d.Address,
Region: region,
AMI: d.Get("launch_template_data.ImageId").String(),
InstanceCount: intPtr(instanceCount),
OnDemandBaseCount: onDemandBaseCount,
OnDemandPercentageAboveBaseCount: onDemandPercentageAboveBaseCount,
Tenancy: d.Get("launch_template_data.Placement.Tenancy").String(),
InstanceType: d.Get("launch_template_data.InstanceType").String(),
EBSOptimized: d.Get("launch_template_data.EbsOptimized").Bool(),
EnableMonitoring: d.Get("launch_template_data.Monitoring.Enabled").Bool(),
CPUCredits: d.Get("CreditSpecification.0.cpu_credits").String(),
}

if d.Get("launch_template_data.ElasticInferenceAccelerators.0.Type").Type != gjson.Null {
a.ElasticInferenceAcceleratorType = strPtr(d.Get("launch_template_data.ElasticInferenceAccelerators.0.Type").String())
}

for i, data := range d.Get("launch_template_data.BlockDeviceMappings.#.Ebs|@flatten").Array() {
ebsBlockDevice := &aws.EBSVolume{
Address: fmt.Sprintf("block_device_mapping[%d]", i),
Region: region,
Type: data.Get("VolumeType").String(),
IOPS: data.Get("Iops").Int(),
}

if data.Get("VolumeSize").Type != gjson.Null {
ebsBlockDevice.Size = intPtr(data.Get("VolumeSize").Int())
}

a.EBSBlockDevices = append(a.EBSBlockDevices, ebsBlockDevice)
}

return a
}

func newMixedInstancesLaunchTemplate(d *schema.ResourceData, u *schema.UsageData, region string, capacity int64, mixedInstancePolicyData gjson.Result) *aws.LaunchTemplate {
overrideInstanceType, instanceCount := getInstanceTypeAndCount(mixedInstancePolicyData, capacity)
if overrideInstanceType != "" {
d.Set("launch_template_data.InstanceType", overrideInstanceType)
}

instanceDistribution := mixedInstancePolicyData.Get("InstancesDistribution")
onDemandBaseCount := int64(0)
if instanceDistribution.Get("OnDemandBaseCapacity").Exists() {
onDemandBaseCount = instanceDistribution.Get("OnDemandBaseCapacity").Int()
}

onDemandPercentageAboveBaseCount := int64(100)
if instanceDistribution.Get("OnDemandPercentageAboveBaseCapacity").Exists() {
onDemandPercentageAboveBaseCount = instanceDistribution.Get("OnDemandPercentageAboveBaseCapacity").Int()
}

return newLaunchTemplate(d, region, instanceCount, onDemandBaseCount, onDemandPercentageAboveBaseCount)
}

func getInstanceTypeAndCount(mixedInstancePolicyData gjson.Result, capacity int64) (string, int64) {
count := capacity
instanceType := ""

override := mixedInstancePolicyData.Get("LaunchTemplate.Overrides.0")
if override.Exists() {
instanceType = override.Get("InstanceType").String()
weightedCapacity := int64(1)
if override.Get("WeightedCapacity").Type != gjson.Null {
weightedCapacity = override.Get("WeightedCapacity").Int()
}

if weightedCapacity == 0 {
count = int64(0)
} else {
count = decimal.NewFromInt(capacity).Div(decimal.NewFromInt(weightedCapacity)).Ceil().IntPart()
}
}

return instanceType, count
}