-
Notifications
You must be signed in to change notification settings - Fork 152
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. Weβll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for GCP scopes #1296
base: main
Are you sure you want to change the base?
Changes from 10 commits
9396bcb
15cb0d2
00a7829
8d57e8e
ea6fb51
45269ac
78cc7a9
0a8f5f8
aca1cf5
8264ab4
dced4e9
1214a02
2ccb9a9
cc82707
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,5 @@ | ||
package config | ||
|
||
type GCPTerraformConfig struct { | ||
Project string `cty:"project"` | ||
Region string `cty:"region"` | ||
Zone string `cty:"zone"` | ||
Scopes []string `cty:"scopes"` | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,6 @@ | ||
package google | ||
|
||
import ( | ||
"os" | ||
|
||
"github.com/snyk/driftctl/pkg/output" | ||
"github.com/snyk/driftctl/pkg/remote/google/config" | ||
"github.com/snyk/driftctl/pkg/remote/terraform" | ||
|
@@ -34,7 +32,7 @@ func NewGCPTerraformProvider(version string, progress output.Progress, configDir | |
tfProvider, err := terraform.NewTerraformProvider(installer, terraform.TerraformProviderConfig{ | ||
Name: p.name, | ||
GetProviderConfig: func(alias string) interface{} { | ||
return p.GetConfig() | ||
return p.SetConfig(nil) | ||
}, | ||
}, progress) | ||
|
||
|
@@ -55,10 +53,8 @@ func (p *GCPTerraformProvider) Version() string { | |
return p.version | ||
} | ||
|
||
func (p *GCPTerraformProvider) GetConfig() config.GCPTerraformConfig { | ||
func (p *GCPTerraformProvider) SetConfig(scopes []string) config.GCPTerraformConfig { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Well there's also nothing to get as during automated tests we have to set the config (scope) |
||
return config.GCPTerraformConfig{ | ||
Project: os.Getenv("CLOUDSDK_CORE_PROJECT"), | ||
Region: os.Getenv("CLOUDSDK_COMPUTE_REGION"), | ||
Zone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), | ||
Comment on lines
-61
to
-62
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you removed Region and Zone parameters ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Because they are not needed since we use scopes. |
||
Scopes: scopes, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think you missed the point, scopes are not what we are expecting here in this struct https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#scopes I think this PR is more complicated than you think, it seems that it is not possible to initialize a We probably need to create another kind of detail fetcher that will lazy instantiate one terraform providers per project. We are doing that for S3 buckets, but it's a really tricky thing. Did you try to run this PR in deep mode ? I think this is broken There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The entire point I had in mind was to support multiple or boarder scopes for scanning. I did not run it in deep mode since it's not recommended, but for a simple mode targeting various projects or an entire GCP org it works really good for me: "summary": {
"total_resources": 400,
"total_changed": 0,
"total_unmanaged": 398,
"total_missing": 1,
"total_managed": 1
} |
||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package repository | |
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
|
||
asset "cloud.google.com/go/asset/apiv1" | ||
|
@@ -75,102 +76,131 @@ func NewAssetRepository(client *asset.Client, config config.GCPTerraformConfig, | |
} | ||
|
||
func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { | ||
req := &assetpb.ListAssetsRequest{ | ||
Parent: fmt.Sprintf("projects/%s", s.config.Project), | ||
ContentType: assetpb.ContentType_RESOURCE, | ||
AssetTypes: []string{ | ||
cloudFunctionsFunction, | ||
bigtableInstanceAssetType, | ||
bigtableTableAssetType, | ||
sqlDatabaseInstanceAssetType, | ||
computeGlobalAddressAssetType, | ||
nodeGroupAssetType, | ||
}, | ||
} | ||
var results []*assetpb.Asset | ||
|
||
cacheKey := "listAllResources" | ||
cachedResults := s.cache.GetAndLock(cacheKey) | ||
defer s.cache.Unlock(cacheKey) | ||
if cachedResults != nil { | ||
results = cachedResults.([]*assetpb.Asset) | ||
} | ||
filteredResults := []*assetpb.Asset{} | ||
var erorString string | ||
bgdanix marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
for _, scope := range s.config.Scopes { | ||
cacheKey := fmt.Sprintf("listAllResources_%s", scope) | ||
cachedResults := s.cache.GetAndLock(cacheKey) | ||
defer s.cache.Unlock(cacheKey) | ||
|
||
req := &assetpb.ListAssetsRequest{ | ||
Parent: scope, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure that a folder is supported for Can you double check that ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Indeed. Ideally we should not use listAssets but only searchAssets as the main difference is that list takes a snapshot of how the assets were at a particular time, which could result in a diff from what it's actually present. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Where did you get this info ? I asked this question here a couple of month ago and I didn't get any answer. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. From here there are a couple of hints about it:
Also in the API it's mentioned the purpose is to list resources from a specific timeframe
While the search is more appropriate for our intent:
|
||
ContentType: assetpb.ContentType_RESOURCE, | ||
AssetTypes: []string{ | ||
cloudFunctionsFunction, | ||
bigtableInstanceAssetType, | ||
bigtableTableAssetType, | ||
sqlDatabaseInstanceAssetType, | ||
computeGlobalAddressAssetType, | ||
nodeGroupAssetType, | ||
}, | ||
} | ||
|
||
var results []*assetpb.Asset | ||
|
||
if results == nil { | ||
it := s.client.ListAssets(context.Background(), req) | ||
for { | ||
resource, err := it.Next() | ||
if err == iterator.Done { | ||
break | ||
if cachedResults != nil { | ||
results = cachedResults.([]*assetpb.Asset) | ||
} | ||
|
||
if results == nil { | ||
it := s.client.ListAssets(context.Background(), req) | ||
for { | ||
resource, err := it.Next() | ||
if err == iterator.Done { | ||
break | ||
} | ||
if err != nil && resource != nil{ | ||
erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) | ||
continue | ||
} | ||
if err != nil && resource == nil{ | ||
return nil, err | ||
} | ||
results = append(results, resource) | ||
} | ||
if err != nil { | ||
return nil, err | ||
s.cache.Put(cacheKey, results) | ||
} | ||
|
||
for _, result := range results { | ||
if result.AssetType == ty { | ||
filteredResults = append(filteredResults, result) | ||
} | ||
results = append(results, resource) | ||
} | ||
s.cache.Put(cacheKey, results) | ||
} | ||
|
||
filteredResults := []*assetpb.Asset{} | ||
for _, result := range results { | ||
if result.AssetType == ty { | ||
filteredResults = append(filteredResults, result) | ||
} | ||
if len(erorString) > 0 { | ||
return filteredResults, errors.New(erorString) | ||
} | ||
|
||
return filteredResults, nil | ||
} | ||
|
||
func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) { | ||
req := &assetpb.SearchAllResourcesRequest{ | ||
Scope: fmt.Sprintf("projects/%s", s.config.Project), | ||
AssetTypes: []string{ | ||
storageBucketAssetType, | ||
computeFirewallAssetType, | ||
computeRouterAssetType, | ||
computeInstanceAssetType, | ||
computeNetworkAssetType, | ||
computeSubnetworkAssetType, | ||
dnsManagedZoneAssetType, | ||
computeInstanceGroupAssetType, | ||
bigqueryDatasetAssetType, | ||
bigqueryTableAssetType, | ||
computeAddressAssetType, | ||
computeDiskAssetType, | ||
computeImageAssetType, | ||
healthCheckAssetType, | ||
cloudRunServiceAssetType, | ||
}, | ||
} | ||
var results []*assetpb.ResourceSearchResult | ||
|
||
cacheKey := "SearchAllResources" | ||
cachedResults := s.cache.GetAndLock(cacheKey) | ||
defer s.cache.Unlock(cacheKey) | ||
if cachedResults != nil { | ||
results = cachedResults.([]*assetpb.ResourceSearchResult) | ||
} | ||
filteredResults := []*assetpb.ResourceSearchResult{} | ||
var erorString string | ||
|
||
for _, scope := range s.config.Scopes { | ||
cacheKey := fmt.Sprintf("SearchAllResources_%s", scope) | ||
cachedResults := s.cache.GetAndLock(cacheKey) | ||
defer s.cache.Unlock(cacheKey) | ||
|
||
req := &assetpb.SearchAllResourcesRequest{ | ||
Scope: scope, | ||
AssetTypes: []string{ | ||
storageBucketAssetType, | ||
computeFirewallAssetType, | ||
computeRouterAssetType, | ||
computeInstanceAssetType, | ||
computeNetworkAssetType, | ||
computeSubnetworkAssetType, | ||
dnsManagedZoneAssetType, | ||
computeInstanceGroupAssetType, | ||
bigqueryDatasetAssetType, | ||
bigqueryTableAssetType, | ||
computeAddressAssetType, | ||
computeDiskAssetType, | ||
computeImageAssetType, | ||
healthCheckAssetType, | ||
cloudRunServiceAssetType, | ||
}, | ||
} | ||
var results []*assetpb.ResourceSearchResult | ||
|
||
if cachedResults != nil { | ||
results = cachedResults.([]*assetpb.ResourceSearchResult) | ||
} | ||
|
||
if results == nil { | ||
it := s.client.SearchAllResources(context.Background(), req) | ||
for { | ||
resource, err := it.Next() | ||
if err == iterator.Done { | ||
break | ||
if results == nil { | ||
it := s.client.SearchAllResources(context.Background(), req) | ||
for { | ||
resource, err := it.Next() | ||
if err == iterator.Done { | ||
break | ||
} | ||
if err != nil && resource != nil{ | ||
erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) | ||
continue | ||
} | ||
if err != nil && resource == nil{ | ||
return nil, err | ||
} | ||
results = append(results, resource) | ||
} | ||
if err != nil { | ||
return nil, err | ||
s.cache.Put(cacheKey, results) | ||
} | ||
|
||
for _, result := range results { | ||
if result.AssetType == ty { | ||
filteredResults = append(filteredResults, result) | ||
} | ||
results = append(results, resource) | ||
} | ||
s.cache.Put(cacheKey, results) | ||
} | ||
|
||
filteredResults := []*assetpb.ResourceSearchResult{} | ||
for _, result := range results { | ||
if result.AssetType == ty { | ||
filteredResults = append(filteredResults, result) | ||
} | ||
if len(erorString) > 0 { | ||
return filteredResults, errors.New(erorString) | ||
} | ||
|
||
return filteredResults, nil | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Plea try to avoid else-if statements, there is not reason do use them here