Skip to content
This repository was archived by the owner on Jul 19, 2023. It is now read-only.

Commit 1c480ff

Browse files
dimabrudimabru
authored
feat: support fetch with url (#103)
1 parent e682684 commit 1c480ff

10 files changed

Lines changed: 185 additions & 56 deletions

File tree

‎.vscode/launch.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,22 @@
1212
"showLog": true
1313
},
1414
{
15-
"name": "Fetch",
15+
"name": "Fetch Github",
1616
"type": "go",
1717
"request": "launch",
1818
"mode": "debug",
1919
"program": "${workspaceFolder}/main.go",
2020
"env": {},
2121
"args": ["fetch", "github", "allero-io"],
22+
},
23+
{
24+
"name": "Fetch",
25+
"type": "go",
26+
"request": "launch",
27+
"mode": "debug",
28+
"program": "${workspaceFolder}/main.go",
29+
"env": {},
30+
"args": ["fetch", "https://github.com/allero-io/allero", "https://gitlab.com/allero"],
2231
}
2332
]
2433
}

‎Makefile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@ clear-token:
2525
create-bin:
2626
goreleaser --snapshot --skip-publish --rm-dist
2727

28+
fetch:
29+
go run main.go fetch https://github.com/allero-io/allero https://gitlab.com/allero
30+
2831
github:
29-
# go run main.go fetch github supran2811/familyApp
30-
go run main.go fetch github curbengh/hexo-yam
32+
go run main.go fetch github supran2811/familyApp
3133

3234
gitlab:
3335
# go run main.go fetch gitlab GitLab-examples/clojure-web-application
34-
go run main.go fetch gitlab allero/demo
36+
go run main.go fetch gitlab allero

‎cmd/fetch/github.go‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,17 @@ package fetch
22

33
import (
44
"github.com/allero-io/allero/pkg/clients"
5+
"github.com/allero-io/allero/pkg/connectors"
56
githubConnector "github.com/allero-io/allero/pkg/connectors/github"
67
"github.com/google/go-github/github"
78
"github.com/spf13/cobra"
89
)
910

1011
type FetchGithubDependencies struct {
11-
GithubClient *github.Client
12+
GithubClient *github.Client
13+
OwnersWithRepos []*connectors.OwnerWithRepo
1214
}
1315

14-
var GITHUB_TOKEN string
15-
1616
func NewGithubCommand(deps *FetchCommandDependencies) *cobra.Command {
1717
githubCmd := &cobra.Command{
1818
Use: "github org/repo...",
@@ -30,17 +30,19 @@ func NewGithubCommand(deps *FetchCommandDependencies) *cobra.Command {
3030
deps.PosthogClient.PublishEventWithArgs("data fetched", args)
3131
},
3232
RunE: func(cmd *cobra.Command, args []string) error {
33-
GITHUB_TOKEN = deps.ConfigurationManager.GetGithubToken()
34-
githubClient := clients.CreateGithubClient(GITHUB_TOKEN)
33+
ownersWithRepos := connectors.SplitParentRepo(args)
34+
githubClient := clients.CreateGithubClient(*deps.ConfigurationManager)
3535

3636
fetchGithubDeps := &FetchGithubDependencies{
37-
GithubClient: githubClient,
37+
GithubClient: githubClient,
38+
OwnersWithRepos: ownersWithRepos,
3839
}
3940

40-
return executeGithub(fetchGithubDeps, args)
41+
return executeGithub(fetchGithubDeps)
4142
},
4243
PostRun: func(cmd *cobra.Command, args []string) {
43-
tokenWasProvided := GITHUB_TOKEN != ""
44+
githubToken := deps.ConfigurationManager.GetGithubToken()
45+
tokenWasProvided := githubToken != ""
4446

4547
analyticsArgs := make(map[string]any)
4648
analyticsArgs["Total Fetched Repos"] = reposFetchCounter
@@ -52,10 +54,13 @@ func NewGithubCommand(deps *FetchCommandDependencies) *cobra.Command {
5254
return githubCmd
5355
}
5456

55-
func executeGithub(deps *FetchGithubDependencies, args []string) error {
57+
func executeGithub(deps *FetchGithubDependencies) error {
5658
githubConnectorDeps := &githubConnector.GithubConnectorDependencies{Client: deps.GithubClient}
5759
githubConnector := githubConnector.New(githubConnectorDeps)
58-
reposFetchCounter, err = githubConnector.Get(args)
60+
61+
var err error
62+
reposFetchCounter, err = githubConnector.Get(deps.OwnersWithRepos)
63+
5964
if err != nil {
6065
return err
6166
}

‎cmd/fetch/gitlab.go‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
package fetch
22

33
import (
4+
"github.com/allero-io/allero/pkg/clients"
5+
"github.com/allero-io/allero/pkg/connectors"
46
gitlabConnector "github.com/allero-io/allero/pkg/connectors/gitlab"
57
"github.com/spf13/cobra"
68
"github.com/xanzy/go-gitlab"
79
)
810

911
type FetchGitlabDependencies struct {
10-
GitlabClient *gitlab.Client
12+
GitlabClient *gitlab.Client
13+
OwnersWithRepos []*connectors.OwnerWithRepo
1114
}
1215

13-
var GITLAB_TOKEN string
14-
1516
func NewGitlabCommand(deps *FetchCommandDependencies) *cobra.Command {
1617
githubCmd := &cobra.Command{
1718
Use: "gitlab org/repo...",
@@ -29,19 +30,20 @@ func NewGitlabCommand(deps *FetchCommandDependencies) *cobra.Command {
2930
deps.PosthogClient.PublishEventWithArgs("data fetched", args)
3031
},
3132
RunE: func(cmd *cobra.Command, args []string) error {
32-
GITLAB_TOKEN = deps.ConfigurationManager.GetGitlabToken()
33-
gitlabClient, err := gitlab.NewClient(GITLAB_TOKEN)
33+
ownersWithRepos := connectors.SplitParentRepo(args)
34+
gitlabClient, err := clients.CreateGitlabClient(*deps.ConfigurationManager)
3435
if err != nil {
3536
return err
3637
}
3738

3839
fetchGitlabDeps := &FetchGitlabDependencies{
39-
GitlabClient: gitlabClient,
40+
GitlabClient: gitlabClient,
41+
OwnersWithRepos: ownersWithRepos,
4042
}
41-
return executeGitlab(fetchGitlabDeps, args)
43+
return executeGitlab(fetchGitlabDeps)
4244
},
4345
PostRun: func(cmd *cobra.Command, args []string) {
44-
tokenWasProvided := GITHUB_TOKEN != ""
46+
tokenWasProvided := deps.ConfigurationManager.GetGitlabToken()
4547

4648
analyticsArgs := make(map[string]any)
4749
analyticsArgs["Total Fetched Repos"] = reposFetchCounter
@@ -53,10 +55,11 @@ func NewGitlabCommand(deps *FetchCommandDependencies) *cobra.Command {
5355
return githubCmd
5456
}
5557

56-
func executeGitlab(deps *FetchGitlabDependencies, args []string) error {
58+
func executeGitlab(deps *FetchGitlabDependencies) error {
5759
gitlabConnectorDeps := &gitlabConnector.GitlabConnectorDependencies{Client: deps.GitlabClient}
5860
gitlabConnector := gitlabConnector.New(gitlabConnectorDeps)
5961

60-
reposFetchCounter, err = gitlabConnector.Get(args)
62+
var err error
63+
reposFetchCounter, err = gitlabConnector.Get(deps.OwnersWithRepos)
6164
return err
6265
}

‎cmd/fetch/main.go‎

Lines changed: 88 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,111 @@
11
package fetch
22

33
import (
4+
"strings"
5+
6+
"github.com/allero-io/allero/pkg/clients"
47
"github.com/allero-io/allero/pkg/configurationManager"
8+
"github.com/allero-io/allero/pkg/connectors"
9+
githubConnector "github.com/allero-io/allero/pkg/connectors/github"
10+
gitlabConnector "github.com/allero-io/allero/pkg/connectors/gitlab"
511
"github.com/allero-io/allero/pkg/posthog"
612
"github.com/spf13/cobra"
713
)
814

915
var (
1016
reposFetchCounter int
11-
err error
1217
)
1318

19+
var scmPrefixes = map[string]string{
20+
"github": "https://github.com/",
21+
"gitlab": "https://gitlab.com/",
22+
}
23+
1424
type FetchCommandDependencies struct {
1525
ConfigurationManager *configurationManager.ConfigurationManager
1626
PosthogClient *posthog.PosthogClient
1727
}
1828

19-
var fetchCmd = &cobra.Command{
20-
Use: "fetch",
21-
Short: "Fetch data of repositories",
22-
Long: "Fetch data of repositories and entire organizations from a specified SCM platform",
23-
Example: `allero fetch github allero-io dapr/dapr`,
24-
}
25-
2629
func New(deps *FetchCommandDependencies) *cobra.Command {
30+
fetchCmd := &cobra.Command{
31+
Use: "fetch",
32+
Short: "Fetch data of repositories",
33+
Long: "Fetch data of repositories and entire organizations from several SCM platforms",
34+
Example: `allero fetch https://github.com/allero-io/allero`,
35+
RunE: func(cmd *cobra.Command, args []string) error {
36+
return executeFetch(deps, args)
37+
},
38+
}
39+
2740
fetchCmd.AddCommand(NewGithubCommand(deps))
2841
fetchCmd.AddCommand(NewGitlabCommand(deps))
2942

3043
return fetchCmd
3144
}
45+
46+
func executeFetch(deps *FetchCommandDependencies, args []string) error {
47+
scmMapping := getAllOwnersWithRepo(args)
48+
var err error
49+
50+
for scm, ownersWithRepos := range scmMapping {
51+
switch scm {
52+
53+
case "github":
54+
githubConnectorDeps := &githubConnector.GithubConnectorDependencies{Client: clients.CreateGithubClient(*deps.ConfigurationManager)}
55+
githubConnector := githubConnector.New(githubConnectorDeps)
56+
57+
reposFetchCounter, err = githubConnector.Get(ownersWithRepos)
58+
if err != nil {
59+
return err
60+
}
61+
62+
case "gitlab":
63+
gitlabClient, err := clients.CreateGitlabClient(*deps.ConfigurationManager)
64+
if err != nil {
65+
return err
66+
}
67+
68+
gitlabConnectorDeps := &gitlabConnector.GitlabConnectorDependencies{Client: gitlabClient}
69+
gitlabConnector := gitlabConnector.New(gitlabConnectorDeps)
70+
71+
reposFetchCounter, err = gitlabConnector.Get(ownersWithRepos)
72+
if err != nil {
73+
return err
74+
}
75+
}
76+
}
77+
78+
return nil
79+
}
80+
81+
func getAllOwnersWithRepo(args []string) map[string][]*connectors.OwnerWithRepo {
82+
scmToArgs := getArgsPerScm(args)
83+
scmMapping := getReposPerScm(scmToArgs)
84+
85+
return scmMapping
86+
}
87+
88+
func getArgsPerScm(args []string) map[string][]string {
89+
scmToArgs := make(map[string][]string)
90+
for _, arg := range args {
91+
for scm, prefix := range scmPrefixes {
92+
if strings.HasPrefix(arg, prefix) {
93+
trimmedArg := strings.TrimPrefix(arg, prefix)
94+
scmToArgs[scm] = append(scmToArgs[scm], trimmedArg)
95+
}
96+
}
97+
}
98+
99+
return scmToArgs
100+
}
101+
102+
func getReposPerScm(scmToArgs map[string][]string) map[string][]*connectors.OwnerWithRepo {
103+
scmMapping := make(map[string][]*connectors.OwnerWithRepo)
104+
105+
for scm, _ := range scmToArgs {
106+
ownersWithRepos := connectors.SplitParentRepo(scmToArgs[scm])
107+
scmMapping[scm] = ownersWithRepos
108+
}
109+
110+
return scmMapping
111+
}

‎pkg/clients/githubClient.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ import (
44
"context"
55
"net/http"
66

7+
"github.com/allero-io/allero/pkg/configurationManager"
78
"github.com/google/go-github/github"
89
"golang.org/x/oauth2"
910
)
1011

11-
func CreateGithubClient(githubToken string) *github.Client {
12+
func CreateGithubClient(configurationManager configurationManager.ConfigurationManager) *github.Client {
13+
githubToken := configurationManager.GetGithubToken()
1214
httpClient := &http.Client{}
1315

1416
if githubToken != "" {

‎pkg/clients/gitlabClient.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package clients
2+
3+
import (
4+
"github.com/allero-io/allero/pkg/configurationManager"
5+
"github.com/xanzy/go-gitlab"
6+
)
7+
8+
func CreateGitlabClient(configurationManager configurationManager.ConfigurationManager) (*gitlab.Client, error) {
9+
GITLAB_TOKEN := configurationManager.GetGitlabToken()
10+
gitlabClient, err := gitlab.NewClient(GITLAB_TOKEN)
11+
if err != nil {
12+
return nil, err
13+
}
14+
15+
return gitlabClient, nil
16+
}

‎pkg/connectors/github/githubConnector.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ func New(deps *GithubConnectorDependencies) *GithubConnector {
3232
}
3333
}
3434

35-
func (gc *GithubConnector) Get(args []string) (int, error) {
36-
repositoriesChan := gc.getAllRepositories(args)
35+
func (gc *GithubConnector) Get(ownersWithRepos []*connectors.OwnerWithRepo) (int, error) {
36+
repositoriesChan := gc.getAllRepositories(ownersWithRepos)
3737

3838
githubJsonObject := make(map[string]*GithubOwner)
3939
reposFetchCounter := 0
@@ -200,32 +200,32 @@ func (gc *GithubConnector) matchedFiles(tree *github.Tree, regex string) []strin
200200
return matchedFiles
201201
}
202202

203-
func (gc *GithubConnector) getAllRepositories(args []string) chan *GithubRepositoryApiResponse {
203+
func (gc *GithubConnector) getAllRepositories(ownersWithRepos []*connectors.OwnerWithRepo) chan *GithubRepositoryApiResponse {
204204
repositoriesChan := make(chan *GithubRepositoryApiResponse)
205205

206206
go func() {
207207
defer close(repositoriesChan)
208208

209-
for _, arg := range args {
210-
ownerWithRepo := connectors.SplitParentRepo(arg)
209+
for _, ownerWithRepo := range ownersWithRepos {
210+
fullName := ownerWithRepo.Owner + "/" + ownerWithRepo.Repo
211211

212212
if ownerWithRepo.Repo != "" {
213-
fmt.Printf("Start fetching repository %s/%s\n", ownerWithRepo.Owner, ownerWithRepo.Repo)
213+
fmt.Printf("Start fetching github repository %s/%s\n", ownerWithRepo.Owner, ownerWithRepo.Repo)
214214
repoMetadata, _, err := gc.client.Repositories.Get(context.Background(), ownerWithRepo.Owner, ownerWithRepo.Repo)
215215
if err != nil {
216-
err = fmt.Errorf("unable to get repository %s", arg)
216+
err = fmt.Errorf("unable to get github repository %s", fullName)
217217
}
218218

219219
repositoriesChan <- &GithubRepositoryApiResponse{
220220
Repository: repoMetadata,
221221
Error: err,
222222
}
223-
fmt.Printf("Finished fetching repository %s/%s\n", ownerWithRepo.Owner, ownerWithRepo.Repo)
223+
fmt.Printf("Finished fetching github repository %s/%s\n", ownerWithRepo.Owner, ownerWithRepo.Repo)
224224

225225
} else {
226226
ownerType, err := gc.getGithubOwnerType(ownerWithRepo.Owner)
227227
if err != nil {
228-
err = fmt.Errorf("unable to get data on owner %s", ownerWithRepo.Owner)
228+
err = fmt.Errorf("unable to get data on github owner %s", ownerWithRepo.Owner)
229229

230230
repositoriesChan <- &GithubRepositoryApiResponse{
231231
Repository: nil,
@@ -234,7 +234,7 @@ func (gc *GithubConnector) getAllRepositories(args []string) chan *GithubReposit
234234
} else {
235235
ownerRepos, err := ListByOwnerWithPagination(gc.client, ownerWithRepo.Owner, ownerType)
236236
if err != nil {
237-
err = fmt.Errorf("unable to get repositories from owner %s", ownerWithRepo.Owner)
237+
err = fmt.Errorf("unable to get repositories from github owner %s", ownerWithRepo.Owner)
238238
}
239239

240240
for repo := range ownerRepos {

0 commit comments

Comments
 (0)