Skip to content

Commit 6053d2b

Browse files
matipanmarcosnils
matipan
and
marcosnils
authored
Support pipeline-generation based CI
This is the first step towards making pocketci work using the pipeline generation concept. In this concept users use SDKs provided by us (or ad hoc) that match with vendor's events. This SDKs generate JSON dagger.File that hold the list of pipelines the user specified. Pocketci will then match those pipelines with the incoming event. Currently only GitHub is supported. Signed-off-by: Marcos Lilljedahl <[email protected]> Signed-off-by: Matias Pan <[email protected]> Co-authored-by: Marcos Lilljedahl <[email protected]>
1 parent fec1626 commit 6053d2b

29 files changed

+1331
-401
lines changed

‎ci/dagger.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,15 @@
22
"name": "ci",
33
"sdk": "go",
44
"dependencies": [
5+
{
6+
"name": "gha",
7+
"source": "github.com/franela/pocketci/modules/gha@1cae77909b2ad0ace74301545a39d50d8af431b7"
8+
},
59
{
610
"name": "pocketci",
711
"source": "github.com/franela/pocketci/modules/pocketci@f184045d17421024adf4b47b3e90e51e3962332d"
812
}
913
],
1014
"source": ".",
11-
"engineVersion": "v0.13.3"
15+
"engineVersion": "v0.13.5"
1216
}

‎ci/main.go

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,32 @@ func (m *Ci) Publish(ctx context.Context, src *dagger.Directory, tag, username s
2020
Publish(ctx, "ghcr.io/franela/pocketci:"+tag)
2121
}
2222

23-
func (m *Ci) Test(ctx context.Context, src *dagger.Directory, ghUsername, ghPassword *dagger.Secret) *dagger.Container {
23+
func (m *Ci) Test(ctx context.Context,
24+
// +defaultPath="../"
25+
src *dagger.Directory,
26+
// +optional
27+
ghUsername *dagger.Secret,
28+
// +optional
29+
ghPassword *dagger.Secret,
30+
) *dagger.Container {
2431
return m.base(src).
25-
WithSecretVariable("GH_USERNAME", ghUsername).
26-
WithSecretVariable("GH_PASSWORD", ghPassword).
32+
With(func(c *dagger.Container) *dagger.Container {
33+
if ghUsername != nil {
34+
c = c.WithSecretVariable("GH_USERNAME", ghUsername)
35+
}
36+
if ghPassword != nil {
37+
c = c.WithSecretVariable("GH_PASSWORD", ghPassword)
38+
}
39+
return c
40+
}).
2741
WithExec([]string{"sh", "-c", "go test -v ./pocketci/..."}, dagger.ContainerWithExecOpts{ExperimentalPrivilegedNesting: true})
2842
}
2943

30-
func (m *Ci) Lint(ctx context.Context, src *dagger.Directory) *dagger.Container {
44+
// testing
45+
46+
func (m *Ci) Lint(ctx context.Context,
47+
// +defaultPath="../"
48+
src *dagger.Directory) *dagger.Container {
3149
return dag.Container().
3250
From("golangci/golangci-lint:"+GolangLintVersion).
3351
WithMountedDirectory("/app", src).

‎ci/pocketci.go

Lines changed: 19 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -5,70 +5,23 @@ import (
55
"dagger/ci/internal/dagger"
66
)
77

8-
func (m *Ci) TestOnGithubPullRequest(ctx context.Context,
9-
// +optional
10-
// +default="**/**.go,go.*"
11-
onChanges string,
12-
// +default="synchronize,opened,reopened"
13-
filter string,
14-
src *dagger.Directory,
15-
eventTrigger *dagger.File,
16-
ghUsername, ghPassword *dagger.Secret) error {
17-
_, err := m.Test(ctx, src, ghUsername, ghPassword).Stdout(ctx)
18-
return err
19-
}
20-
21-
func (m *Ci) LintOnGithubPullRequest(ctx context.Context,
22-
// +optional
23-
// +default="**/**.go,go.*"
24-
onChanges string,
25-
// +default="synchronize,opened,reopened"
26-
filter string,
27-
src *dagger.Directory,
28-
eventTrigger *dagger.File,
29-
ghUsername, ghPassword *dagger.Secret) error {
30-
_, err := m.Lint(ctx, src).Stdout(ctx)
31-
return err
32-
}
33-
34-
func (m *Ci) PublishOnGithubPush(ctx context.Context,
35-
// +optional
36-
// +default="**/**.go,go.*"
37-
onChanges string,
38-
// +default="main"
39-
filter string,
40-
src *dagger.Directory,
41-
eventTrigger *dagger.File,
42-
ghUsername, ghPassword *dagger.Secret) error {
43-
44-
sha, err := dag.Pocketci(eventTrigger).CommitPush().Sha(ctx)
45-
if err != nil {
46-
return err
47-
}
48-
49-
username, _ := ghUsername.Plaintext(ctx)
50-
_, err = m.Publish(ctx, src, sha, username, ghPassword)
51-
return err
52-
}
53-
54-
func (m *Ci) TestOnGithubPushMain(ctx context.Context,
55-
// +optional
56-
// +default="**/**.go,go.*"
57-
onChanges string,
58-
src *dagger.Directory,
59-
eventTrigger *dagger.File,
60-
ghUsername, ghPassword *dagger.Secret) error {
61-
_, err := m.Test(ctx, src, ghUsername, ghPassword).Stdout(ctx)
62-
return err
63-
}
64-
65-
func (m *Ci) LintOnGithubPushMain(ctx context.Context,
66-
// +optional
67-
// +default="**/**.go,go.*"
68-
onChanges string,
69-
src *dagger.Directory,
70-
eventTrigger *dagger.File,
71-
ghUsername, ghPassword *dagger.Secret) error {
72-
_, err := m.Lint(ctx, src).Stdout(ctx)
73-
return err
8+
func (m *Ci) Pipelines(ctx context.Context) *dagger.File {
9+
changes := []string{"**/**.go", "go.*"}
10+
branches := []string{"main"}
11+
12+
checks := dag.Gha().WithPipeline("checks").
13+
WithOnChanges(changes).
14+
WithOnPullRequest([]dagger.GhaAction{dagger.Opened, dagger.Synchronize, dagger.Reopened}).
15+
WithOnPush(branches).
16+
WithModule("ci").
17+
Call("test & lint")
18+
19+
publish := dag.Gha().WithPipeline("publish").
20+
WithOnChanges(changes).
21+
WithOnPush([]string{"main"}).
22+
WithModule("ci").
23+
Call("publish --sha env:COMMIT_SHA --username env:GH_USERNAME --password env:GH_PASSWORD").
24+
After([]*dagger.GhaPipeline{checks})
25+
26+
return dag.Gha().Pipelines([]*dagger.GhaPipeline{checks, publish})
7427
}

‎cmd/agent/main.go

Lines changed: 144 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,173 @@
11
package main
22

33
import (
4+
"bytes"
45
"context"
6+
"encoding/json"
7+
"errors"
58
"flag"
6-
"io"
9+
"fmt"
10+
"log"
711
"log/slog"
812
"net/http"
913
"os"
14+
"strconv"
15+
"time"
1016

1117
"dagger.io/dagger"
1218
"github.com/franela/pocketci/pocketci"
1319
)
1420

15-
var verbose = flag.Bool("verbose", false, "whether to enable verbose output")
21+
var (
22+
controlPlane = flag.String("control-plane", "", "url to control plane host")
23+
interval = flag.Duration("interval", 5*time.Second, "interval between pipeline polls")
24+
runnerName = flag.String("runner-name", "", "name of the runner that identifies it")
25+
parallelism = flag.Int("parallelism", 10, "max number of dagger calls to run in parallel")
26+
27+
ErrNoPipeline = errors.New("no pipeline to run")
28+
)
1629

1730
func main() {
1831
flag.Parse()
1932

33+
if *controlPlane == "" {
34+
log.Fatalf("control-plane must be specified and be a valid url")
35+
}
36+
if *runnerName == "" {
37+
log.Fatalf("runner-name must be specified")
38+
}
39+
2040
ctx := context.Background()
21-
out := io.Discard
22-
if *verbose {
23-
out = os.Stderr
41+
42+
client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
43+
if err != nil {
44+
log.Fatalf("failed to connect to dagger client: %s", err)
45+
}
46+
47+
mu := make(chan bool, *parallelism)
48+
for i := 0; i < *parallelism; i++ {
49+
mu <- true
2450
}
25-
client, err := dagger.Connect(ctx, dagger.WithLogOutput(out))
51+
52+
githubUser := os.Getenv("GITHUB_USERNAME")
53+
githubPass := os.Getenv("GITHUB_TOKEN")
54+
netrc := client.SetSecret("github_auth", fmt.Sprintf("machine github.com login %s password %s", githubUser, githubPass))
55+
56+
for {
57+
pipeline, err := getPipeline(ctx)
58+
if err != nil && !errors.Is(err, ErrNoPipeline) {
59+
log.Fatalf("failed to fetch pipeline: %s", err)
60+
}
61+
62+
if errors.Is(err, ErrNoPipeline) {
63+
slog.Info("no pipeline to run")
64+
time.Sleep(*interval)
65+
continue
66+
}
67+
68+
go func() {
69+
// wait for parallelism
70+
<-mu
71+
defer func() {
72+
mu <- true
73+
}()
74+
75+
run(ctx, client, netrc, pipeline)
76+
pipelineDone(pipeline)
77+
}()
78+
79+
time.Sleep(*interval)
80+
}
81+
}
82+
83+
func pipelineDone(pipeline *pocketci.PocketciPipeline) {
84+
res, err := http.Post(*controlPlane+"/pipelines/"+strconv.Itoa(pipeline.ID), "application/json", nil)
2685
if err != nil {
27-
slog.Error("failed to connect to dagger", slog.String("error", err.Error()))
86+
slog.Error("could not mark pipeline as done", slog.String("error", err.Error()))
87+
return
2888
}
29-
defer client.Close()
89+
defer res.Body.Close()
3090

31-
server, err := pocketci.NewServer(client, pocketci.ServerOptions{
32-
GithubUsername: os.Getenv("GITHUB_USERNAME"),
33-
GithubPassword: os.Getenv("GITHUB_TOKEN"),
34-
GithubSignature: os.Getenv("X_HUB_SIGNATURE"),
35-
})
91+
if res.StatusCode != http.StatusNoContent {
92+
slog.Error("could not mark pipeline as done", slog.Int("status_code", res.StatusCode))
93+
}
94+
slog.Info("pipeline is done", slog.Int("pipeline", pipeline.ID))
95+
}
96+
97+
func getPipeline(ctx context.Context) (*pocketci.PocketciPipeline, error) {
98+
buf := bytes.NewBuffer([]byte{})
99+
if err := json.NewEncoder(buf).Encode(pocketci.PipelineClaimRequest{RunnerName: *runnerName}); err != nil {
100+
return nil, err
101+
}
102+
103+
res, err := http.Post(*controlPlane+"/pipelines/claim", "application/json", buf)
36104
if err != nil {
37-
slog.Error("failed to create pocketci server", slog.String("error", err.Error()))
105+
return nil, err
106+
}
107+
if res.StatusCode == http.StatusNoContent {
108+
return nil, ErrNoPipeline
38109
}
39110

40-
mux := http.NewServeMux()
41-
mux.Handle("/", server)
42-
srv := &http.Server{
43-
Addr: ":8080",
44-
Handler: mux,
111+
pipeline := &pocketci.PocketciPipeline{}
112+
if err := json.NewDecoder(res.Body).Decode(pipeline); err != nil {
113+
return nil, err
45114
}
46115

47-
slog.Info("starting pocketci at 8080")
48-
if err = srv.ListenAndServe(); err != nil {
49-
slog.Error("server exited", slog.String("error", err.Error()))
116+
return pipeline, nil
117+
}
118+
119+
func run(ctx context.Context, dag *dagger.Client, netrc *dagger.Secret, req *pocketci.PocketciPipeline) {
120+
repoUrl := "https://github.com/" + req.Repository
121+
slog.Info("cloning repository", slog.String("repository", repoUrl),
122+
slog.String("ref", req.GitInfo.Branch), slog.String("sha", req.GitInfo.SHA))
123+
124+
repo, err := pocketci.BaseContainer(dag).
125+
WithEnvVariable("CACHE_BUST", time.Now().String()).
126+
WithMountedSecret("/root/.netrc", netrc).
127+
WithExec([]string{"git", "clone", "--single-branch", "--branch", req.GitInfo.Branch, "--depth", "1", repoUrl, "/app"}).
128+
WithWorkdir("/app").
129+
WithExec([]string{"git", "checkout", req.GitInfo.SHA}).
130+
Directory("/app").
131+
Sync(ctx)
132+
if err != nil {
133+
slog.Error("failed to clonse github repository", slog.String("error", err.Error()),
134+
slog.String("repository", repoUrl), slog.String("ref", req.GitInfo.Branch), slog.String("sha", req.GitInfo.SHA))
135+
return
136+
}
137+
138+
vars := map[string]string{
139+
"GITHUB_SHA": req.GitInfo.SHA,
140+
"GITHUB_ACTIONS": "true",
141+
}
142+
143+
slog.Info("launching pocketci agent container",
144+
slog.String("repository_name", req.Repository), slog.String("pipeline", req.Name),
145+
slog.String("ref", req.GitInfo.Branch), slog.String("sha", req.GitInfo.SHA),
146+
slog.String("module", req.Module), slog.String("exec", req.Call),
147+
slog.String("runs_on", req.Runner))
148+
149+
call := fmt.Sprintf("dagger call --progress plain %s", req.Call)
150+
if req.Module != "" && req.Module != "." {
151+
call = fmt.Sprintf("dagger call -m %s --progress plain %s", req.Module, req.Call)
152+
}
153+
stdout, err := pocketci.AgentContainer(dag).
154+
WithEnvVariable("CACHE_BUST", time.Now().String()).
155+
WithEnvVariable("DAGGER_CLOUD_TOKEN", os.Getenv("DAGGER_CLOUD_TOKEN")).
156+
WithDirectory("/app", repo).
157+
WithWorkdir("/app").
158+
WithEnvVariable("CI", "pocketci").
159+
With(func(c *dagger.Container) *dagger.Container {
160+
for key, val := range vars {
161+
c = c.WithEnvVariable(key, val)
162+
}
163+
script := fmt.Sprintf("unset TRACEPARENT;unset OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf;unset OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:38015;unset OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf;unset OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:38015/v1/traces;unset OTEL_EXPORTER_OTLP_TRACES_LIVE=1;unset OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf;unset OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:38015/v1/logs;unset OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf;unset OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://127.0.0.1:38015/v1/metrics; %s", call)
164+
return c.WithExec([]string{"sh", "-c", script}, dagger.ContainerWithExecOpts{
165+
ExperimentalPrivilegedNesting: true,
166+
})
167+
}).
168+
Stdout(ctx)
169+
if err != nil {
170+
return
50171
}
172+
fmt.Println(stdout)
51173
}

‎cmd/server/main.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"io"
7+
"log/slog"
8+
"net/http"
9+
"os"
10+
11+
"dagger.io/dagger"
12+
"github.com/franela/pocketci/pocketci"
13+
)
14+
15+
var verbose = flag.Bool("verbose", false, "whether to enable verbose output")
16+
17+
func main() {
18+
flag.Parse()
19+
20+
ctx := context.Background()
21+
out := io.Discard
22+
if *verbose {
23+
out = os.Stderr
24+
}
25+
client, err := dagger.Connect(ctx, dagger.WithLogOutput(out))
26+
if err != nil {
27+
slog.Error("failed to connect to dagger", slog.String("error", err.Error()))
28+
}
29+
defer client.Close()
30+
31+
server, err := pocketci.NewServer(client, pocketci.ServerOptions{
32+
GithubUsername: os.Getenv("GITHUB_USERNAME"),
33+
GithubPassword: os.Getenv("GITHUB_TOKEN"),
34+
GithubSignature: os.Getenv("X_HUB_SIGNATURE"),
35+
})
36+
if err != nil {
37+
slog.Error("failed to create pocketci server", slog.String("error", err.Error()))
38+
}
39+
40+
mux := http.NewServeMux()
41+
mux.Handle("/", server)
42+
mux.HandleFunc("POST /pipelines/{pipeline_id}", server.PipelineDoneHandler)
43+
mux.HandleFunc("POST /pipelines/claim", server.PipelineClaimHandler)
44+
srv := &http.Server{
45+
Addr: ":8080",
46+
Handler: mux,
47+
}
48+
49+
slog.Info("starting pocketci at 8080")
50+
if err = srv.ListenAndServe(); err != nil {
51+
slog.Error("server exited", slog.String("error", err.Error()))
52+
}
53+
}

0 commit comments

Comments
 (0)