Skip to content

Commit 489a6a4

Browse files
author
idodod
authored
Add new UDC to perform an optimized git deep clone (earthly#22)
* Add new udc to perform an optimized deep clone * move git, ssh, dind under utils
1 parent 4371143 commit 489a6a4

File tree

11 files changed

+194
-75
lines changed

11 files changed

+194
-75
lines changed

‎.github/workflows/ci.yml

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,34 @@ jobs:
3232
mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'"
3333
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
3434
- name: Run tests
35-
run: earthly --ci -P --org earthly-technologies --satellite lib +test
35+
run: |-
36+
earthly config git "{github.com: {auth: ssh, user: git}}" && \
37+
eval $(ssh-agent) && \
38+
earthly secrets --org earthly-technologies --project core get littleredcorvette-id_rsa | ssh-add - && \
39+
earthly --ci -P --org earthly-technologies --satellite lib +test
40+
test-native-arm:
41+
name: +test-native-arm
42+
runs-on: ubuntu-latest
43+
env:
44+
FORCE_COLOR: 1
45+
EARTHLY_CONVERSION_PARALLELISM: "5"
46+
EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}"
47+
EARTHLY_INSTALL_ID: "earthly-lib-githubactions"
48+
steps:
49+
- uses: earthly/actions/setup-earthly@v1
50+
- uses: actions/checkout@v2
51+
- name: Docker mirror login (non fork only)
52+
run: docker login registry-1.docker.io.mirror.corp.earthly.dev --username "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" --password "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}"
53+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
54+
- name: Configure Earthly to use mirror (non fork only)
55+
run: |-
56+
earthly config global.buildkit_additional_config "'[registry.\"docker.io\"]
57+
58+
mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'"
59+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
60+
- name: Run tests
61+
run: |-
62+
earthly config git "{github.com: {auth: ssh, user: git}}" && \
63+
eval $(ssh-agent) && \
64+
earthly secrets --org earthly-technologies --project core get littleredcorvette-id_rsa | ssh-add - && \
65+
earthly --ci -P --org earthly-technologies --satellite lib-arm ./utils/dind+test

‎Earthfile

Lines changed: 2 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -2,56 +2,7 @@ VERSION 0.7
22

33
INSTALL_DIND:
44
COMMAND
5-
COPY +install-dind-script/install-dind.sh /tmp/install-dind.sh
6-
RUN /tmp/install-dind.sh
7-
8-
install-dind-script:
9-
FROM alpine:3.17
10-
COPY ./install-dind.sh ./
11-
SAVE ARTIFACT ./install-dind.sh
5+
DO ./utils/dind+INSTALL
126

137
test:
14-
BUILD +test-install-dind-amd64
15-
BUILD ./ssh+test-add-known-hosts
16-
17-
test-install-dind-amd64:
18-
BUILD --platform=linux/amd64 +test-install-dind-for-image \
19-
--base_image=docker:dind \
20-
--base_image=alpine:latest \
21-
--base_image=debian:stable \
22-
--base_image=debian:stable-slim \
23-
--base_image=debian:oldstable \
24-
--base_image=ubuntu:latest \
25-
--base_image=amazonlinux:1 \
26-
--base_image=amazonlinux:2 \
27-
--base_image=earthly/dind:alpine \
28-
--base_image=earthly/dind:ubuntu
29-
30-
test-install-dind-arm64:
31-
BUILD --platform=linux/arm64 +test-install-dind-for-image \
32-
--base_image=docker:dind \
33-
--base_image=alpine:latest \
34-
--base_image=ubuntu:latest \
35-
--base_image=earthly/dind:alpine \
36-
--base_image=earthly/dind:ubuntu
37-
38-
test-install-dind-for-image:
39-
ARG base_image
40-
FROM "$base_image"
41-
42-
# docker-compose will fail with:
43-
# Error response from daemon: Invalid container name (-hello-1), only [a-zA-Z0-9][a-zA-Z0-9_.-] are allowed
44-
# if docker-compose.yml is in the root directory, due to the --project flag defaulting to "".
45-
# This bug does not exist in the official earthly/dind:alpine image.
46-
WORKDIR /dind-test
47-
48-
DO +INSTALL_DIND
49-
RUN echo "
50-
version: \"3\"
51-
services:
52-
hello:
53-
image: hello-world:latest
54-
" >./docker-compose.yml
55-
WITH DOCKER --compose docker-compose.yml
56-
RUN true
57-
END
8+
BUILD ./utils+test

‎utils/Earthfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
VERSION 0.7
2+
3+
test:
4+
BUILD ./dind+test
5+
BUILD ./git+test
6+
BUILD ./ssh+test

‎utils/dind/Earthfile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
VERSION 0.7
2+
3+
INSTALL:
4+
COMMAND
5+
COPY +install-dind-script/install-dind.sh /tmp/install-dind.sh
6+
RUN /tmp/install-dind.sh
7+
8+
install-dind-script:
9+
FROM alpine:3.18.4
10+
COPY ./install-dind.sh ./
11+
SAVE ARTIFACT ./install-dind.sh
12+
13+
test:
14+
BUILD ./tests+all

‎install-dind.sh renamed to ‎utils/dind/install-dind.sh

File renamed without changes.

‎utils/dind/tests/Earthfile

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
VERSION 0.7
2+
3+
all:
4+
BUILD +test-install-dind-for-image \
5+
--base_image=docker:dind \
6+
--base_image=alpine:latest \
7+
--base_image=debian:stable \
8+
--base_image=debian:stable-slim \
9+
--base_image=debian:oldstable \
10+
--base_image=ubuntu:latest \
11+
--base_image=amazonlinux:1 \
12+
--base_image=amazonlinux:2 \
13+
--base_image=earthly/dind:alpine \
14+
--base_image=earthly/dind:ubuntu
15+
16+
test-install-dind-for-image:
17+
ARG --required base_image
18+
FROM alpine
19+
ARG TARGETPLATFORM
20+
IF [ "${base_image%:*}" = "amazonlinux" ] && [ "$TARGETPLATFORM" = "linux/arm64" ] # no amazonlinux:1 for arm64/UDC not supported atm for amazonlinux:2 on arm - skipping
21+
RUN echo skipping $base_image with platform $TARGETPLATFORM
22+
ELSE
23+
FROM "$base_image"
24+
# docker-compose will fail with:
25+
# Error response from daemon: Invalid container name (-hello-1), only [a-zA-Z0-9][a-zA-Z0-9_.-] are allowed
26+
# if docker-compose.yml is in the root directory, due to the --project flag defaulting to "".
27+
# This bug does not exist in the official earthly/dind:alpine image.
28+
WORKDIR /dind-test
29+
30+
DO ..+INSTALL
31+
RUN echo "
32+
version: \"3\"
33+
services:
34+
hello:
35+
image: hello-world:latest
36+
" >./docker-compose.yml
37+
WITH DOCKER --compose docker-compose.yml
38+
RUN true
39+
END
40+
END

‎utils/git/Earthfile

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
VERSION --arg-scope-and-set 0.7
2+
3+
# DEEP_CLONE deep clones the provided git url
4+
DEEP_CLONE:
5+
COMMAND
6+
ARG USE_GIT_ENV="false"
7+
IF [ $USE_GIT_ENV = "true" ]
8+
FROM alpine/git:v2.40.1
9+
END
10+
DO ../ssh+ADD_KNOWN_HOSTS
11+
ARG --required GIT_URL
12+
ARG DEST_DIR
13+
LET dest_dir=$DEST_DIR
14+
IF [ -z $dest_dir ]
15+
SET dest_dir=$(basename ${GIT_URL%.git})
16+
END
17+
GIT CLONE $GIT_URL $dest_dir
18+
WORKDIR $dest_dir
19+
RUN git remote set-url origin $GIT_URL
20+
ARG git_hash=$(git rev-parse HEAD)
21+
ARG SECRET_PATH
22+
IF [ -z $SECRET_PATH ]
23+
RUN --ssh git fetch --unshallow
24+
ELSE
25+
RUN --mount=type=secret,id=$SECRET_PATH,mode=0400,target=/root/.ssh/id_rsa \
26+
git fetch --unshallow
27+
END
28+
29+
test:
30+
BUILD ./tests+all

‎utils/git/tests/Earthfile

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
VERSION --arg-scope-and-set --pass-args 0.7
2+
3+
PROJECT earthly-technologies/core
4+
5+
all:
6+
BUILD --platform=linux/amd64 --platform=linux/arm64 +test-deep-clone-image \
7+
--base_image=alpine/git:latest \
8+
--base_image=alpine:latest \
9+
--base_image=debian:stable \
10+
--base_image=debian:stable-slim \
11+
--base_image=ubuntu:latest \
12+
--base_image=amazonlinux:1 \
13+
--base_image=amazonlinux:2 \
14+
--GIT_URL=https://github.com/earthly/hello-world.git \
15+
--GIT_URL=[email protected]:earthly/hello-world.git \
16+
--GIT_URL=[email protected]:earthly/hello-world \
17+
--SECRET_PATH="" \
18+
--SECRET_PATH="littleredcorvette-id_rsa" \
19+
--DEST_DIR="" \
20+
--DEST_DIR="some-other-dir"
21+
22+
test-deep-clone-image:
23+
ARG --required base_image
24+
ARG TARGETPLATFORM
25+
FROM alpine
26+
IF [ "$base_image" = "amazonlinux:1" ] && [ "$TARGETPLATFORM" = "linux/arm64" ] # no amazonlinux:1 for arm64, skipping
27+
RUN echo skipping $base_image with platform $TARGETPLATFORM
28+
ELSE
29+
FROM "$base_image"
30+
LET use_git_env="true"
31+
IF [ "${base_image%:*}" = "alpine/git" ]
32+
SET use_git_env="false"
33+
RUN apk add git
34+
ELSE IF [ "${base_image%:*}" = "alpine" ]
35+
RUN apk add git
36+
ELSE IF [ "${base_image%:*}" = "debian" ]
37+
RUN apt update && apt install -y git
38+
ELSE IF [ "${base_image%:*}" = "ubuntu" ]
39+
RUN apt-get update && apt-get -y install git
40+
ELSE IF [[ $base_image == amazonlinux* ]]
41+
RUN yum -y install git
42+
END
43+
DO --pass-args ..+DEEP_CLONE --USE_GIT_ENV=$use_git_env
44+
IF [ -n $DEST_DIR ]
45+
WORKDIR $DEST_DIR
46+
END
47+
RUN git checkout main
48+
END

‎utils/ssh/Earthfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
VERSION 0.7
2+
3+
# ADD_KNOWN_HOSTS appends some known public keys into the $target_file
4+
ADD_KNOWN_HOSTS:
5+
COMMAND
6+
ARG target_file=~/.ssh/known_hosts
7+
LET expanded_target_file="$(eval echo $target_file)"
8+
COPY +known-hosts/known_hosts /tmp/.
9+
RUN mkdir -p $(dirname $expanded_target_file) && cat /tmp/known_hosts >> $expanded_target_file
10+
RUN rm /tmp/known_hosts
11+
12+
# known-hosts is used to copy the known_hosts file into the build context (internal use)
13+
known-hosts:
14+
FROM alpine:3.18.4
15+
COPY known_hosts .
16+
SAVE ARTIFACT known_hosts
17+
18+
test:
19+
BUILD ./tests+all

‎ssh/known_hosts renamed to ‎utils/ssh/known_hosts

File renamed without changes.

0 commit comments

Comments
 (0)