Add new UDC to perform an optimized git deep clone

.github/workflows/ci.yml

@@ -32,4 +32,34 @@ jobs:
           mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'"
         if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
       - name: Run tests
-        run: earthly --ci -P --org earthly-technologies --satellite lib +test
+        run: |-
+          earthly config git "{github.com: {auth: ssh, user: git}}" && \
+          eval $(ssh-agent) && \
+          earthly secrets --org earthly-technologies --project core get littleredcorvette-id_rsa | ssh-add - && \
+          earthly --ci -P --org earthly-technologies --satellite lib +test
+  test-native-arm:
+    name: +test-native-arm
+    runs-on: ubuntu-latest
+    env:
+      FORCE_COLOR: 1
+      EARTHLY_CONVERSION_PARALLELISM: "5"
+      EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}"
+      EARTHLY_INSTALL_ID: "earthly-lib-githubactions"
+    steps:
+      - uses: earthly/actions/setup-earthly@v1
+      - uses: actions/checkout@v2
+      - name: Docker mirror login (non fork only)
+        run: docker login registry-1.docker.io.mirror.corp.earthly.dev --username "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" --password "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}"
+        if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+      - name: Configure Earthly to use mirror (non fork only)
+        run: |-
+          earthly config global.buildkit_additional_config "'[registry.\"docker.io\"]
+
+          mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'"
+        if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+      - name: Run tests
+        run: |-
+          earthly config git "{github.com: {auth: ssh, user: git}}" && \
+          eval $(ssh-agent) && \
+          earthly secrets --org earthly-technologies --project core get littleredcorvette-id_rsa | ssh-add - && \
+          earthly --ci -P --org earthly-technologies --satellite lib-arm ./utils/dind+test

Earthfile

@@ -2,56 +2,7 @@ VERSION 0.7
 
 INSTALL_DIND:
     COMMAND
-    COPY +install-dind-script/install-dind.sh /tmp/install-dind.sh
-    RUN /tmp/install-dind.sh
-
-install-dind-script:
-    FROM alpine:3.17
-    COPY ./install-dind.sh ./
-    SAVE ARTIFACT ./install-dind.sh
+    DO ./utils/dind+INSTALL
 
 test:
-    BUILD +test-install-dind-amd64
-    BUILD ./ssh+test-add-known-hosts
-
-test-install-dind-amd64:
-    BUILD --platform=linux/amd64 +test-install-dind-for-image \
-        --base_image=docker:dind \
-        --base_image=alpine:latest \
-        --base_image=debian:stable \
-        --base_image=debian:stable-slim \
-        --base_image=debian:oldstable \
-        --base_image=ubuntu:latest \
-        --base_image=amazonlinux:1 \
-        --base_image=amazonlinux:2 \
-        --base_image=earthly/dind:alpine \
-        --base_image=earthly/dind:ubuntu
-
-test-install-dind-arm64:
-    BUILD --platform=linux/arm64 +test-install-dind-for-image \
-        --base_image=docker:dind \
-        --base_image=alpine:latest \
-        --base_image=ubuntu:latest \
-        --base_image=earthly/dind:alpine \
-        --base_image=earthly/dind:ubuntu
-
-test-install-dind-for-image:
-    ARG base_image
-    FROM "$base_image"
-
-    # docker-compose will fail with:
-    #     Error response from daemon: Invalid container name (-hello-1), only [a-zA-Z0-9][a-zA-Z0-9_.-] are allowed
-    # if docker-compose.yml is in the root directory, due to the --project flag defaulting to "".
-    # This bug does not exist in the official earthly/dind:alpine image.
-    WORKDIR /dind-test
-
-    DO +INSTALL_DIND
-    RUN echo "
-version: \"3\"
-services:
-    hello:
-        image: hello-world:latest
-    " >./docker-compose.yml
-    WITH DOCKER --compose docker-compose.yml
-        RUN true
-    END
+    BUILD ./utils+test

utils/Earthfile

@@ -0,0 +1,6 @@
+VERSION 0.7
+
+test:
+    BUILD ./dind+test
+    BUILD ./git+test
+    BUILD ./ssh+test

utils/dind/Earthfile

@@ -0,0 +1,14 @@
+VERSION 0.7
+
+INSTALL:
+    COMMAND
+    COPY +install-dind-script/install-dind.sh /tmp/install-dind.sh
+    RUN /tmp/install-dind.sh
+
+install-dind-script:
+    FROM alpine:3.18.4
+    COPY ./install-dind.sh ./
+    SAVE ARTIFACT ./install-dind.sh
+
+test:
+    BUILD ./tests+all

utils/dind/tests/Earthfile

@@ -0,0 +1,40 @@
+VERSION 0.7
+
+all:
+    BUILD +test-install-dind-for-image \
+        --base_image=docker:dind \
+        --base_image=alpine:latest \
+        --base_image=debian:stable \
+        --base_image=debian:stable-slim \
+        --base_image=debian:oldstable \
+        --base_image=ubuntu:latest \
+        --base_image=amazonlinux:1 \
+        --base_image=amazonlinux:2 \
+        --base_image=earthly/dind:alpine \
+        --base_image=earthly/dind:ubuntu
+
+test-install-dind-for-image:
+    ARG --required base_image
+    FROM alpine
+    ARG TARGETPLATFORM
+    IF [ "${base_image%:*}" = "amazonlinux" ] && [ "$TARGETPLATFORM" = "linux/arm64" ] # no amazonlinux:1 for arm64/UDC not supported atm for amazonlinux:2 on arm - skipping
+        RUN echo skipping $base_image with platform $TARGETPLATFORM
+    ELSE
+        FROM "$base_image"
+        # docker-compose will fail with:
+        #     Error response from daemon: Invalid container name (-hello-1), only [a-zA-Z0-9][a-zA-Z0-9_.-] are allowed
+        # if docker-compose.yml is in the root directory, due to the --project flag defaulting to "".
+        # This bug does not exist in the official earthly/dind:alpine image.
+        WORKDIR /dind-test
+
+        DO ..+INSTALL
+        RUN echo "
+    version: \"3\"
+    services:
+        hello:
+            image: hello-world:latest
+        " >./docker-compose.yml
+        WITH DOCKER --compose docker-compose.yml
+            RUN true
+        END
+    END

utils/git/Earthfile

@@ -0,0 +1,30 @@
+VERSION --arg-scope-and-set 0.7
+
+# DEEP_CLONE deep clones the provided git url
+DEEP_CLONE:
+    COMMAND
+    ARG USE_GIT_ENV="false"
+    IF [ $USE_GIT_ENV = "true" ]
+        FROM alpine/git:v2.40.1
+    END
+    DO ../ssh+ADD_KNOWN_HOSTS
+    ARG --required GIT_URL
+    ARG DEST_DIR
+    LET dest_dir=$DEST_DIR
+    IF [ -z $dest_dir ]
+        SET dest_dir=$(basename ${GIT_URL%.git})
+    END
+    GIT CLONE $GIT_URL $dest_dir
+    WORKDIR $dest_dir
+    RUN git remote set-url origin $GIT_URL
+    ARG git_hash=$(git rev-parse HEAD)
+    ARG SECRET_PATH
+    IF [ -z $SECRET_PATH ]
+        RUN --ssh git fetch --unshallow
+    ELSE
+        RUN --mount=type=secret,id=$SECRET_PATH,mode=0400,target=/root/.ssh/id_rsa \
+            git fetch --unshallow
+    END
+
+test:
+    BUILD ./tests+all

utils/git/tests/Earthfile

@@ -0,0 +1,48 @@
+VERSION --arg-scope-and-set --pass-args 0.7
+
+PROJECT earthly-technologies/core
+
+all:
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +test-deep-clone-image \
+        --base_image=alpine/git:latest \
+        --base_image=alpine:latest \
+        --base_image=debian:stable \
+        --base_image=debian:stable-slim \
+        --base_image=ubuntu:latest \
+        --base_image=amazonlinux:1 \
+        --base_image=amazonlinux:2 \
+        --GIT_URL=https://github.com/earthly/hello-world.git \
+        --GIT_URL=git@github.com:earthly/hello-world.git \
+        --GIT_URL=git@github.com:earthly/hello-world \
+        --SECRET_PATH="" \
+        --SECRET_PATH="littleredcorvette-id_rsa" \
+        --DEST_DIR="" \
+        --DEST_DIR="some-other-dir"
+
+test-deep-clone-image:
+    ARG --required base_image
+    ARG TARGETPLATFORM
+    FROM alpine
+    IF [ "$base_image" = "amazonlinux:1" ] && [ "$TARGETPLATFORM" = "linux/arm64" ] # no amazonlinux:1 for arm64, skipping
+        RUN echo skipping $base_image with platform $TARGETPLATFORM
+    ELSE
+        FROM "$base_image"
+        LET use_git_env="true"
+        IF [ "${base_image%:*}" = "alpine/git" ]
+            SET use_git_env="false"
+            RUN apk add git
+        ELSE IF [ "${base_image%:*}" = "alpine" ]
+            RUN apk add git
+        ELSE IF [ "${base_image%:*}" = "debian" ]
+            RUN apt update && apt install -y git
+        ELSE IF [ "${base_image%:*}" = "ubuntu" ]
+            RUN apt-get update && apt-get -y install git
+        ELSE IF [[ $base_image == amazonlinux* ]]
+            RUN yum -y install git
+        END
+        DO --pass-args ..+DEEP_CLONE --USE_GIT_ENV=$use_git_env
+        IF [ -n $DEST_DIR ]
+            WORKDIR $DEST_DIR
+        END
+        RUN git checkout main
+    END

utils/ssh/Earthfile

@@ -0,0 +1,19 @@
+VERSION 0.7
+
+# ADD_KNOWN_HOSTS appends some known public keys into the $target_file
+ADD_KNOWN_HOSTS:
+    COMMAND
+    ARG target_file=~/.ssh/known_hosts
+    LET expanded_target_file="$(eval echo $target_file)"
+    COPY +known-hosts/known_hosts /tmp/.
+    RUN mkdir -p $(dirname $expanded_target_file) && cat /tmp/known_hosts >> $expanded_target_file
+    RUN rm /tmp/known_hosts
+
+# known-hosts is used to copy the known_hosts file into the build context (internal use)
+known-hosts:
+    FROM alpine:3.18.4
+    COPY known_hosts .
+    SAVE ARTIFACT known_hosts
+
+test:
+    BUILD ./tests+all

ssh/Earthfile → utils/ssh/tests/Earthfile

@@ -1,26 +1,7 @@
 VERSION --pass-args --arg-scope-and-set 0.7
 
-# ADD_KNOWN_HOSTS will append some known public keys into the $target_file
-ADD_KNOWN_HOSTS:
-    COMMAND
-    ARG target_file=~/.ssh/known_hosts
-    LET expanded_target_file="$(eval echo $target_file)"
-    COPY +known-hosts/known_hosts /tmp/.
-    RUN mkdir -p $(dirname $expanded_target_file) && cat /tmp/known_hosts >> $expanded_target_file
-    RUN rm /tmp/known_hosts
-
-# known-hosts is used to copy the known_hosts file into the build context (internal use)
-known-hosts:
-    FROM alpine
-    COPY known_hosts .
-    SAVE ARTIFACT known_hosts
-
-#######################
-# Tests:
-#######################
-
-test-add-known-hosts:
-    BUILD --platform=linux/amd64 +test-add-known-hosts-image \
+all:
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +test-add-known-hosts-image \
         --base_image=alpine:latest \
         --base_image=debian:stable \
         --base_image=debian:stable-slim \
@@ -43,15 +24,15 @@ test-add-known-hosts-image:
         IF [ "$base_image" = "amazonlinux:1" ]
             RUN yum -y install diffutils.x86_64
         END
-        COPY ./known_hosts /tmp/expected-temp
+        COPY ..+known-hosts/known_hosts /tmp/expected-temp
         RUN test -s /tmp/expected-temp
         ARG target_file
         IF [ "$target_file" = "existing_known_hosts" ]
             RUN echo some-key >> /tmp/expected
             RUN echo some-key >> $target_file
         END
         RUN cat /tmp/expected-temp >> /tmp/expected
-        DO --pass-args +ADD_KNOWN_HOSTS
+        DO --pass-args ..+ADD_KNOWN_HOSTS
         LET expanded_target_file="$(eval echo $target_file)"
         RUN diff /tmp/expected $(eval echo $expanded_target_file)
     END