diff --git a/Dockerfile b/Dockerfile index 0847b4a..73341c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,13 +7,13 @@ ARG GO_TAG=1.26 ARG RUST_TAG=1.90.0 +ARG DEBIAN_RELEASE=trixie # These layers include Debian apt caches, so layers that extend `apt-base` # should not be published. Instead, these layers should be used to provide # cached data to individual `RUN` commands. -FROM docker.io/library/debian:bookworm-slim as apt-base -RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >>/etc/apt/sources.list +FROM docker.io/library/debian:${DEBIAN_RELEASE}-backports AS apt-base RUN DEBIAN_FRONTEND=noninteractive apt-get update RUN DEBIAN_FRONTEND=noninteractive apt-get install -y curl unzip xz-utils COPY --link bin/scurl /usr/local/bin/ @@ -28,37 +28,38 @@ RUN apt-get update && apt-get install nodejs -y # See https://apt.llvm.org/. FROM apt-base as apt-llvm RUN DEBIAN_FRONTEND=noninteractive apt-get install -y gnupg2 -RUN curl --tlsv1.2 -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key |apt-key add - -RUN ( echo 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-19 main' \ - && echo 'deb-src http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-19 main' ) >> /etc/apt/sources.list +RUN /usr/bin/curl --tlsv1.2 -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key |/usr/bin/tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + +ARG DEBIAN_RELEASE +# https://docs.docker.com/reference/dockerfile#understand-how-arg-and-from-interact +# ^ declare arg to use the default value in RUN below +RUN ( echo "deb http://apt.llvm.org/${DEBIAN_RELEASE}/ llvm-toolchain-${DEBIAN_RELEASE}-22 main" \ + && echo "deb-src http://apt.llvm.org/${DEBIAN_RELEASE}/ llvm-toolchain-${DEBIAN_RELEASE}-22 main" ) > /etc/apt/sources.list.d/llvm-toolchain.list +RUN cat /etc/apt/sources.list.d/llvm-toolchain.list RUN DEBIAN_FRONTEND=noninteractive apt-get update ## ## Scripting tools ## -# j5j Turns JSON5 into plain old JSON (i.e. to be processed by jq). -FROM apt-base as j5j -ARG J5J_VERSION=v0.2.0 # repo=olix0r/j5j -RUN url="https://github.com/olix0r/j5j/releases/download/${J5J_VERSION}/j5j-${J5J_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ - scurl "$url" | tar zvxf - -C /usr/local/bin j5j +# json5 transforms json +FROM apt-base AS apt-json5 +RUN DEBIAN_FRONTEND=noninteractive apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y node-json5 +# consumers use j5j as the interface to stripping comments from json +RUN ln -s /usr/bin/json5 /usr/local/bin/j5j # just runs build/test recipes. Like `make` but a bit more ergonomic. FROM apt-base as just -ARG JUST_VERSION=1.54.0 # repo=casey/just -RUN url="https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ - scurl "$url" | tar zvxf - -C /usr/local/bin just +RUN DEBIAN_FRONTEND=noninteractive apt update +RUN DEBIAN_FRONTEND=noninteractive apt install just # yq is kind of like jq, but for YAML. FROM apt-base as yq -ARG YQ_VERSION=v4.53.3 # repo=mikefarah/yq -RUN url="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" ; \ - scurl -o /yq "$url" && chmod +x /yq +RUN DEBIAN_FRONTEND=noninteractive apt update +RUN DEBIAN_FRONTEND=noninteractive apt install yq FROM scratch as tools-script -COPY --link --from=j5j /usr/local/bin/j5j /bin/ -COPY --link --from=just /usr/local/bin/just /bin/ -COPY --link --from=yq /yq /bin/ COPY --link bin/scurl /bin/ ## @@ -68,20 +69,22 @@ COPY --link bin/scurl /bin/ # helm templates kubernetes manifests. FROM apt-base as helm ARG HELM_VERSION=v3.21.2 # repo=helm/helm -RUN url="https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" ; \ - scurl "$url" | tar xzvf - --strip-components=1 -C /usr/local/bin linux-amd64/helm - +RUN arch=$(uname -m | sed -e 's/aarch/arm/' -e 's/x86_/amd/'); \ + url="https://get.helm.sh/helm-${HELM_VERSION}-linux-${arch}.tar.gz" ; \ + scurl "$url" | tar xzvf - --strip-components=1 -C /usr/local/bin linux-${arch}/helm # helm-docs generates documentation from helm charts. FROM apt-base as helm-docs ARG HELM_DOCS_VERSION=v1.14.2 # repo=norwoodj/helm-docs -RUN url="https://github.com/norwoodj/helm-docs/releases/download/$HELM_DOCS_VERSION/helm-docs_${HELM_DOCS_VERSION#v}_Linux_x86_64.tar.gz" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/'); \ + url="https://github.com/norwoodj/helm-docs/releases/download/$HELM_DOCS_VERSION/helm-docs_${HELM_DOCS_VERSION#v}_Linux_${arch}.tar.gz" ; \ scurl "$url" | tar xzvf - -C /usr/local/bin helm-docs # kubectl controls kubernetes clusters. FROM apt-base as kubectl ARG KUBECTL_VERSION=v1.36.2 # repo=kubernetes/kubernetes -RUN url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/'); \ + url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${arch}/kubectl" ; \ scurl -o /usr/local/bin/kubectl "$url" && chmod +x /usr/local/bin/kubectl # k3d runs kubernetes clusters in docker. @@ -116,7 +119,8 @@ COPY --link --from=ghcr.io/anchore/grype:v0.96.1 /grype /bin/ # actionlint lints github actions workflows. FROM apt-base as actionlint ARG ACTIONLINT_VERSION=v1.7.12 # repo=rhysd/actionlint -RUN url="https://github.com/rhysd/actionlint/releases/download/${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION#v}_linux_amd64.tar.gz" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/' -e 's/x86_/amd/'); \ + url="https://github.com/rhysd/actionlint/releases/download/${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION#v}_linux_${arch}.tar.gz" ; \ scurl "$url" | tar xzvf - -C /usr/local/bin actionlint # checksec checks binaries for security issues. @@ -137,7 +141,8 @@ COPY --link bin/action-* bin/just-dev bin/just-sh /bin/ FROM apt-base as protobuf ARG PROTOC_VERSION=v35.1 # repo=protocolbuffers/protobuf -RUN url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/protoc-${PROTOC_VERSION#v}-linux-$(uname -m).zip" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/aarch_/'); \ + url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/protoc-${PROTOC_VERSION#v}-linux-${arch}.zip" ; \ cd $(mktemp -d) && \ scurl -o protoc.zip "$url" && \ unzip protoc.zip bin/protoc include/** && \ @@ -153,30 +158,36 @@ RUN url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/pr # cargo-action-fmt formats `cargo build` JSON output to Github Actions annotations. FROM apt-base as cargo-action-fmt ARG CARGO_ACTION_FMT_VERSION=v1.0.4 # ignore -RUN url="https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-${CARGO_ACTION_FMT_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ +RUN arch=$(uname -m); \ + url="https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-${CARGO_ACTION_FMT_VERSION}-${arch}-unknown-linux-musl.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin cargo-action-fmt FROM apt-base as cargo-auditable ARG CARGO_AUDITABLE_VERSION=v0.7.5 # repo=rust-secure-code/cargo-auditable -RUN url="https://github.com/rust-secure-code/cargo-auditable/releases/download/${CARGO_AUDITABLE_VERSION}/cargo-auditable-x86_64-unknown-linux-gnu.tar.xz" ; \ - scurl "$url" | tar xJvf - --strip-components=1 -C /usr/local/bin cargo-auditable-x86_64-unknown-linux-gnu/cargo-auditable +RUN arch=$(uname -m); \ + libc=$([ "$arch" = "x86_64" ] && echo "musl" || echo "gnu"); \ + url="https://github.com/rust-secure-code/cargo-auditable/releases/download/${CARGO_AUDITABLE_VERSION}/cargo-auditable-${arch}-unknown-linux-${libc}.tar.xz" ; \ + scurl "$url" | tar xJvf - --strip-components=1 -C /usr/local/bin cargo-auditable-${arch}-unknown-linux-${libc}/cargo-auditable # cargo-deny checks cargo dependencies for licensing and RUSTSEC security issues. FROM apt-base as cargo-deny ARG CARGO_DENY_VERSION=0.19.9 # repo=EmbarkStudios/cargo-deny -RUN url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${CARGO_DENY_VERSION}/cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ - scurl "$url" | tar zvxf - --strip-components=1 -C /usr/local/bin "cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" +RUN arch=$(uname -m); \ + url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${CARGO_DENY_VERSION}/cargo-deny-${CARGO_DENY_VERSION}-${arch}-unknown-linux-musl.tar.gz" ; \ + scurl "$url" | tar zvxf - --strip-components=1 -C /usr/local/bin "cargo-deny-${CARGO_DENY_VERSION}-${arch}-unknown-linux-musl/cargo-deny" # cargo-nextest is a nicer test runner. FROM apt-base as cargo-nextest ARG NEXTEST_VERSION=0.9.138 # repo=nextest-rs/nextest,prefix=cargo-nextest- -RUN url="https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${NEXTEST_VERSION}/cargo-nextest-${NEXTEST_VERSION}-x86_64-unknown-linux-gnu.tar.gz" ; \ +RUN arch=$(uname -m); \ + url="https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${NEXTEST_VERSION}/cargo-nextest-${NEXTEST_VERSION}-${arch}-unknown-linux-gnu.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin cargo-nextest # cargo-tarpaulin is a code coverage tool. FROM apt-base as cargo-tarpaulin ARG CARGO_TARPAULIN_VERSION=0.35.5 # repo=xd009642/tarpaulin -RUN url="https://github.com/xd009642/tarpaulin/releases/download/${CARGO_TARPAULIN_VERSION}/cargo-tarpaulin-x86_64-unknown-linux-musl.tar.gz" ;\ +RUN arch=$(uname -m); \ + url="https://github.com/xd009642/tarpaulin/releases/download/${CARGO_TARPAULIN_VERSION}/cargo-tarpaulin-${arch}-unknown-linux-musl.tar.gz" ;\ scurl "$url" | tar xzvf - -C /usr/local/bin cargo-tarpaulin FROM scratch as tools-rust @@ -279,7 +290,7 @@ ENV PROTOC_NO_VENDOR=1 \ PROTOC_INCLUDE=/usr/local/include # A Rust build environment. -FROM docker.io/library/rust:${RUST_TAG}-slim-bookworm as rust +FROM docker.io/library/rust:${RUST_TAG}-slim-${DEBIAN_RELEASE} as rust RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ @@ -313,7 +324,6 @@ ENV CARGO_INCREMENTAL=0 \ RUSTUP_MAX_RETRIES=10 ENTRYPOINT ["/usr/local/bin/just-cargo"] -COPY --link --from=just /usr/local/bin/just /usr/local/bin/ FROM rust as rust-musl RUN rustup target add \ aarch64-unknown-linux-musl \ @@ -331,7 +341,7 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ ## Devcontainer ## -FROM docker.io/library/debian:bookworm as devcontainer +FROM docker.io/library/debian:${DEBIAN_RELEASE} AS devcontainer RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ @@ -340,6 +350,7 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ curl \ dnsutils \ file \ + git \ iproute2 \ jo \ jq \ @@ -369,12 +380,6 @@ RUN groupadd --gid=1000 code \ && echo "code ALL=(root) NOPASSWD:ALL" >/etc/sudoers.d/code \ && chmod 0440 /etc/sudoers.d/code -# git v2.34+ has new subcommands and supports code signing via SSH. -RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ - DEBIAN_FRONTEND=noninteractive apt-get install -y -t bookworm-backports git - RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ @@ -382,17 +387,20 @@ RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ ENV CC=clang-19 \ CXX=clang++-19 -# Use microsoft's Docker setup script to install the Docker CLI. -# +# Install docker-compose since it breaks in the docker-debian script on arm64 +RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/apt \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose + +# devcontainers/features: install docker do not use moby (not supported on trixie) # A distinct cache is used because the script adds an apt repo that we don't # want to pull in for other layers. -# -# TODO(ver): replace this with a devcontainer feature? RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/apt \ --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ --mount=type=bind,from=tools,source=/bin/scurl,target=/usr/local/bin/scurl \ - scurl https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/docker-debian.sh | bash -s + scurl https://raw.githubusercontent.com/devcontainers/features/refs/heads/main/src/docker-outside-of-docker/install.sh | MOBY=false bash -s ENV DOCKER_BUILDKIT=1 ARG MARKDOWNLINT_VERSION=0.22.1 diff --git a/README.md b/README.md index dafa52d..768b3b3 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ Update k3s images for all channels: ```sh :; just sync-k3s-images ``` + ### 2. Update Versions in Dockerfile Update the versions of all the tooling pulled in Dockerfile: diff --git a/justfile b/justfile index da01125..4356eab 100644 --- a/justfile +++ b/justfile @@ -1,33 +1,71 @@ version := '' image := 'ghcr.io/linkerd/dev' -_tag := if version != '' { "--tag=" + image + ':' + version } else { "" } + +# Auto-detect latest git version if version is 'latest-git-tag' +_version := if version == 'latest-git-tag' { + shell('which git > /dev/null || (echo >&2 "$1error$2: git not available" && exit 1) && git --git-dir="$3/.git" --work-tree="$3" tag -l --sort=-version:refname "v*" | head -n 1', style('error'), NORMAL, justfile_directory()) +} else { + version +} + +_tag := if _version != '' { "--tag=" + image + ':' + _version } else { "" } k3s-image := 'docker.io/rancher/k3s' +dry_run := 'false' +docker_arch := '' + +# Detect docker_bin if not specified: try docker first, then podman +docker_bin := shell('which docker 2> /dev/null || which podman 2> /dev/null || (echo >&2 "$1error$2: neither docker nor podman found" && exit 1)', style('error'), NORMAL) +# Extract basename of docker_bin to identify implementation +_docker_bin_name := file_name(docker_bin) +# Auto-detect if podman is in remote mode (unless explicitly overridden) +podman_remote := if _docker_bin_name == 'podman' { + shell('if $1 info 2>/dev/null | grep -q "remoteSocket"; then echo "true"; else echo "false"; fi', docker_bin) +} else { + 'false' +} + +# pull policy +_pull_policy := if _docker_bin_name == 'podman' { + '=never' +} else { + '' +} + targets := 'go rust rust-musl tools devcontainer' load := 'false' push := 'false' -output := if push == 'true' { - 'type=registry' - } else if load == 'true' { - 'type=docker' - } else { - 'type=image' - } + +# Remote mode cannot use the --output flag +output := if podman_remote == 'true' { + '' +} else if push == 'true' { + '--output=type=registry' +} else if load == 'true' { + '--output=type=docker' +} else { + '--output=type=image' +} export DOCKER_PROGRESS := env_var_or_default('DOCKER_PROGRESS', 'auto') all: sync-k3s-images build -build: && _list-if-load +build *args='': && _list-if-load #!/usr/bin/env bash set -euo pipefail for tgt in {{ targets }} ; do just output='{{ output }}' \ image='{{ image }}' \ - version='{{ version }}' \ - _target "$tgt" + version='{{ _version }}' \ + docker_arch='{{ docker_arch }}' \ + dry_run='{{ dry_run }}' \ + docker_bin='{{ docker_bin }}' \ + podman_remote='{{ podman_remote }}' \ + _target "$tgt" \ + {{ args }} done _list-if-load: @@ -36,23 +74,30 @@ _list-if-load: if [ '{{ load }}' = 'true' ] ; then just image='{{ image }}' \ targets='{{ targets }}' \ - version='{{ version }}' \ + version='{{ _version }}' \ list fi list: #!/usr/bin/env bash set -euo pipefail - if [ -z '{{ version }}' ]; then + if [ -z '{{ _version }}' ]; then echo "Usage: just version= list" >&2 exit 64 fi for tgt in {{ targets }} ; do if [ "$tgt" == "devcontainer" ]; then - docker image ls {{ image }}:{{ version }} | sed 1d + cmd="{{ docker_bin }} image ls {{ image }}:{{ _version }} | sed 1d" else - docker image ls {{ image }}:{{ version }}-$tgt | sed 1d + cmd="{{ docker_bin }} image ls {{ image }}:{{ _version }}-$tgt | sed 1d" fi + + echo "{{ style('error') }}$cmd{{ NORMAL }}" + if [ "{{ dry_run }}" = "true" ]; then + continue + fi + + eval "$cmd" done # Fetch the latest version of k3s images and record their tags and digests. @@ -96,19 +141,36 @@ _k3s-channels: | {key:.id, value:$tag} ] | from_entries' -_target target='': +_target target='' *args='': @just \ output='{{ output }}' \ image='{{ image }}' \ + version='{{ _version }}' \ + docker_arch='{{ docker_arch }}' \ + dry_run='{{ dry_run }}' \ + docker_bin='{{ docker_bin }}' \ + podman_remote='{{ podman_remote }}' \ _build --target='{{ target }}' \ - {{ if version == '' { '' } else { '--tag=' + image + ':' + version + if target == 'devcontainer' { '' } else { '-' + target } } }} + {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} \ + {{ args }} # Build the devcontainer image _build *args='': - docker buildx build . {{ _tag }} --pull \ + #!/usr/bin/env bash + set -euo pipefail + + cmd="{{ docker_bin }} buildx build . {{ _tag }} --pull{{ _pull_policy }} \ --progress='{{ DOCKER_PROGRESS }}' \ - --output='{{ output }}' \ - {{ args }} + {{ output }} \ + {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ + {{ args }}" + + echo "{{ style('error') }}$cmd{{ NORMAL }}" + if [ "{{ dry_run }}" = "true" ]; then + exit 0 + fi + + eval "$cmd" md-lint *patterns="'**/*.md' '!repos/**'":