From cd873608ffe880c90734d26a81971aff79dffb5e Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 11:11:17 +0500 Subject: [PATCH 01/12] Add native RPM/DEB packaging + release pipeline for ColdFront --- .github/workflows/release.yml | 728 ++++++++++++++++-- .goreleaser.yaml | 25 +- common/build.sh | 27 + packaging/coldfront-bin/build-deb.sh | 67 ++ packaging/coldfront-bin/build-rpm.sh | 58 ++ packaging/coldfront-bin/common.sh | 25 + packaging/coldfront-bin/deb/debian/control | 20 + .../deb/debian/pgedge-coldfront.install | 4 + packaging/coldfront-bin/deb/debian/rules | 24 + .../coldfront-bin/deb/debian/source/format | 1 + packaging/coldfront-bin/rpm/coldfront.spec | 53 ++ .../coldfront-duckdb-extensions/build-deb.sh | 110 +++ .../coldfront-duckdb-extensions/build-rpm.sh | 110 +++ .../coldfront-duckdb-extensions/common.sh | 52 ++ .../coldfront-duckdb-extensions.conf.sample | 35 + .../deb/debian/control | 21 + .../deb/debian/rules | 32 + .../deb/debian/source/format | 1 + .../rpm/coldfront-duckdb-extensions.spec | 70 ++ packaging/coldfront/build-deb.sh | 61 ++ packaging/coldfront/build-rpm.sh | 49 ++ packaging/coldfront/common.sh | 30 + packaging/coldfront/deb/debian/control.in | 25 + .../pgedge-postgresql-coldfront.install | 1 + packaging/coldfront/deb/debian/pgversions | 1 + packaging/coldfront/deb/debian/rules | 30 + packaging/coldfront/deb/debian/source/format | 1 + packaging/coldfront/rpm/coldfront.spec | 61 ++ packaging/pg_duckdb/build-deb.sh | 101 +++ packaging/pg_duckdb/build-rpm.sh | 69 ++ packaging/pg_duckdb/common.sh | 39 + packaging/pg_duckdb/deb/debian/control.in | 45 ++ .../pgedge-postgresql-pg-duckdb.install | 1 + packaging/pg_duckdb/deb/debian/pgversions | 1 + packaging/pg_duckdb/deb/debian/rules | 40 + packaging/pg_duckdb/deb/debian/source/format | 1 + packaging/pg_duckdb/rpm/pg_duckdb.spec | 112 +++ 37 files changed, 2072 insertions(+), 59 deletions(-) create mode 100644 common/build.sh create mode 100644 packaging/coldfront-bin/build-deb.sh create mode 100644 packaging/coldfront-bin/build-rpm.sh create mode 100644 packaging/coldfront-bin/common.sh create mode 100644 packaging/coldfront-bin/deb/debian/control create mode 100644 packaging/coldfront-bin/deb/debian/pgedge-coldfront.install create mode 100644 packaging/coldfront-bin/deb/debian/rules create mode 100644 packaging/coldfront-bin/deb/debian/source/format create mode 100644 packaging/coldfront-bin/rpm/coldfront.spec create mode 100644 packaging/coldfront-duckdb-extensions/build-deb.sh create mode 100644 packaging/coldfront-duckdb-extensions/build-rpm.sh create mode 100644 packaging/coldfront-duckdb-extensions/common.sh create mode 100644 packaging/coldfront-duckdb-extensions/config/coldfront-duckdb-extensions.conf.sample create mode 100644 packaging/coldfront-duckdb-extensions/deb/debian/control create mode 100644 packaging/coldfront-duckdb-extensions/deb/debian/rules create mode 100644 packaging/coldfront-duckdb-extensions/deb/debian/source/format create mode 100644 packaging/coldfront-duckdb-extensions/rpm/coldfront-duckdb-extensions.spec create mode 100644 packaging/coldfront/build-deb.sh create mode 100644 packaging/coldfront/build-rpm.sh create mode 100644 packaging/coldfront/common.sh create mode 100644 packaging/coldfront/deb/debian/control.in create mode 100644 packaging/coldfront/deb/debian/pgedge-postgresql-coldfront.install create mode 100644 packaging/coldfront/deb/debian/pgversions create mode 100644 packaging/coldfront/deb/debian/rules create mode 100644 packaging/coldfront/deb/debian/source/format create mode 100644 packaging/coldfront/rpm/coldfront.spec create mode 100644 packaging/pg_duckdb/build-deb.sh create mode 100644 packaging/pg_duckdb/build-rpm.sh create mode 100644 packaging/pg_duckdb/common.sh create mode 100644 packaging/pg_duckdb/deb/debian/control.in create mode 100644 packaging/pg_duckdb/deb/debian/pgedge-postgresql-pg-duckdb.install create mode 100644 packaging/pg_duckdb/deb/debian/pgversions create mode 100644 packaging/pg_duckdb/deb/debian/rules create mode 100644 packaging/pg_duckdb/deb/debian/source/format create mode 100644 packaging/pg_duckdb/rpm/pg_duckdb.spec diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38ac0f3..62207bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,71 +1,685 @@ +# ------------------------------------------------------------------------- +# +# pgEdge ColdFront — release pipeline (in-repo packaging) +# +# Tag-driven. A pushed v* tag builds and publishes, in one GitHub Actions run: +# 1. GoReleaser: the Go binaries (archiver, partitioner, compactor) + the +# GitHub release, and uploads the linux tarballs as a workflow artifact. +# 2. Native RPM/DEB packages for three components, across PG 16/17/18 and +# el9/el10 + jammy/noble/bullseye/bookworm/trixie, amd64 + arm64: +# * packaging/coldfront-bin — the Go tools (PG-independent, per-arch) +# * packaging/coldfront — the coldfront PG extension (per PG major) +# * packaging/pg_duckdb — pg_duckdb 1.5.4 (per PG major) +# * packaging/coldfront-duckdb-extensions — patched iceberg/avro/azure/ +# postgres_scanner .duckdb_extension (PG-independent, per distro/arch) +# pg_duckdb + duckdb-extensions keep their own version (1.5.4); coldfront-bin +# + the extension track the release tag. +# 3. Push RPMs to dnf.pgedge.com and DEBs to apt.pgedge.com, back up to S3, +# and publish a promotion manifest + Slack notification. +# +# common-functions.sh is supplied by pgedge-builder-action at build time. +# ------------------------------------------------------------------------- + name: Release -# Tag-driven releases. Push a v* tag (e.g. v0.1.0) to cut a release; the -# workflow_dispatch entry builds a snapshot (no GitHub release) for dry runs. on: - push: - tags: - - 'v*' - workflow_dispatch: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + simulate_tag: + description: 'Pretend a v* tag was pushed (e.g. "v1.0.0-test1") to exercise the pipeline on a branch (GoReleaser --snapshot, no live release/repos).' + required: false + default: '' + force_push: + description: 'Publish RPMs/DEBs even if some matrix cells failed. Only honored on real tag pushes.' + required: false + default: 'false' permissions: - contents: write + contents: read + +concurrency: + group: release-${{ github.ref_name }} + cancel-in-progress: false + +env: + # Umbrella name for the backup/manifest grouping; per-component identity + # lives in the manifest cells[]. + COMPONENT_NAME: ${{ github.event.repository.name }} jobs: - goreleaser: - name: Build and release - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - fetch-depth: 0 + # ========================================================================= + # Tag parsing → (repo_type, version, buildnum). + # ========================================================================= + determine-repo-type: + name: Determine repo routing + runs-on: ubuntu-latest + if: github.ref_type == 'tag' || github.event.inputs.simulate_tag != '' + outputs: + repo-type: ${{ steps.route.outputs.repo_type }} + component-version: ${{ steps.parse.outputs.version }} + component-buildnum: ${{ steps.parse.outputs.buildnum }} + effective-tag: ${{ steps.parse.outputs.effective_tag }} + simulated: ${{ steps.parse.outputs.simulated }} + steps: + - name: Checkout pgedge-parse-release-tag + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-parse-release-tag.git" \ + ".github/actions/pgedge-parse-release-tag" + - name: Parse release tag + id: parse + uses: ./.github/actions/pgedge-parse-release-tag + with: + simulate_tag: ${{ github.event.inputs.simulate_tag }} + - name: Route suffix → repo_type + id: route + env: + SUFFIX: ${{ steps.parse.outputs.suffix }} + EFFECTIVE_TAG: ${{ steps.parse.outputs.effective_tag }} + run: | + set -euo pipefail + case "$SUFFIX" in + test*|dev*) repo_type=daily ;; + rc*|beta*|alpha*|"") repo_type=staging ;; + *) + echo "::error::Unsupported suffix '$SUFFIX' (effective tag: $EFFECTIVE_TAG)" + exit 1 + ;; + esac + echo "repo_type=$repo_type" >> "$GITHUB_OUTPUT" + echo "Routed suffix '$SUFFIX' → repo_type=$repo_type" - - name: Fetch tags - run: git fetch --force --tags + # ========================================================================= + # GoReleaser — builds archiver/partitioner/compactor, publishes the GitHub + # release (real tag) or a snapshot (branch dispatch), and uploads the linux + # tarballs as the release-artifacts the binaries package cells consume. + # ========================================================================= + build: + name: Build (GoReleaser) + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + - name: Fetch tags + run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + cache: true + - name: Vet + run: go vet ./... + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@9a127d869fb706213d29cdf8eef3a4ea2b869415 # v7 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean ${{ (github.event_name == 'workflow_dispatch' && github.ref_type != 'tag') && '--snapshot' || '' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Verify Linux binaries are statically linked + run: | + set -euo pipefail + for tarball in dist/coldfront_*_linux_*.tar.gz; do + echo "== $tarball ==" + tmp="$(mktemp -d)" + tar xzf "$tarball" -C "$tmp" + for bin in archiver partitioner compactor; do + if ! file "$tmp/$bin" | grep -q "statically linked"; then + echo "ERROR: $bin in $tarball is not statically linked" >&2 + file "$tmp/$bin" >&2 + exit 1 + fi + echo "OK: $bin statically linked" + done + done + - name: Upload release artifacts (for packaging) + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: release-artifacts + path: dist/*.tar.gz + retention-days: 1 - - name: Set up Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - cache: true + # ========================================================================= + # Build matrix — detect per component, then merge into one combined matrix + # whose cells carry component_name / pg_version / component_version. + # ========================================================================= + detect-matrix: + name: Detect build matrix + runs-on: ubuntu-latest + needs: [determine-repo-type] + outputs: + rpm_matrix: ${{ steps.merge.outputs.rpm_matrix }} + deb_matrix: ${{ steps.merge.outputs.deb_matrix }} + has_rpm: ${{ steps.merge.outputs.has_rpm }} + has_deb: ${{ steps.merge.outputs.has_deb }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Checkout pgedge-detect-build-matrix + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-detect-build-matrix.git" \ + ".github/actions/pgedge-detect-build-matrix" + - name: Detect — pg_duckdb (PG 16/17/18) + id: pgduckdb + uses: ./.github/actions/pgedge-detect-build-matrix + with: + component_name: packaging/pg_duckdb + pg_versions: '16,17,18' + archs: '["amd64","arm64"]' + rpm_images: '["almalinux:9","almalinux:10"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + - name: Detect — coldfront extension (PG 16/17/18) + id: coldfront + uses: ./.github/actions/pgedge-detect-build-matrix + with: + component_name: packaging/coldfront + pg_versions: '16,17,18' + archs: '["amd64","arm64"]' + rpm_images: '["almalinux:9","almalinux:10"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + - name: Detect — coldfront binaries (PG-independent) + id: bin + uses: ./.github/actions/pgedge-detect-build-matrix + with: + component_name: packaging/coldfront-bin + archs: '["amd64","arm64"]' + rpm_images: '["almalinux:9","almalinux:10"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + - name: Detect — coldfront duckdb-extensions (PG-independent) + id: duckdbext + uses: ./.github/actions/pgedge-detect-build-matrix + with: + component_name: packaging/coldfront-duckdb-extensions + archs: '["amd64","arm64"]' + rpm_images: '["almalinux:9","almalinux:10"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + - name: Merge component matrices + id: merge + env: + TAG_VERSION: ${{ needs.determine-repo-type.outputs.component-version }} + RPM_PGD: ${{ steps.pgduckdb.outputs.rpm_matrix }} + DEB_PGD: ${{ steps.pgduckdb.outputs.deb_matrix }} + RPM_CF: ${{ steps.coldfront.outputs.rpm_matrix }} + DEB_CF: ${{ steps.coldfront.outputs.deb_matrix }} + RPM_BIN: ${{ steps.bin.outputs.rpm_matrix }} + DEB_BIN: ${{ steps.bin.outputs.deb_matrix }} + RPM_DUX: ${{ steps.duckdbext.outputs.rpm_matrix }} + DEB_DUX: ${{ steps.duckdbext.outputs.deb_matrix }} + run: | + set -euo pipefail + # Tag a component's cells with component_name + component_version. + # pg_duckdb + duckdb-extensions keep their own version (empty -> + # common.sh default, the DuckDB version); coldfront-bin + the + # extension take the release tag. + tag_cells() { # + jq -c --arg c "$2" --arg v "$3" \ + '[.include[] | . + {component_name:$c, component_version:$v}]' <<<"$1" + } + rpm_inc="$(jq -c -n \ + --argjson a "$(tag_cells "$RPM_PGD" packaging/pg_duckdb '')" \ + --argjson b "$(tag_cells "$RPM_CF" packaging/coldfront "$TAG_VERSION")" \ + --argjson d "$(tag_cells "$RPM_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ + --argjson e "$(tag_cells "$RPM_DUX" packaging/coldfront-duckdb-extensions '')" \ + '$a + $b + $d + $e')" + deb_inc="$(jq -c -n \ + --argjson a "$(tag_cells "$DEB_PGD" packaging/pg_duckdb '')" \ + --argjson b "$(tag_cells "$DEB_CF" packaging/coldfront "$TAG_VERSION")" \ + --argjson d "$(tag_cells "$DEB_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ + --argjson e "$(tag_cells "$DEB_DUX" packaging/coldfront-duckdb-extensions '')" \ + '$a + $b + $d + $e')" + echo "rpm_matrix={\"include\":$rpm_inc}" >> "$GITHUB_OUTPUT" + echo "deb_matrix={\"include\":$deb_inc}" >> "$GITHUB_OUTPUT" + echo "has_rpm=$( [ "$(jq 'length' <<<"$rpm_inc")" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT" + echo "has_deb=$( [ "$(jq 'length' <<<"$deb_inc")" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT" - # Both binaries are pure-Go (CGO_ENABLED=0); all four targets - # (linux/darwin x amd64/arm64) cross-compile cleanly on this one runner. - - name: Vet - run: go vet ./... + # ========================================================================= + # RPM build matrix — one cell per (component, PG major, EL major, arch). + # ========================================================================= + package-rpm: + name: RPM ${{ matrix.component_name }} ${{ matrix.image }} pg${{ matrix.pg_version }} ${{ matrix.arch }} + needs: [determine-repo-type, build, detect-matrix] + if: | + always() && + needs.determine-repo-type.result == 'success' && + needs.detect-matrix.result == 'success' && + needs.build.result == 'success' && + needs.detect-matrix.outputs.has_rpm == 'true' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-matrix.outputs.rpm_matrix) }} + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_BUILDNUM: ${{ needs.determine-repo-type.outputs.component-buildnum }} + COMPONENT_BRANCH: ${{ needs.determine-repo-type.outputs.effective-tag }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Derive os_version / safe component + id: cell + env: + IMAGE: ${{ matrix.image }} + COMP: ${{ matrix.component_name }} + run: | + set -euo pipefail + echo "os_version=${IMAGE##*:}" >> "$GITHUB_OUTPUT" + echo "safe_comp=${COMP##*/}" >> "$GITHUB_OUTPUT" + - name: Stage binaries tarball (coldfront-bin only) + if: matrix.component_name == 'packaging/coldfront-bin' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: release-artifacts + path: go-dist + - name: Prepare release-artifacts/ (coldfront-bin only) + if: matrix.component_name == 'packaging/coldfront-bin' + env: + ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + mkdir -p release-artifacts + cp go-dist/coldfront_*_linux_${ARCH}.tar.gz release-artifacts/coldfront.tar.gz + rm -rf go-dist + - name: Checkout pgedge-builder-action + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-builder-action.git" \ + ".github/actions/pgedge-builder-action" + - name: Build RPM + uses: ./.github/actions/pgedge-builder-action + with: + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + component_name: ${{ matrix.component_name }} + pg_version: ${{ matrix.pg_version }} + component_branch: ${{ env.COMPONENT_BRANCH }} + component_version: ${{ matrix.component_version }} + component_buildnum: ${{ env.COMPONENT_BUILDNUM }} + repo_type: ${{ env.REPO_TYPE }} + gpg_private_key: ${{ secrets.GPG_FIPS_RPM_PRIVATE_KEY }} + gpg_public_key: ${{ secrets.GPG_FIPS_RPM_PUBLIC_KEY }} + pgedge_builder_token: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + - name: Upload RPM cell artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: rpm__${{ steps.cell.outputs.safe_comp }}__el-${{ steps.cell.outputs.os_version }}__pg${{ matrix.pg_version }}__${{ matrix.arch }} + path: output/el${{ steps.cell.outputs.os_version }}-${{ matrix.arch }}/ + retention-days: 7 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@9a127d869fb706213d29cdf8eef3a4ea2b869415 # v7 - with: - distribution: goreleaser - version: "~> v2" - args: release --clean ${{ github.ref_type != 'tag' && '--snapshot' || '' }} + # ========================================================================= + # DEB build matrix — one cell per (component, PG major, distro, arch). + # ========================================================================= + package-deb: + name: DEB ${{ matrix.component_name }} ${{ matrix.image }} pg${{ matrix.pg_version }} ${{ matrix.arch }} + needs: [determine-repo-type, build, detect-matrix] + if: | + always() && + needs.determine-repo-type.result == 'success' && + needs.detect-matrix.result == 'success' && + needs.build.result == 'success' && + needs.detect-matrix.outputs.has_deb == 'true' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-matrix.outputs.deb_matrix) }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_BUILDNUM: ${{ needs.determine-repo-type.outputs.component-buildnum }} + COMPONENT_BRANCH: ${{ needs.determine-repo-type.outputs.effective-tag }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Derive distro / safe component + id: cell + env: + IMAGE: ${{ matrix.image }} + COMP: ${{ matrix.component_name }} + run: | + set -euo pipefail + echo "distro=${IMAGE##*:}" >> "$GITHUB_OUTPUT" + echo "safe_comp=${COMP##*/}" >> "$GITHUB_OUTPUT" + - name: Stage binaries tarball (coldfront-bin only) + if: matrix.component_name == 'packaging/coldfront-bin' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: release-artifacts + path: go-dist + - name: Prepare release-artifacts/ (coldfront-bin only) + if: matrix.component_name == 'packaging/coldfront-bin' + env: + ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + mkdir -p release-artifacts + cp go-dist/coldfront_*_linux_${ARCH}.tar.gz release-artifacts/coldfront.tar.gz + rm -rf go-dist + - name: Checkout pgedge-builder-action + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-builder-action.git" \ + ".github/actions/pgedge-builder-action" + - name: Build DEB + uses: ./.github/actions/pgedge-builder-action + with: + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + component_name: ${{ matrix.component_name }} + pg_version: ${{ matrix.pg_version }} + component_branch: ${{ env.COMPONENT_BRANCH }} + component_version: ${{ matrix.component_version }} + component_buildnum: ${{ env.COMPONENT_BUILDNUM }} + repo_type: ${{ env.REPO_TYPE }} + gpg_private_key: ${{ secrets.GPG_FIPS_DEB_PRIVATE_KEY }} + gpg_public_key: ${{ secrets.GPG_FIPS_DEB_PUBLIC_KEY }} + pgedge_builder_token: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + - name: Upload DEB cell artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: deb__${{ steps.cell.outputs.safe_comp }}__${{ steps.cell.outputs.distro }}__pg${{ matrix.pg_version }}__${{ matrix.arch }} + path: output/${{ steps.cell.outputs.distro }}-${{ matrix.arch }}/ + retention-days: 7 - - name: Verify Linux binaries are statically linked - run: | - set -euo pipefail - for tarball in dist/coldfront_*_linux_*.tar.gz; do - echo "== $tarball ==" - tmp="$(mktemp -d)" - tar xzf "$tarball" -C "$tmp" - for bin in archiver partitioner; do - if ! file "$tmp/$bin" | grep -q "statically linked"; then - echo "ERROR: $bin in $tarball is not statically linked" >&2 - file "$tmp/$bin" >&2 - exit 1 - fi - echo "OK: $bin statically linked" - done - done + # ========================================================================= + # Push RPMs → dnf.pgedge.com (all components, batched per EL major) + S3 backup. + # ========================================================================= + push-dnf: + name: Push DNF (all components, batched) + needs: [determine-repo-type, detect-matrix, package-rpm] + if: | + always() && github.ref_type == 'tag' && + (needs.package-rpm.result == 'success' || + (github.event.inputs.force_push == 'true' && needs.package-rpm.result != 'cancelled')) + runs-on: ubuntu-latest + concurrency: + group: dnf-push-${{ needs.determine-repo-type.outputs.repo-type }} + cancel-in-progress: false + outputs: + timestamp: ${{ steps.ts.outputs.dnf_ts }} + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_VERSION: ${{ needs.determine-repo-type.outputs.component-version }} + RPM_MATRIX: ${{ needs.detect-matrix.outputs.rpm_matrix }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Timestamp + id: ts + run: echo "dnf_ts=$(date -u +'%Y%m%d-%H%M%S')" >> "$GITHUB_OUTPUT" + - name: Download all RPM cell artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: rpm__* + path: rpm-artifacts + merge-multiple: false + - name: Organize RPMs by EL major + build targets + id: targets + run: | + set -euo pipefail + # Each cell artifact dir holds el- rpms for one + # (component, pg, arch); merge them all by EL major. + os_versions=$(echo "$RPM_MATRIX" | jq -r '[.include[].image | split(":")[1]] | unique | .[]') + for os in $os_versions; do + dest="rpms-el${os}"; mkdir -p "$dest" + find rpm-artifacts -type f -name "*.el${os}.*.rpm" -exec mv -v {} "$dest/" \; 2>/dev/null || true + echo "EL${os}: $(find "$dest" -maxdepth 1 -name '*.rpm' | wc -l) rpms" + done + rm -rf rpm-artifacts + push="$(echo "$os_versions" | jq -R . | jq -cs 'map({"os-version": ., "rpm-dir": ("rpms-el" + .)})')" + echo "push=$push" >> "$GITHUB_OUTPUT" + echo "backup=$(echo "$push" | jq -c 'map({"os-version": .["os-version"], "output-dir": .["rpm-dir"]})')" >> "$GITHUB_OUTPUT" + - name: Checkout pgEdge action repos @multi-target + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + for repo in pgedge-dnf-repo-builder pgedge-backup-artifacts; do + git clone --depth 1 --branch multi-target \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/${repo}.git" ".github/actions/${repo}" + done + - name: Push RPMs to yum repo + uses: ./.github/actions/pgedge-dnf-repo-builder + with: + s3-bucket: ${{ secrets.S3_BUCKET_NAME }} + cf-distribution: ${{ secrets.CLOUDFRONT_DISTRIBUTION }} + repo-type: ${{ env.REPO_TYPE }} + targets: ${{ steps.targets.outputs.push }} + lock-bucket: ${{ secrets.S3_BACKUP_BUCKET_NAME }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Back up RPMs to S3 + uses: ./.github/actions/pgedge-backup-artifacts + with: + s3-backup-bucket: ${{ secrets.S3_BACKUP_BUCKET_NAME }} + repo-type: ${{ env.REPO_TYPE }} + component-name: ${{ env.COMPONENT_NAME }} + component-version: ${{ env.COMPONENT_VERSION }} + timestamp: ${{ steps.ts.outputs.dnf_ts }} + targets: ${{ steps.targets.outputs.backup }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # ========================================================================= + # Push DEBs → apt.pgedge.com (all components, batched per distro) + S3 backup. + # ========================================================================= + push-apt: + name: Push APT (all components, batched) + needs: [determine-repo-type, detect-matrix, package-deb] + if: | + always() && github.ref_type == 'tag' && + (needs.package-deb.result == 'success' || + (github.event.inputs.force_push == 'true' && needs.package-deb.result != 'cancelled')) + runs-on: ubuntu-latest + concurrency: + group: apt-push-${{ needs.determine-repo-type.outputs.repo-type }} + cancel-in-progress: false + outputs: + timestamp: ${{ steps.ts.outputs.apt_ts }} + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_VERSION: ${{ needs.determine-repo-type.outputs.component-version }} + DEB_MATRIX: ${{ needs.detect-matrix.outputs.deb_matrix }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Timestamp + id: ts + run: echo "apt_ts=$(date -u +'%Y%m%d-%H%M%S')" >> "$GITHUB_OUTPUT" + - name: Download all DEB cell artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: deb__* + path: deb-artifacts + merge-multiple: false + - name: Organize DEBs by distro + build targets + id: targets + run: | + set -euo pipefail + unique_images=$(echo "$DEB_MATRIX" | jq -c '[.include[].image] | unique') + push='[]' + for image in $(echo "$unique_images" | jq -r '.[]'); do + distro="${image##*:}"; dest="debs-${distro}"; mkdir -p "$dest" + find deb-artifacts -type f -name "*.${distro}_*.deb" -exec mv -v {} "$dest/" \; 2>/dev/null || true + echo "${distro}: $(find "$dest" -maxdepth 1 -name '*.deb' | wc -l) debs" + push="$(jq -c --arg n "$image" --arg d "$dest" '. + [{"os-name":$n,"deb-dir":$d}]' <<<"$push")" + done + rm -rf deb-artifacts + echo "push=$push" >> "$GITHUB_OUTPUT" + echo "backup=$(echo "$push" | jq -c 'map({"os-version": (.["os-name"] | split(":")[1]), "output-dir": .["deb-dir"]})')" >> "$GITHUB_OUTPUT" + - name: Checkout pgEdge action repos @multi-target + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + for repo in pgedge-apt-repo-builder pgedge-backup-artifacts; do + git clone --depth 1 --branch multi-target \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/${repo}.git" ".github/actions/${repo}" + done + - name: Push DEBs to apt repo + uses: ./.github/actions/pgedge-apt-repo-builder + with: + s3-bucket: ${{ secrets.APT_S3_BUCKET_NAME }} + cf-distribution: ${{ secrets.APT_CLOUDFRONT_DISTRIBUTION }} + repo-type: ${{ env.REPO_TYPE }} + targets: ${{ steps.targets.outputs.push }} + lock-bucket: ${{ secrets.S3_BACKUP_BUCKET_NAME }} + gpg-private-key: ${{ secrets.GPG_FIPS_DEB_PRIVATE_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Back up DEBs to S3 + uses: ./.github/actions/pgedge-backup-artifacts + with: + s3-backup-bucket: ${{ secrets.S3_BACKUP_BUCKET_NAME }} + repo-type: ${{ env.REPO_TYPE }} + component-name: ${{ env.COMPONENT_NAME }} + component-version: ${{ env.COMPONENT_VERSION }} + timestamp: ${{ steps.ts.outputs.apt_ts }} + targets: ${{ steps.targets.outputs.backup }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-artifacts - path: | - dist/*.tar.gz - dist/checksums.txt - retention-days: 30 + # ========================================================================= + # Aggregated manifest (per-component cells) + Slack. + # ========================================================================= + publish-manifest: + name: Publish aggregated backup manifest + needs: [determine-repo-type, detect-matrix, push-dnf, push-apt] + if: always() && needs.determine-repo-type.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Install jq + awscli + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y python3-pip jq + pip3 install --user awscli + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Build manifest (cells[] for qa-certify/promote) + id: build_manifest + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + CV: ${{ needs.determine-repo-type.outputs.component-version }} + CBN: ${{ needs.determine-repo-type.outputs.component-buildnum }} + EFFECTIVE_TAG: ${{ needs.determine-repo-type.outputs.effective-tag }} + SIMULATED: ${{ needs.determine-repo-type.outputs.simulated }} + PUSH_DNF_RESULT: ${{ needs.push-dnf.result }} + PUSH_APT_RESULT: ${{ needs.push-apt.result }} + DNF_TS: ${{ needs.push-dnf.outputs.timestamp }} + APT_TS: ${{ needs.push-apt.outputs.timestamp }} + RPM_MATRIX: ${{ needs.detect-matrix.outputs.rpm_matrix }} + DEB_MATRIX: ${{ needs.detect-matrix.outputs.deb_matrix }} + run: | + set -euo pipefail + mkdir -p aggregated + cells='[]' + if [ "${PUSH_DNF_RESULT}" = "success" ] && [ -n "${DNF_TS}" ]; then + while IFS=$'\t' read -r comp cver image os arch; do + path="${REPO_TYPE}/${COMPONENT_NAME}/${CV}/${DNF_TS}/dnf/${os}/" + cell="$(jq -nc --arg fam rpm --arg comp "$comp" --arg cv "$cver" --arg img "$image" --arg ver "$os" --arg arch "$arch" --arg bp "$path" --arg ts "$DNF_TS" --arg rt "$REPO_TYPE" '{family:$fam,component:$comp,component_version:$cv,image:$img,arch:$arch,os_version:$ver,backup_path:$bp,timestamp:$ts,repo_type:$rt}')" + cells="$(jq -c --argjson c "$cell" '. + [$c]' <<<"$cells")" + done < <(echo "$RPM_MATRIX" | jq -r '.include[] | [.component_name, .component_version, .image, (.image|split(":")[1]), .arch] | @tsv') + fi + if [ "${PUSH_APT_RESULT}" = "success" ] && [ -n "${APT_TS}" ]; then + while IFS=$'\t' read -r comp cver image distro arch; do + path="${REPO_TYPE}/${COMPONENT_NAME}/${CV}/${APT_TS}/apt/${distro}/" + cell="$(jq -nc --arg fam deb --arg comp "$comp" --arg cv "$cver" --arg img "$image" --arg distro "$distro" --arg arch "$arch" --arg bp "$path" --arg ts "$APT_TS" --arg rt "$REPO_TYPE" '{family:$fam,component:$comp,component_version:$cv,image:$img,arch:$arch,distro:$distro,backup_path:$bp,timestamp:$ts,repo_type:$rt}')" + cells="$(jq -c --argjson c "$cell" '. + [$c]' <<<"$cells")" + done < <(echo "$DEB_MATRIX" | jq -r '.include[] | [.component_name, .component_version, .image, (.image|split(":")[1]), .arch] | @tsv') + fi + jq -n \ + --arg run_id "${GITHUB_RUN_ID}" --arg run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg repo "${GITHUB_REPOSITORY}" --arg tag "${EFFECTIVE_TAG}" \ + --argjson simulated "${SIMULATED}" --arg commit_sha "${GITHUB_SHA}" \ + --arg component_name "${COMPONENT_NAME}" --arg component_version "${CV}" \ + --arg component_buildnum "${CBN}" --arg repo_type "${REPO_TYPE}" \ + --arg dnf_timestamp "${DNF_TS}" --arg apt_timestamp "${APT_TS}" \ + --arg push_dnf_outcome "${PUSH_DNF_RESULT}" --arg push_apt_outcome "${PUSH_APT_RESULT}" \ + --argjson cells "${cells}" \ + '{run_id:$run_id, run_attempt:$run_attempt, repo:$repo, tag:$tag, simulated:$simulated, commit_sha:$commit_sha, component_name:$component_name, component_version:$component_version, component_buildnum:$component_buildnum, repo_type:$repo_type, timestamps:{dnf:$dnf_timestamp, apt:$apt_timestamp}, push_results:{dnf:$push_dnf_outcome, apt:$push_apt_outcome}, cells:$cells}' \ + > aggregated/manifest.json + cat aggregated/manifest.json + { echo "manifest_json<<__EOF__"; jq -c '.' aggregated/manifest.json; echo "__EOF__"; } >> "$GITHUB_OUTPUT" + - name: Checkout pgedge-build-publisher + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-build-publisher.git" \ + ".github/actions/pgedge-build-publisher" + - name: Compose Slack fields + id: compose_slack + env: + CV: ${{ needs.determine-repo-type.outputs.component-version }} + CBN: ${{ needs.determine-repo-type.outputs.component-buildnum }} + EFFECTIVE_TAG: ${{ needs.determine-repo-type.outputs.effective-tag }} + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + PUSH_DNF_RESULT: ${{ needs.push-dnf.result }} + PUSH_APT_RESULT: ${{ needs.push-apt.result }} + ACTOR: ${{ github.actor }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + run: | + set -euo pipefail + unix_ts="$(date -u +%s)"; fallback_ts="$(date -u +'%Y-%m-%d %H:%M UTC')" + subject="${COMPONENT_NAME} <${REPO_URL}/releases/tag/${EFFECTIVE_TAG}|${EFFECTIVE_TAG}>" + s=0; f=0; k=0 + for r in "${PUSH_DNF_RESULT}" "${PUSH_APT_RESULT}"; do + case "$r" in success) s=$((s+1));; skipped) k=$((k+1));; *) f=$((f+1));; esac + done + total=$((s+f+k)) + if [ "$f" -eq 0 ] && [ "$k" -eq 0 ]; then status="Success"; elif [ "$s" -gt 0 ]; then status="Partial"; elif [ "$k" -eq "$total" ]; then status="Skipped"; else status="Failed"; fi + archs="—" + if [ -f aggregated/manifest.json ]; then a=$(jq -r '(.cells // []) | map(.arch) | unique | join(", ")' aggregated/manifest.json); [ -n "$a" ] && archs="$a"; fi + fields=$(jq -nc --arg channel "${REPO_TYPE}" --arg version "${CV}" --arg buildnum "${CBN}" --arg archs "${archs}" --arg actor "${ACTOR}" --arg unix_ts "${unix_ts}" --arg fallback_ts "${fallback_ts}" '[{title:"Channel",value:("`"+$channel+"`")},{title:"Version",value:("`"+$version+"`")},{title:"Build #",value:("`"+$buildnum+"`")},{title:"Archs",value:("`"+$archs+"`")},{title:"Triggered by",value:$actor},{title:"When",value:("")}]') + result_fields=$(jq -nc --arg dnf "${PUSH_DNF_RESULT}" --arg apt "${PUSH_APT_RESULT}" '[{title:"Push DNF",value:("`"+$dnf+"`")},{title:"Push APT",value:("`"+$apt+"`")}]') + { echo "status=${status}"; echo "subject<<__S__"; echo "${subject}"; echo "__S__"; echo "fields<<__F__"; echo "${fields}"; echo "__F__"; echo "result_fields<<__RF__"; echo "${result_fields}"; echo "__RF__"; } >> "$GITHUB_OUTPUT" + - name: Publish manifest + Slack notification + uses: ./.github/actions/pgedge-build-publisher + with: + manifest_json: ${{ steps.build_manifest.outputs.manifest_json }} + s3_backup_bucket: ${{ secrets.S3_BACKUP_BUCKET_NAME }} + repo_type: ${{ needs.determine-repo-type.outputs.repo-type }} + component_name: ${{ env.COMPONENT_NAME }} + skip_publish: ${{ needs.determine-repo-type.outputs.simulated == 'true' || (needs.push-dnf.result != 'success' && needs.push-apt.result != 'success') }} + status: ${{ steps.compose_slack.outputs.status }} + push_dnf_result: ${{ needs.push-dnf.result }} + push_apt_result: ${{ needs.push-apt.result }} + slack_webhook: ${{ secrets.SLACK_CHANNEL_URL }} + subject: ${{ steps.compose_slack.outputs.subject }} + fields_json: ${{ steps.compose_slack.outputs.fields }} + result_fields_json: ${{ steps.compose_slack.outputs.result_fields }} + tagline: "pgEdge ColdFront release pipeline" + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Upload aggregated manifest as workflow artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: aggregated-manifest + path: aggregated/manifest.json + retention-days: 90 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a3c6624..aeca3b9 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -50,13 +50,34 @@ builds: - amd64 - arm64 -# One tarball per platform containing both binaries — archiver and partitioner -# are deployed together. + # compactor lives in its own Go module (cmd/compactor/go.mod, heavy iceberg-go + # deps), so it is built with dir: pointed at that module. Still pure-Go + # (CGO_ENABLED=0), so it cross-compiles statically like the others. + - id: compactor + binary: compactor + main: . + dir: cmd/compactor + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + +# One tarball per platform containing all three binaries — archiver, +# partitioner and compactor are deployed together. archives: - id: coldfront ids: - archiver - partitioner + - compactor formats: - tar.gz name_template: >- diff --git a/common/build.sh b/common/build.sh new file mode 100644 index 0000000..4c8ec6f --- /dev/null +++ b/common/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Entry point for the pgEdge builder-action. Invoked as: +# common/build.sh +# where is a packaging dir under the repo root that holds a +# common.sh + build-rpm.sh / build-deb.sh (e.g. "packaging/pg_duckdb"). +COMPONENT_NAME=$1 +source "$(dirname "$0")/../${COMPONENT_NAME}/common.sh" + +# common-functions.sh is copied into common/ by the workflow (builder-action) +# from the shared private repo; not committed here. +COMMON_FILE="$(dirname "$0")/common-functions.sh" +if [ -f "$COMMON_FILE" ]; then + source "$COMMON_FILE" +else + echo "Error: $COMMON_FILE not found!" >&2 + exit 1 +fi + +########### +# Main +########### +detect_os_type +prepare +build +post_build diff --git a/packaging/coldfront-bin/build-deb.sh b/packaging/coldfront-bin/build-deb.sh new file mode 100644 index 0000000..d3c2306 --- /dev/null +++ b/packaging/coldfront-bin/build-deb.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUILD_DIR="/tmp/pg_deb_build" +SRC_DIR="${BUILD_DIR}/coldfront-${COLDFRONT_VERSION}" +CWD="$(pwd)" + +export DEBIAN_FRONTEND=noninteractive +ARCH=$(dpkg --print-architecture) # amd64 | arm64 (== GoReleaser arch) + +TAG_VERSION="${COLDFRONT_BRANCH#v}" +ARTIFACT_DIR="${ARTIFACT_DIR:-${CWD}/release-artifacts}" +RELEASE_URL="https://github.com/pgEdge/ColdFront/releases/download/${COLDFRONT_BRANCH}" + +# Prefer the workflow-staged tarball (release-artifacts/); else download. +stage() { + local local_name="$1" remote_name="$2" dest="$3" + if [ -f "${ARTIFACT_DIR}/${local_name}" ]; then + cp "${ARTIFACT_DIR}/${local_name}" "${dest}" + else + wget -q "${RELEASE_URL}/${remote_name}" -O "${dest}" + fi +} + +prepare() { + + setup_apt_build_env + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "Staging ColdFront binaries tarball (${ARCH})..." + rm -rf "$BUILD_DIR" + mkdir -p "$SRC_DIR" + stage "coldfront.tar.gz" "coldfront_${TAG_VERSION}_linux_${ARCH}.tar.gz" \ + "${BUILD_DIR}/coldfront.tar.gz" + tar -C "$SRC_DIR" -xzf "${BUILD_DIR}/coldfront.tar.gz" + + echo "Moving Debian packaging into source directory..." + cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" + + echo "Installing build dependencies..." + cd "$SRC_DIR" + sudo apt-get update + sudo apt-get build-dep -y . +} + +build() { + + cd "$SRC_DIR" + echo "Building Debian package..." + DISTRO=$(lsb_release -cs) + rm -rf debian/changelog + echo "pgedge-coldfront (${COLDFRONT_VERSION}-${COLDFRONT_BUILDNUM}.${DISTRO}) unstable; urgency=low" >> debian/changelog + echo " * Initial Release." >> debian/changelog + echo " -- pgEdge Build Team $(date -R)" >> debian/changelog + dch -D "$DISTRO" --force-distribution -v "${COLDFRONT_VERSION}-${COLDFRONT_BUILDNUM}.${DISTRO}" "pgEdge ColdFront tools ${COLDFRONT_VERSION} for $DISTRO" + + dpkg-buildpackage -us -uc -b +} + +post_build() { + echo "Copying .deb packages to output..." + sudo mkdir -p "/output" + rename_ddeb_packages "$BUILD_DIR" + sudo cp "$BUILD_DIR"/*.deb "/output" || echo "No .deb packages found." +} diff --git a/packaging/coldfront-bin/build-rpm.sh b/packaging/coldfront-bin/build-rpm.sh new file mode 100644 index 0000000..98d417c --- /dev/null +++ b/packaging/coldfront-bin/build-rpm.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -euo pipefail + +RHEL="$(rpm --eval %rhel)" +ARCH=$(uname -m) # x86_64 | aarch64 (rpm arch) +case "$ARCH" in + x86_64) GOARCH=amd64 ;; + aarch64) GOARCH=arm64 ;; + *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; +esac + +# Release assets are named with the full tag version (e.g. 1.0.0). The binaries +# come from the GoReleaser tarball built in the same release run. +TAG_VERSION="${COLDFRONT_BRANCH#v}" +ARTIFACT_DIR="${ARTIFACT_DIR:-$(pwd)/release-artifacts}" +RELEASE_URL="https://github.com/pgEdge/ColdFront/releases/download/${COLDFRONT_BRANCH}" + +# Prefer the workflow-staged tarball (release-artifacts/) so the package cell +# uses THIS run's binaries; else download from the GitHub release. +stage() { + local local_name="$1" remote_name="$2" dest="$3" + if [ -f "${ARTIFACT_DIR}/${local_name}" ]; then + cp "${ARTIFACT_DIR}/${local_name}" "${dest}" + else + wget -q "${RELEASE_URL}/${remote_name}" -O "${dest}" + fi +} + +prepare() { + setup_dnf_build_env + + echo "Copying packaging files..." + cp "${COMPONENT_NAME}/rpm/coldfront.spec" ~/rpmbuild/SPECS/ + + echo "Staging ColdFront binaries tarball (${GOARCH})..." + stage "coldfront.tar.gz" "coldfront_${TAG_VERSION}_linux_${GOARCH}.tar.gz" \ + ~/rpmbuild/SOURCES/coldfront-${COLDFRONT_VERSION}-${ARCH}.tar.gz + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys +} + +build() { + echo "Building RPM and SRPM..." + QA_RPATHS=$(( 0xffff )) rpmbuild -ba ~/rpmbuild/SPECS/coldfront.spec \ + --define "coldfront_version ${COLDFRONT_VERSION}" \ + --define "coldfront_buildnum ${COLDFRONT_BUILDNUM}" +} + +post_build() { + echo "📤 Copying built RPMs to /output..." + mkdir -p /output + cp -v ~/rpmbuild/RPMS/*/*.rpm /output/ || echo "No binary RPMs found" + cp -v ~/rpmbuild/SRPMS/*.src.rpm /output/ || echo "No SRPM found" + + sign_rpms /output/*.rpm + validate_signatures /output/*.rpm +} diff --git a/packaging/coldfront-bin/common.sh b/packaging/coldfront-bin/common.sh new file mode 100644 index 0000000..510ee14 --- /dev/null +++ b/packaging/coldfront-bin/common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# common.sh - ColdFront binaries package (archiver, partitioner, compactor). +# +# PG-INDEPENDENT: these are pure static Go binaries (CGO_ENABLED=0), packaged +# from the GoReleaser tarball built in the SAME release run — not compiled here. +# One build per arch (no PG-major fan-out). + +export COLDFRONT_REPO="https://github.com/pgEdge/ColdFront" +export COLDFRONT_BRANCH="${COMPONENT_BRANCH:-v1.0.0}" +export COLDFRONT_VERSION="${COMPONENT_VERSION:-1.0.0}" +export COLDFRONT_BUILDNUM="${COMPONENT_BUILDNUM:-1}" + +# DEB only: move a pre-release pretag (e.g. BUILDNUM='rc1_1') into the upstream +# VERSION with a leading '~' (1.0.0~rc1, BUILDNUM=1) so '~' sorts pre-releases +# BELOW stable in dpkg/reprepro. Gated on apt-get so RPM keeps the pretag in +# Release (rpmvercmp already sorts rc1_1 below 1). +if command -v apt-get &>/dev/null; then + if [[ "$COLDFRONT_BUILDNUM" == *_* ]]; then + COLDFRONT_PRETAG="${COLDFRONT_BUILDNUM%%_*}" + export COLDFRONT_VERSION="${COLDFRONT_VERSION}~${COLDFRONT_PRETAG}" + COLDFRONT_BUILDNUM="${COLDFRONT_BUILDNUM#*_}" + fi +fi + +export REPO_TYPE="${REPO_TYPE:-daily}" diff --git a/packaging/coldfront-bin/deb/debian/control b/packaging/coldfront-bin/deb/debian/control new file mode 100644 index 0000000..5cb7c28 --- /dev/null +++ b/packaging/coldfront-bin/deb/debian/control @@ -0,0 +1,20 @@ +Source: pgedge-coldfront +Section: database +Priority: optional +Maintainer: pgEdge Build Team +Build-Depends: + debhelper-compat (= 13), +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Homepage: https://github.com/pgEdge/ColdFront + +Package: pgedge-coldfront +Architecture: any +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Description: ColdFront tiered-storage command-line tools + ColdFront provides transparent tiered storage for PostgreSQL (hot PG + cold + Iceberg over object storage). This package contains the ColdFront tools: + archiver (cold-tier writer), partitioner, and compactor (Iceberg + maintenance) — PostgreSQL-version-independent, statically linked binaries. diff --git a/packaging/coldfront-bin/deb/debian/pgedge-coldfront.install b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.install new file mode 100644 index 0000000..87fc33b --- /dev/null +++ b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.install @@ -0,0 +1,4 @@ +archiver usr/bin/ +partitioner usr/bin/ +compactor usr/bin/ +debian/tmp/sbom/* usr/share/pgedge-coldfront/ diff --git a/packaging/coldfront-bin/deb/debian/rules b/packaging/coldfront-bin/deb/debian/rules new file mode 100644 index 0000000..b22ee34 --- /dev/null +++ b/packaging/coldfront-bin/deb/debian/rules @@ -0,0 +1,24 @@ +#!/usr/bin/make -f + +# Prebuilt static Go binaries — nothing to compile or test here. +%: + dh $@ + +override_dh_auto_build: + +override_dh_auto_test: + +override_dh_builddeb: + dh_builddeb -- -Zgzip + +execute_before_dh_install: + # --- SBOM generation and signing (per-format DEB key) --- + mkdir -p debian/tmp/sbom + syft dir:$(CURDIR) -o cyclonedx-json > debian/tmp/sbom/coldfront-sbom.json || exit 1 + KEY_ID=$$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $$5}' | head -n 1); \ + gpg --armor --detach-sign --local-user "$$KEY_ID" \ + --output debian/tmp/sbom/coldfront-sbom.json.asc \ + debian/tmp/sbom/coldfront-sbom.json || exit 1 + +override_dh_installdocs: + dh_installdocs --all README.* diff --git a/packaging/coldfront-bin/deb/debian/source/format b/packaging/coldfront-bin/deb/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/packaging/coldfront-bin/deb/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/coldfront-bin/rpm/coldfront.spec b/packaging/coldfront-bin/rpm/coldfront.spec new file mode 100644 index 0000000..12ad681 --- /dev/null +++ b/packaging/coldfront-bin/rpm/coldfront.spec @@ -0,0 +1,53 @@ +%global sname coldfront + +Name: pgedge-%{sname} +Version: %{coldfront_version} +Release: %{coldfront_buildnum}%{?dist} +Summary: ColdFront tiered-storage command-line tools +License: PostgreSQL +URL: https://github.com/pgEdge/ColdFront +# Prebuilt static Go binaries from the GoReleaser tarball (built in the same +# release run), staged into SOURCES by build-rpm.sh. Arch-specific, no compile. +Source0: %{sname}-%{version}-%{_arch}.tar.gz + +# Pure static Go binaries (CGO_ENABLED=0) — no runtime library dependencies and +# nothing to compile here, so no BuildRequires beyond syft (installed by +# setup_dnf_build_env) for the SBOM. +BuildArch: %{_arch} + +%description +ColdFront provides transparent tiered storage for PostgreSQL (hot PG + cold +Iceberg over object storage). This package contains the ColdFront tools: +archiver (cold-tier writer), partitioner, and compactor (Iceberg maintenance). +These are PostgreSQL-version-independent, statically linked binaries. + +%prep +# The GoReleaser archive has no wrapping directory; -c creates one and extracts. +%setup -q -c -n %{sname}-%{version} + +%build +syft dir:%{_builddir}/%{sname}-%{version} -o cyclonedx-json > %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1); export KEY_ID +gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +%install +%{__rm} -rf %{buildroot} +install -D -m 0755 archiver %{buildroot}%{_bindir}/archiver +install -D -m 0755 partitioner %{buildroot}%{_bindir}/partitioner +install -D -m 0755 compactor %{buildroot}%{_bindir}/compactor +install -D -m 0644 %{sname}-sbom.json %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%files +%license LICENSE.md +%doc README.md +%{_bindir}/archiver +%{_bindir}/partitioner +%{_bindir}/compactor +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%changelog +* Mon Jun 30 2026 Muhammad Aqeel - 1.0.0-1 +- Initial build of the pgEdge ColdFront tools (archiver, partitioner, compactor) diff --git a/packaging/coldfront-duckdb-extensions/build-deb.sh b/packaging/coldfront-duckdb-extensions/build-deb.sh new file mode 100644 index 0000000..fe702c6 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/build-deb.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +CWD="$(pwd)" +export DEBIAN_FRONTEND=noninteractive +ARCH=$(dpkg --print-architecture) # amd64 | arm64 (== GoReleaser/DuckDB arch) +case "$ARCH" in + amd64) DUCKDB_PLATFORM=linux_amd64; VCPKG_TRIPLET=x64-linux ;; + arm64) DUCKDB_PLATFORM=linux_arm64; VCPKG_TRIPLET=arm64-linux ;; + *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; +esac + +BUILD_ROOT="/tmp/cf-duckdb-ext-build" +ICE="${BUILD_ROOT}/duckdb-iceberg" +SRC_DIR="/tmp/pg_deb_build/coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}" + +prepare() { + + setup_apt_build_env + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "Installing duckdb-iceberg build toolchain..." + sudo apt-get install -y \ + build-essential cmake ninja-build git ccache jq wget zip unzip \ + tar autoconf libtool flex bison pkg-config perl + + echo "Cloning + patching duckdb-iceberg (against DuckDB v${DUCKDB_VERSION})..." + rm -rf "$BUILD_ROOT"; mkdir -p "$BUILD_ROOT" + git clone --filter=blob:none --no-checkout "${ICEBERG_REPO}" "$ICE" + git -C "$ICE" fetch --depth 80 origin "${ICEBERG_BRANCH}" + git -C "$ICE" checkout "${ICEBERG_REF}" + git -C "$ICE" submodule update --init --recursive --depth 1 --jobs 8 + rm -rf "$ICE/duckdb" + git clone --depth 1 --branch "v${DUCKDB_VERSION}" --recurse-submodules "${DUCKDB_REPO}" "$ICE/duckdb" + git clone "${VCPKG_REPO}" "${BUILD_ROOT}/vcpkg" + "${BUILD_ROOT}/vcpkg/bootstrap-vcpkg.sh" -disableMetrics + + cp "${CWD}/docker/iceberg-azure-extension-config-v15.cmake" "$ICE/extension_config.cmake" + for p in iceberg-bakery-aware-commit-refresh-v15 \ + iceberg-manifest-list-format-version-v15 \ + iceberg-manifest-content-v15 \ + iceberg-data-file-format-v15; do + git -C "$ICE" apply --check "${CWD}/docker/${p}.patch" + git -C "$ICE" apply "${CWD}/docker/${p}.patch" + done + + echo "Building extensions (make release)..." + ( + cd "$ICE" + export VCPKG_TOOLCHAIN_PATH="${BUILD_ROOT}/vcpkg/scripts/buildsystems/vcpkg.cmake" + export VCPKG_ROOT="${BUILD_ROOT}/vcpkg" + export VCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" VCPKG_HOST_TRIPLET="$VCPKG_TRIPLET" + export USE_MERGED_VCPKG_MANIFEST=1 + export EXT_CONFIG="${ICE}/extension_config.cmake" + export OVERRIDE_GIT_DESCRIBE="v${DUCKDB_VERSION}" + # Cap parallelism to bound peak memory (DuckDB can OOM at high -j). + export CMAKE_BUILD_PARALLEL_LEVEL=4 + make -j4 release + ) + + echo "Staging built extensions + docs + debian packaging..." + rm -rf "$SRC_DIR"; mkdir -p "$SRC_DIR" + for e in iceberg avro azure postgres_scanner; do + cp "${ICE}/build/release/extension/${e}/${e}.duckdb_extension" "$SRC_DIR/" + done + cp "${COMPONENT_NAME}/config/coldfront-duckdb-extensions.conf.sample" "$SRC_DIR/" + cp "${CWD}/LICENSE.md" "${CWD}/README.md" "$SRC_DIR/" + cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" + + # Generate the .install with the real versioned/platform path. DuckDB locates + # extensions under /v// — the version segment is the + # clean engine version (DUCKDB_VERSION), not the ~-decorated package version. + cd "$SRC_DIR" + extrel="${COLDFRONT_EXTDIR#/}" + dest="${extrel}/v${DUCKDB_VERSION}/${DUCKDB_PLATFORM}" + { + for e in iceberg avro azure postgres_scanner; do + echo "${e}.duckdb_extension ${dest}/" + done + echo "coldfront-duckdb-extensions.conf.sample usr/share/pgedge-coldfront-duckdb-extensions/" + echo "debian/tmp/sbom/* usr/share/pgedge-coldfront-duckdb-extensions/" + } > debian/pgedge-coldfront-duckdb-extensions.install + + echo "Installing build dependencies..." + sudo apt-get update + sudo apt-get build-dep -y . +} + +build() { + cd "$SRC_DIR" + echo "Building Debian package..." + DISTRO=$(lsb_release -cs) + rm -rf debian/changelog + echo "pgedge-coldfront-duckdb-extensions (${DUCKDB_EXT_VERSION}-${DUCKDB_EXT_BUILDNUM}.${DISTRO}) unstable; urgency=low" >> debian/changelog + echo " * Initial Release." >> debian/changelog + echo " -- pgEdge Build Team $(date -R)" >> debian/changelog + dch -D "$DISTRO" --force-distribution -v "${DUCKDB_EXT_VERSION}-${DUCKDB_EXT_BUILDNUM}.${DISTRO}" "pgEdge ColdFront DuckDB extensions ${DUCKDB_EXT_VERSION} for $DISTRO" + + dpkg-buildpackage -us -uc -b +} + +post_build() { + echo "Copying .deb packages to output..." + sudo mkdir -p "/output" + # dpkg-buildpackage writes the .deb to the parent of the source dir. + rename_ddeb_packages "/tmp/pg_deb_build" + sudo cp /tmp/pg_deb_build/*.deb "/output" || echo "No .deb packages found." +} diff --git a/packaging/coldfront-duckdb-extensions/build-rpm.sh b/packaging/coldfront-duckdb-extensions/build-rpm.sh new file mode 100644 index 0000000..80bc0e7 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/build-rpm.sh @@ -0,0 +1,110 @@ +#!/bin/bash +set -euo pipefail + +RHEL="$(rpm --eval %rhel)" +CWD="$(pwd)" +ARCH=$(uname -m) +case "$ARCH" in + x86_64) DUCKDB_PLATFORM=linux_amd64; VCPKG_TRIPLET=x64-linux ;; + aarch64) DUCKDB_PLATFORM=linux_arm64; VCPKG_TRIPLET=arm64-linux ;; + *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; +esac + +BUILD_ROOT="/tmp/cf-duckdb-ext-build" +ICE="${BUILD_ROOT}/duckdb-iceberg" +STAGE="/tmp/coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}" + +prepare() { + setup_dnf_build_env + + echo "Installing duckdb-iceberg build toolchain..." + # DuckDB 1.5.4's extensions (e.g. postgres_scanner) need a modern C++ compiler: + # EL9's native gcc 11 rejects them, so use gcc-toolset-14 (gcc 14, as the docker + # recipe does); EL10 ships gcc 14 natively. libasan/libubsan: the + # extension_configuration phase builds a Debug binary that links ASan. + if [ "${RHEL}" -lt 10 ]; then + dnf install -y gcc-toolset-14-gcc-c++ gcc-toolset-14-libasan-devel gcc-toolset-14-libubsan-devel + else + dnf install -y gcc gcc-c++ libasan libubsan + fi + # perl-core: openssl-from-source (vcpkg) Configure needs FindBin + other core + # perl modules that EL's minimal perl splits out; perl-core pulls the full set. + dnf install -y make ninja-build perl-core perl-FindBin perl-IPC-Cmd perl-Time-Piece \ + ccache jq wget zip unzip tar autoconf libtool kernel-headers cmake git flex bison + + echo "Cloning + patching duckdb-iceberg (against DuckDB v${DUCKDB_VERSION})..." + rm -rf "$BUILD_ROOT"; mkdir -p "$BUILD_ROOT" + git clone --filter=blob:none --no-checkout "${ICEBERG_REPO}" "$ICE" + git -C "$ICE" fetch --depth 80 origin "${ICEBERG_BRANCH}" + git -C "$ICE" checkout "${ICEBERG_REF}" + git -C "$ICE" submodule update --init --recursive --depth 1 --jobs 8 + # Pin the duckdb submodule to the engine version pg_duckdb links. + rm -rf "$ICE/duckdb" + git clone --depth 1 --branch "v${DUCKDB_VERSION}" --recurse-submodules "${DUCKDB_REPO}" "$ICE/duckdb" + git clone "${VCPKG_REPO}" "${BUILD_ROOT}/vcpkg" + "${BUILD_ROOT}/vcpkg/bootstrap-vcpkg.sh" -disableMetrics + + # extension set (avro/azure/postgres_scanner refs) + the 4 ColdFront patches + # live in the repo's docker/ dir; git apply --check fails loudly on patch rot. + cp "${CWD}/docker/iceberg-azure-extension-config-v15.cmake" "$ICE/extension_config.cmake" + for p in iceberg-bakery-aware-commit-refresh-v15 \ + iceberg-manifest-list-format-version-v15 \ + iceberg-manifest-content-v15 \ + iceberg-data-file-format-v15; do + git -C "$ICE" apply --check "${CWD}/docker/${p}.patch" + git -C "$ICE" apply "${CWD}/docker/${p}.patch" + done + + echo "Building extensions (make release)..." + ( + cd "$ICE" + # Activate gcc-toolset-14 on EL9 (gcc 14 for the C++ extensions); no-op on + # EL10 / where the toolset is absent (native gcc used). + # shellcheck disable=SC1090 + source /opt/rh/gcc-toolset-*/enable 2>/dev/null || true + export VCPKG_TOOLCHAIN_PATH="${BUILD_ROOT}/vcpkg/scripts/buildsystems/vcpkg.cmake" + export VCPKG_ROOT="${BUILD_ROOT}/vcpkg" + export VCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" VCPKG_HOST_TRIPLET="$VCPKG_TRIPLET" + export USE_MERGED_VCPKG_MANIFEST=1 + export EXT_CONFIG="${ICE}/extension_config.cmake" + export OVERRIDE_GIT_DESCRIBE="v${DUCKDB_VERSION}" + # Cap parallelism to bound peak memory (DuckDB can OOM at high -j); matches + # pg_duckdb and keeps a single cell within a GitHub runner / the Docker VM. + export CMAKE_BUILD_PARALLEL_LEVEL=4 + make -j4 release + ) + + echo "Staging built extensions + docs for packaging..." + rm -rf "$STAGE"; mkdir -p "$STAGE" + for e in iceberg avro azure postgres_scanner; do + cp "${ICE}/build/release/extension/${e}/${e}.duckdb_extension" "$STAGE/" + done + cp "${COMPONENT_NAME}/config/coldfront-duckdb-extensions.conf.sample" "$STAGE/" + cp "${CWD}/LICENSE.md" "${CWD}/README.md" "$STAGE/" + cp "${COMPONENT_NAME}/rpm/coldfront-duckdb-extensions.spec" ~/rpmbuild/SPECS/ + tar czf ~/rpmbuild/SOURCES/coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}.tar.gz \ + -C /tmp "coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}" + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys +} + +build() { + echo "Building RPM and SRPM..." + QA_RPATHS=$(( 0xffff )) rpmbuild -ba ~/rpmbuild/SPECS/coldfront-duckdb-extensions.spec \ + --define "duckdb_ext_version ${DUCKDB_EXT_VERSION}" \ + --define "duckdb_ext_buildnum ${DUCKDB_EXT_BUILDNUM}" \ + --define "duckdb_version ${DUCKDB_VERSION}" \ + --define "duckdb_platform ${DUCKDB_PLATFORM}" \ + --define "extdir ${COLDFRONT_EXTDIR}" +} + +post_build() { + echo "📤 Copying built RPMs to /output..." + mkdir -p /output + cp -v ~/rpmbuild/RPMS/*/*.rpm /output/ || echo "No binary RPMs found" + cp -v ~/rpmbuild/SRPMS/*.src.rpm /output/ || echo "No SRPM found" + + sign_rpms /output/*.rpm + validate_signatures /output/*.rpm +} diff --git a/packaging/coldfront-duckdb-extensions/common.sh b/packaging/coldfront-duckdb-extensions/common.sh new file mode 100644 index 0000000..5e3ab8c --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/common.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# common.sh - ColdFront DuckDB extensions (iceberg, avro, azure, postgres_scanner). +# +# PG-INDEPENDENT, arch+distro-keyed. Built from the patched duckdb-iceberg tree +# against DuckDB ${DUCKDB_VERSION} and placed under +# ${COLDFRONT_EXTDIR}/v${DUCKDB_VERSION}/linux_/ +# pg_duckdb's DuckDB engine locates extensions under v//, +# so that path segment MUST be the clean engine version (see below). + +# --- DuckDB ENGINE version: the SINGLE source of truth -------------------------- +# Drives the install-path segment, OVERRIDE_GIT_DESCRIBE, and the duckdb clone +# tag. It is ALWAYS the clean engine version (e.g. 1.5.4) — never decorated with a +# pre-release pretag, because DuckDB looks up extensions under v/, +# which is fixed regardless of the ColdFront release's rc/beta status. Keep in +# lockstep with pg_duckdb's bundled DuckDB. Overridable via COMPONENT_VERSION. +export DUCKDB_VERSION="${COMPONENT_VERSION:-1.5.4}" + +# --- Package version -------------------------------------------------------------- +# The rpm/deb package Version. Starts equal to the engine version; on DEB a +# pre-release pretag is folded in with '~' for correct apt ordering (below). The +# package version may become 1.5.4~rc1, but DUCKDB_VERSION stays 1.5.4. +export DUCKDB_EXT_VERSION="$DUCKDB_VERSION" +export DUCKDB_EXT_BUILDNUM="${COMPONENT_BUILDNUM:-1}" + +# --- Build pins ------------------------------------------------------------------- +# duckdb-iceberg: the 4 patches target ICEBERG_REF (branch fetched first so the +# ref resolves). avro/azure/postgres_scanner refs live in the extension_config +# cmake (packaging copies docker/iceberg-azure-extension-config-v15.cmake). +export ICEBERG_REPO="https://github.com/duckdb/duckdb-iceberg" +export ICEBERG_BRANCH="${ICEBERG_BRANCH:-v1.5-variegata}" +export ICEBERG_REF="${ICEBERG_REF:-0fad545a}" +export DUCKDB_REPO="https://github.com/duckdb/duckdb" +export VCPKG_REPO="https://github.com/microsoft/vcpkg" + +# --- Install location (PG-independent) -------------------------------------------- +# DuckDB appends v/linux_/. Point duckdb.extension_directory at this +# in postgresql.conf (see the shipped config sample). +export COLDFRONT_EXTDIR="${COLDFRONT_EXTDIR:-/usr/lib/pgedge/coldfront/duckdb-extensions}" + +# DEB only: fold a pre-release pretag (BUILDNUM='rc1_1') into the PACKAGE version +# with '~' (1.5.4~rc1, BUILDNUM=1) so '~' sorts pre-releases below stable in +# dpkg/reprepro. DUCKDB_VERSION (path/build) stays clean. RPM keeps the pretag in +# Release (rpmvercmp sorts rc1_1 below 1). +if command -v apt-get &>/dev/null; then + if [[ "$DUCKDB_EXT_BUILDNUM" == *_* ]]; then + DUCKDB_EXT_PRETAG="${DUCKDB_EXT_BUILDNUM%%_*}" + export DUCKDB_EXT_VERSION="${DUCKDB_VERSION}~${DUCKDB_EXT_PRETAG}" + DUCKDB_EXT_BUILDNUM="${DUCKDB_EXT_BUILDNUM#*_}" + fi +fi + +export REPO_TYPE="${REPO_TYPE:-daily}" diff --git a/packaging/coldfront-duckdb-extensions/config/coldfront-duckdb-extensions.conf.sample b/packaging/coldfront-duckdb-extensions/config/coldfront-duckdb-extensions.conf.sample new file mode 100644 index 0000000..c79bcdd --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/config/coldfront-duckdb-extensions.conf.sample @@ -0,0 +1,35 @@ +# ============================================================================= +# ColdFront DuckDB extensions — required configuration +# ============================================================================= +# +# This package installs the patched DuckDB extensions (iceberg, avro, azure, +# postgres_scanner) under: +# +# /usr/lib/pgedge/coldfront/duckdb-extensions/v/linux_/ +# +# For pg_duckdb to load them from there (instead of downloading unpatched copies), +# add the three settings below to your cluster and RESTART PostgreSQL. +# +# NOTE: duckdb.extension_directory is read at server start only — a reload is not +# enough; you must restart. autoinstall_known_extensions MUST stay off: with it +# on, a missing extension file makes DuckDB silently download the UNPATCHED +# upstream extension. +# +# ----------------------------------------------------------------------------- +# Option A — append to postgresql.conf, then restart: +# ----------------------------------------------------------------------------- +# +# duckdb.extension_directory = '/usr/lib/pgedge/coldfront/duckdb-extensions' +# duckdb.allow_unsigned_extensions = true +# duckdb.autoinstall_known_extensions = false +# +# ----------------------------------------------------------------------------- +# Option B — via ALTER SYSTEM (writes postgresql.auto.conf), then restart: +# ----------------------------------------------------------------------------- +# +# ALTER SYSTEM SET duckdb.extension_directory = '/usr/lib/pgedge/coldfront/duckdb-extensions'; +# ALTER SYSTEM SET duckdb.allow_unsigned_extensions = true; +# ALTER SYSTEM SET duckdb.autoinstall_known_extensions = false; +# -- then: restart the PostgreSQL service +# +# ============================================================================= diff --git a/packaging/coldfront-duckdb-extensions/deb/debian/control b/packaging/coldfront-duckdb-extensions/deb/debian/control new file mode 100644 index 0000000..8dbba96 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/deb/debian/control @@ -0,0 +1,21 @@ +Source: pgedge-coldfront-duckdb-extensions +Section: database +Priority: optional +Maintainer: pgEdge Build Team +Build-Depends: + debhelper-compat (= 13), +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Homepage: https://github.com/pgEdge/ColdFront + +Package: pgedge-coldfront-duckdb-extensions +Architecture: any +Depends: + ${misc:Depends}, + ${shlibs:Depends}, +Description: Patched DuckDB extensions for ColdFront tiered storage + The patched DuckDB extensions ColdFront's cold tier needs — iceberg, avro, + azure, and postgres_scanner — carrying ColdFront's bakery-aware-commit-refresh + and interop patches. They are loaded by pg_duckdb; point + duckdb.extension_directory at the install dir (see the installed config sample) + so pg_duckdb loads these instead of downloading the unpatched upstream builds. diff --git a/packaging/coldfront-duckdb-extensions/deb/debian/rules b/packaging/coldfront-duckdb-extensions/deb/debian/rules new file mode 100644 index 0000000..de4a83a --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/deb/debian/rules @@ -0,0 +1,32 @@ +#!/usr/bin/make -f + +# Prebuilt .duckdb_extension binaries (built by build-deb.sh) — nothing to +# compile or test here. +%: + dh $@ + +override_dh_auto_build: + +override_dh_auto_test: + +override_dh_strip: + # .duckdb_extension files carry a DuckDB metadata footer that strip would + # corrupt, making DuckDB reject them — do NOT strip. + +override_dh_dwz: + # Same reason: leave the extension binaries untouched. + +override_dh_builddeb: + dh_builddeb -- -Zgzip + +execute_before_dh_install: + # --- SBOM generation and signing (per-format DEB key) --- + mkdir -p debian/tmp/sbom + syft dir:$(CURDIR) -o cyclonedx-json > debian/tmp/sbom/coldfront-duckdb-extensions-sbom.json || exit 1 + KEY_ID=$$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $$5}' | head -n 1); \ + gpg --armor --detach-sign --local-user "$$KEY_ID" \ + --output debian/tmp/sbom/coldfront-duckdb-extensions-sbom.json.asc \ + debian/tmp/sbom/coldfront-duckdb-extensions-sbom.json || exit 1 + +override_dh_installdocs: + dh_installdocs --all README.* diff --git a/packaging/coldfront-duckdb-extensions/deb/debian/source/format b/packaging/coldfront-duckdb-extensions/deb/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/deb/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/coldfront-duckdb-extensions/rpm/coldfront-duckdb-extensions.spec b/packaging/coldfront-duckdb-extensions/rpm/coldfront-duckdb-extensions.spec new file mode 100644 index 0000000..61b4081 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/rpm/coldfront-duckdb-extensions.spec @@ -0,0 +1,70 @@ +%global sname coldfront-duckdb-extensions + +# DuckDB .duckdb_extension files are shared objects with a DuckDB metadata footer +# appended; rpm's strip / debuginfo extraction would corrupt that footer and make +# DuckDB reject the extension. Disable all binary post-processing. +%global debug_package %{nil} +%global __strip /bin/true +%global __brp_strip %{nil} +%global __brp_strip_static_archive %{nil} +%global __brp_strip_comment_note %{nil} + +Name: pgedge-%{sname} +Version: %{duckdb_ext_version} +Release: %{duckdb_ext_buildnum}%{?dist} +Summary: Patched DuckDB extensions for ColdFront tiered storage +License: MIT +URL: https://github.com/pgEdge/ColdFront +# Prebuilt .duckdb_extension binaries (built by build-rpm.sh from the patched +# duckdb-iceberg tree against DuckDB v%{duckdb_version}), staged into SOURCES. +Source0: %{sname}-%{version}.tar.gz + +%description +The patched DuckDB extensions ColdFront's cold tier needs — iceberg, avro, +azure, and postgres_scanner — built against DuckDB v%{duckdb_version} and +carrying ColdFront's bakery-aware-commit-refresh and interop patches. They are +loaded by pg_duckdb; point duckdb.extension_directory at %{extdir} (see the +installed config sample) so pg_duckdb loads these instead of downloading the +unpatched upstream builds. + +%prep +# build-rpm.sh's tarball already has a %{sname}-%{version}/ wrapping dir, so a +# plain %setup (no -c) extracts + cd's into it (files at the build CWD). +%setup -q -n %{sname}-%{version} + +%build +# No compilation here — the extensions are prebuilt. Generate + sign the SBOM. +syft dir:%{_builddir}/%{sname}-%{version} -o cyclonedx-json > %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1); export KEY_ID +gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +%install +%{__rm} -rf %{buildroot} +# DuckDB locates extensions under /v// — the version +# segment is the clean engine version (%{duckdb_version}), NOT the package +# Version (which may carry a pre-release pretag). +extdir_full="%{buildroot}%{extdir}/v%{duckdb_version}/%{duckdb_platform}" +mkdir -p "$extdir_full" +for e in iceberg avro azure postgres_scanner; do + install -m 0644 "${e}.duckdb_extension" "$extdir_full/${e}.duckdb_extension" +done +install -D -m 0644 %{sname}.conf.sample %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}.conf.sample +install -D -m 0644 %{sname}-sbom.json %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%files +%license LICENSE.md +%doc README.md +%dir %{extdir} +%{extdir}/v%{duckdb_version}/%{duckdb_platform}/iceberg.duckdb_extension +%{extdir}/v%{duckdb_version}/%{duckdb_platform}/avro.duckdb_extension +%{extdir}/v%{duckdb_version}/%{duckdb_platform}/azure.duckdb_extension +%{extdir}/v%{duckdb_version}/%{duckdb_platform}/postgres_scanner.duckdb_extension +%{_datadir}/pgedge-%{sname}/%{sname}.conf.sample +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%changelog +* Tue Jul 01 2026 Muhammad Aqeel - 1.5.4-1 +- Initial build of the ColdFront patched DuckDB extensions (DuckDB 1.5.4) diff --git a/packaging/coldfront/build-deb.sh b/packaging/coldfront/build-deb.sh new file mode 100644 index 0000000..022d042 --- /dev/null +++ b/packaging/coldfront/build-deb.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Environment variables +BUILD_DIR="/tmp/pg_deb_build" +SRC_DIR="${BUILD_DIR}/coldfront-${COLDFRONT_VERSION}" + +CWD="$(pwd)" + +export DEBIAN_FRONTEND=noninteractive + +prepare() { + + setup_apt_build_env + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "Packing the coldfront extension source..." + rm -rf "$BUILD_DIR" + mkdir -p "$SRC_DIR" + cp -rp "${CWD}/extension/coldfront/." "$SRC_DIR/" + # License + README travel with the package (PostgreSQL-licensed). + cp "${CWD}/LICENSE.md" "${CWD}/README.md" "$SRC_DIR/" + + echo "Moving Debian packaging into source directory..." + cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" + cd "$SRC_DIR" + cp debian/control.in debian/control + sed -i "s|PG_MAJOR_VERSION|${PG_MAJOR_VERSION}|g" debian/control + mv debian/pgedge-postgresql-coldfront.install \ + debian/pgedge-postgresql-${PG_MAJOR_VERSION}-coldfront.install + sed -i "s|PG_MAJOR_VERSION|${PG_MAJOR_VERSION}|g" \ + debian/pgedge-postgresql-${PG_MAJOR_VERSION}-coldfront.install + + echo "Installing build dependencies..." + sudo apt-get update + sudo apt-get build-dep -y . +} + +build() { + + cd "$SRC_DIR" + echo "Building Debian package..." + DISTRO=$(lsb_release -cs) + rm -rf debian/changelog + echo "pgedge-coldfront (${COLDFRONT_VERSION}-${COLDFRONT_BUILDNUM}.${DISTRO}) unstable; urgency=low" >> debian/changelog + echo " * Initial Release." >> debian/changelog + echo " -- pgEdge Build Team $(date -R)" >> debian/changelog + dch -D "$DISTRO" --force-distribution -v "${COLDFRONT_VERSION}-${COLDFRONT_BUILDNUM}.${DISTRO}" "pgEdge ColdFront ${COLDFRONT_VERSION} for $DISTRO" + + PATH=/usr/lib/postgresql/${PG_MAJOR_VERSION}/bin:$PATH dpkg-buildpackage -us -uc -b +} + +post_build() { + echo "Copying .deb packages to output..." + sudo mkdir -p "/output" + # Rename .ddeb files to .deb files + rename_ddeb_packages "$BUILD_DIR" + sudo cp "$BUILD_DIR"/*.deb "/output" || echo "No .deb packages found." +} diff --git a/packaging/coldfront/build-rpm.sh b/packaging/coldfront/build-rpm.sh new file mode 100644 index 0000000..2e535c8 --- /dev/null +++ b/packaging/coldfront/build-rpm.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -euo pipefail + +RHEL="$(rpm --eval %rhel)" + +prepare() { + setup_dnf_build_env + + echo "Copying packaging files..." + cp "${COMPONENT_NAME}/rpm/coldfront.spec" ~/rpmbuild/SPECS/ + + echo "Packing the coldfront extension source..." + rm -rf "/tmp/coldfront-${COLDFRONT_VERSION}" + cp -rp extension/coldfront "/tmp/coldfront-${COLDFRONT_VERSION}" + # License + README travel with the package (PostgreSQL-licensed). + cp LICENSE.md README.md "/tmp/coldfront-${COLDFRONT_VERSION}/" + tar czf ~/rpmbuild/SOURCES/coldfront-${COLDFRONT_VERSION}.tar.gz \ + -C /tmp "coldfront-${COLDFRONT_VERSION}" + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "🔧 Installing RPM build dependencies..." + dnf builddep -y \ + --define "pgmajorversion ${PG_MAJOR_VERSION}" \ + --define "pginstdir /usr/pgsql-${PG_MAJOR_VERSION}" \ + --define "coldfront_version ${COLDFRONT_VERSION}" \ + --define "coldfront_buildnum ${COLDFRONT_BUILDNUM}" \ + ~/rpmbuild/SPECS/coldfront.spec +} + +build() { + echo "Building RPM and SRPM..." + QA_RPATHS=$(( 0xffff )) rpmbuild -ba ~/rpmbuild/SPECS/coldfront.spec \ + --define "pgmajorversion ${PG_MAJOR_VERSION}" \ + --define "pginstdir /usr/pgsql-${PG_MAJOR_VERSION}" \ + --define "coldfront_version ${COLDFRONT_VERSION}" \ + --define "coldfront_buildnum ${COLDFRONT_BUILDNUM}" +} + +post_build() { + echo "📤 Copying built RPMs to /output..." + mkdir -p /output + cp -v ~/rpmbuild/RPMS/*/*.rpm /output/ || echo "No binary RPMs found" + cp -v ~/rpmbuild/SRPMS/*.src.rpm /output/ || echo "No SRPM found" + + sign_rpms /output/*.rpm + validate_signatures /output/*.rpm +} diff --git a/packaging/coldfront/common.sh b/packaging/coldfront/common.sh new file mode 100644 index 0000000..27d57e9 --- /dev/null +++ b/packaging/coldfront/common.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# common.sh - Common environment variables for the ColdFront extension package. + +# Build once per PostgreSQL major version (fan-out across the matrix). +PER_PG_VERSION=true + +# Default PostgreSQL version and derived values +export PG_VERSION="${PG_VERSION:-17.7}" +export PG_MAJOR_VERSION="$(echo "$PG_VERSION" | cut -d. -f1)" + +# The coldfront extension source lives in THIS repo (extension/coldfront), so +# there is no upstream repo/branch to fetch. The package version is the +# ColdFront release version (tag-driven via COMPONENT_VERSION); the extension's +# SQL version (coldfront.control default_version) is independent of it. +export COLDFRONT_VERSION="${COMPONENT_VERSION:-1.0.0}" +export COLDFRONT_BUILDNUM="${COMPONENT_BUILDNUM:-1}" + +# DEB only: move a pre-release pretag (e.g. BUILDNUM='rc1_1') into the upstream +# VERSION with a leading '~' (1.0.0~rc1, BUILDNUM=1) so '~' sorts pre-releases +# BELOW stable in dpkg/reprepro. Gated on apt-get so RPM keeps the pretag in +# Release (rpmvercmp already sorts rc1_1 below 1). +if command -v apt-get &>/dev/null; then + if [[ "$COLDFRONT_BUILDNUM" == *_* ]]; then + COLDFRONT_PRETAG="${COLDFRONT_BUILDNUM%%_*}" + export COLDFRONT_VERSION="${COLDFRONT_VERSION}~${COLDFRONT_PRETAG}" + COLDFRONT_BUILDNUM="${COLDFRONT_BUILDNUM#*_}" + fi +fi + +export REPO_TYPE="${REPO_TYPE:-daily}" diff --git a/packaging/coldfront/deb/debian/control.in b/packaging/coldfront/deb/debian/control.in new file mode 100644 index 0000000..e36a9e1 --- /dev/null +++ b/packaging/coldfront/deb/debian/control.in @@ -0,0 +1,25 @@ +Source: pgedge-coldfront +Section: database +Priority: optional +Maintainer: pgEdge Build Team +Build-Depends: + debhelper-compat (= 13), + pgedge-postgresql-all , + pgedge-postgresql-server-dev-PG_MAJOR_VERSION, + pgedge-postgresql-server-dev-all, +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Homepage: https://github.com/pgEdge/ColdFront + +Package: pgedge-postgresql-PG_MAJOR_VERSION-coldfront +Architecture: any +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + pgedge-postgresql-PG_MAJOR_VERSION, + pgedge-postgresql-PG_MAJOR_VERSION-pg-duckdb, +Description: Transparent tiered storage for PostgreSQL (hot PG + cold Iceberg) + ColdFront routes DML on tiered views across a hot tier (PostgreSQL) and a + cold tier (Apache Iceberg over object storage), with cold reads/writes served + through pg_duckdb. This package contains the coldfront PostgreSQL extension + (server module + SQL). diff --git a/packaging/coldfront/deb/debian/pgedge-postgresql-coldfront.install b/packaging/coldfront/deb/debian/pgedge-postgresql-coldfront.install new file mode 100644 index 0000000..19f2a62 --- /dev/null +++ b/packaging/coldfront/deb/debian/pgedge-postgresql-coldfront.install @@ -0,0 +1 @@ +debian/tmp/sbom/* usr/lib/postgresql/PG_MAJOR_VERSION/sbom/ diff --git a/packaging/coldfront/deb/debian/pgversions b/packaging/coldfront/deb/debian/pgversions new file mode 100644 index 0000000..21244ca --- /dev/null +++ b/packaging/coldfront/deb/debian/pgversions @@ -0,0 +1 @@ +15+ diff --git a/packaging/coldfront/deb/debian/rules b/packaging/coldfront/deb/debian/rules new file mode 100644 index 0000000..3dc0790 --- /dev/null +++ b/packaging/coldfront/deb/debian/rules @@ -0,0 +1,30 @@ +#!/usr/bin/make -f + +%: + dh $@ + +override_dh_auto_build: + # with_llvm=no matches the proven ColdFront recipe (no JIT bitcode shipped). + USE_PGXS=1 dh_auto_build -- with_llvm=no + +override_dh_auto_install: + USE_PGXS=1 dh_auto_install -- with_llvm=no + +override_dh_auto_test: + # coldfront's regression suite needs a running server and is not + # packaging-relevant; skip it during the build. + +override_dh_builddeb: + dh_builddeb -- -Zgzip + +execute_before_dh_install: + # --- SBOM generation and signing (per-format DEB key) --- + mkdir -p debian/tmp/sbom + syft dir:$(CURDIR) -o cyclonedx-json > debian/tmp/sbom/coldfront-sbom.json || exit 1 + KEY_ID=$$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $$5}' | head -n 1); \ + gpg --armor --detach-sign --local-user "$$KEY_ID" \ + --output debian/tmp/sbom/coldfront-sbom.json.asc \ + debian/tmp/sbom/coldfront-sbom.json || exit 1 + +override_dh_installdocs: + dh_installdocs --all README.* diff --git a/packaging/coldfront/deb/debian/source/format b/packaging/coldfront/deb/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/packaging/coldfront/deb/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/coldfront/rpm/coldfront.spec b/packaging/coldfront/rpm/coldfront.spec new file mode 100644 index 0000000..3032b6f --- /dev/null +++ b/packaging/coldfront/rpm/coldfront.spec @@ -0,0 +1,61 @@ +%global sname coldfront + +Name: pgedge-%{sname}_%{pgmajorversion} +Version: %{coldfront_version} +Release: %{coldfront_buildnum}%{?dist} +Summary: Transparent tiered storage for PostgreSQL (hot PG + cold Iceberg) +License: PostgreSQL +URL: https://github.com/pgEdge/ColdFront +# Local source: the coldfront extension tree (extension/coldfront) packed by +# build-rpm.sh, with LICENSE.md + README.md added at the top level. +Source0: %{sname}-%{version}.tar.gz + +BuildRequires: pgedge-postgresql%{pgmajorversion}-devel +BuildRequires: gcc +BuildRequires: make +BuildRequires: redhat-rpm-config +BuildRequires: diffutils +Requires: pgedge-postgresql%{pgmajorversion}-server +# coldfront drives DuckDB through pg_duckdb (shared_preload_libraries + +# duckdb.raw_query); it is non-functional without it. +Requires: pgedge-pg-duckdb_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} + +%description +ColdFront provides transparent tiered storage for PostgreSQL: DML on tiered +views is routed across a hot tier (PostgreSQL) and a cold tier (Apache Iceberg +over object storage), with cold reads/writes served through pg_duckdb. This +package contains the coldfront PostgreSQL extension (server module + SQL). + +%prep +%setup -q -n %{sname}-%{version} + +%build +# Pure PGXS C build (links libpq for the XactCallback loopback). with_llvm=no +# matches the proven ColdFront recipe (no JIT bitcode shipped). +USE_PGXS=1 PATH=%{pginstdir}/bin:$PATH %{__make} with_llvm=no + +syft dir:%{_builddir}/%{sname}-%{version} -o cyclonedx-json > %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1); export KEY_ID +gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +%install +%{__rm} -rf %{buildroot} +USE_PGXS=1 PATH=%{pginstdir}/bin:$PATH %{__make} install DESTDIR=%{buildroot} with_llvm=no +mkdir -p %{buildroot}/%{pginstdir}/sbom +install -p -m 0644 %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json %{buildroot}/%{pginstdir}/sbom/%{sname}-sbom.json +install -p -m 0644 %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{buildroot}/%{pginstdir}/sbom/%{sname}-sbom.json.asc + +%files +%doc README.md +%license LICENSE.md +%{pginstdir}/lib/%{sname}.so +%{pginstdir}/share/extension/%{sname}.control +%{pginstdir}/share/extension/%{sname}*.sql +%{pginstdir}/sbom/%{sname}-sbom.json +%{pginstdir}/sbom/%{sname}-sbom.json.asc + +%changelog +* Mon Jun 30 2026 Muhammad Aqeel - 1.0.0-1 +- Initial build of the pgEdge ColdFront extension diff --git a/packaging/pg_duckdb/build-deb.sh b/packaging/pg_duckdb/build-deb.sh new file mode 100644 index 0000000..e16870d --- /dev/null +++ b/packaging/pg_duckdb/build-deb.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Environment variables +BUILD_DIR="/tmp/pg_deb_build" +SRC_DIR="${BUILD_DIR}/pg_duckdb-${PG_DUCKDB_VERSION}" + +CWD="$(pwd)" + +export DEBIAN_FRONTEND=noninteractive + +prepare() { + + setup_apt_build_env + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "Vendoring pg_duckdb @ ${PG_DUCKDB_COMMIT} with bundled DuckDB submodule (v1.5.4)..." + rm -rf "$BUILD_DIR" + mkdir -p "$BUILD_DIR" + git init -q "$SRC_DIR" + ( + cd "$SRC_DIR" + git remote add origin "${PG_DUCKDB_REPO}" + git fetch -q --depth 1 origin "${PG_DUCKDB_COMMIT}" + git checkout -q FETCH_HEAD + git submodule update --init --recursive --depth 1 + # See build-rpm.sh: GitHub tarballs omit submodules and the Makefile gates + # the DuckDB build on a `.git/modules/third_party/duckdb/HEAD` marker. Drop + # .git but leave the marker so `make` treats the submodule as checked out. + rm -rf .git + mkdir -p .git/modules/third_party/duckdb + touch .git/modules/third_party/duckdb/HEAD + # pgEdge propagates -fexcess-precision=standard into CXXFLAGS, which gcc + # rejects for C++. Strip it after PGXS is included (no-op if absent). + printf '\noverride CXXFLAGS := $(filter-out -fexcess-precision=standard,$(CXXFLAGS))\n' >> Makefile.global + ) + + echo "Moving Debian packaging into source directory..." + cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" + cd "$SRC_DIR" + cp debian/control.in debian/control + sed -i "s|PG_MAJOR_VERSION|${PG_MAJOR_VERSION}|g" debian/control + mv debian/pgedge-postgresql-pg-duckdb.install \ + debian/pgedge-postgresql-${PG_MAJOR_VERSION}-pg-duckdb.install + sed -i "s|PG_MAJOR_VERSION|${PG_MAJOR_VERSION}|g" \ + debian/pgedge-postgresql-${PG_MAJOR_VERSION}-pg-duckdb.install + + echo "Installing build dependencies..." + sudo apt-get update + sudo apt-get build-dep -y . + + # --- bullseye only: build curl >= 7.77 from source for the build (NOT shipped). + # bullseye ships libcurl 7.74, too old to compile the DuckDB 1.5.x httpfs + # (CURLSSLOPT_AUTO_CLIENT_CERT, curl >= 7.77). Build it to a build-local prefix + # and expose it to find_package(CURL); we do NOT rpath it, so the produced .so + # resolves libcurl.so.4 from the system at runtime (the curl path is unused — + # ColdFront forces httplib). These exports persist into dpkg-buildpackage below. + DISTRO=$(lsb_release -cs) + if [ "$DISTRO" = "bullseye" ]; then + echo "bullseye: building curl ${CURL_VERSION} from source (build-time only)..." + local curl_prefix="${BUILD_DIR}/curl-install" + mkdir -p "$curl_prefix" + wget -q "https://curl.se/download/curl-${CURL_VERSION}.tar.gz" -O "${BUILD_DIR}/curl.tar.gz" + tar xf "${BUILD_DIR}/curl.tar.gz" -C "$BUILD_DIR" + ( + cd "${BUILD_DIR}/curl-${CURL_VERSION}" + ./configure --with-openssl --prefix="$curl_prefix" --disable-static \ + --without-libpsl --without-libssh2 --without-nghttp2 --without-brotli --without-zstd + make -j"$(nproc)" + make install + ) + export CMAKE_PREFIX_PATH="${curl_prefix}${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" + export PKG_CONFIG_PATH="${curl_prefix}/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" + export LIBRARY_PATH="${curl_prefix}/lib${LIBRARY_PATH:+:$LIBRARY_PATH}" + export CPATH="${curl_prefix}/include${CPATH:+:$CPATH}" + fi +} + +build() { + + cd "$SRC_DIR" + echo "Building Debian package..." + DISTRO=$(lsb_release -cs) + rm -rf debian/changelog + echo "pgedge-pg-duckdb (${PG_DUCKDB_VERSION}-${PG_DUCKDB_BUILDNUM}.${DISTRO}) unstable; urgency=low" >> debian/changelog + echo " * Initial Release." >> debian/changelog + echo " -- pgEdge Build Team $(date -R)" >> debian/changelog + dch -D "$DISTRO" --force-distribution -v "${PG_DUCKDB_VERSION}-${PG_DUCKDB_BUILDNUM}.${DISTRO}" "pgEdge pg_duckdb ${PG_DUCKDB_VERSION} for $DISTRO" + + PATH=/usr/lib/postgresql/${PG_MAJOR_VERSION}/bin:$PATH dpkg-buildpackage -us -uc -b +} + +post_build() { + echo "Copying .deb packages to output..." + sudo mkdir -p "/output" + # Rename .ddeb files to .deb files + rename_ddeb_packages "$BUILD_DIR" + sudo cp "$BUILD_DIR"/*.deb "/output" || echo "No .deb packages found." +} diff --git a/packaging/pg_duckdb/build-rpm.sh b/packaging/pg_duckdb/build-rpm.sh new file mode 100644 index 0000000..0c5952a --- /dev/null +++ b/packaging/pg_duckdb/build-rpm.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +RHEL="$(rpm --eval %rhel)" + +prepare() { + setup_dnf_build_env + + echo "Copying packaging files..." + cp "${COMPONENT_NAME}/rpm/pg_duckdb.spec" ~/rpmbuild/SPECS/ + + echo "Vendoring pg_duckdb @ ${PG_DUCKDB_COMMIT} with bundled DuckDB submodule (v1.5.4)..." + rm -rf "/tmp/pg_duckdb-${PG_DUCKDB_VERSION}" + git init -q "/tmp/pg_duckdb-${PG_DUCKDB_VERSION}" + ( + cd "/tmp/pg_duckdb-${PG_DUCKDB_VERSION}" + git remote add origin "${PG_DUCKDB_REPO}" + git fetch -q --depth 1 origin "${PG_DUCKDB_COMMIT}" + git checkout -q FETCH_HEAD + git submodule update --init --recursive --depth 1 + # GitHub auto-tarballs omit submodules, and pg_duckdb's Makefile gates the + # DuckDB build on a `.git/modules/third_party/duckdb/HEAD` marker. Drop the + # heavy .git history but leave that marker so `make` treats the submodule as + # already checked out and never tries to fetch it (offline-safe build). + rm -rf .git + mkdir -p .git/modules/third_party/duckdb + touch .git/modules/third_party/duckdb/HEAD + ) + tar czf ~/rpmbuild/SOURCES/pg_duckdb-${PG_DUCKDB_VERSION}.tar.gz \ + -C /tmp "pg_duckdb-${PG_DUCKDB_VERSION}" + + # curl source for the EL9 build-time-only libcurl (Source1). Always staged so + # rpmbuild finds Source1; only consumed by %build on EL9 (el10 uses system curl). + echo "Staging curl ${CURL_VERSION} source (build-time, EL9 only)..." + wget -q "https://curl.se/download/curl-${CURL_VERSION}.tar.gz" \ + -O ~/rpmbuild/SOURCES/curl-${CURL_VERSION}.tar.gz + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "🔧 Installing RPM build dependencies..." + dnf builddep -y \ + --define "pgmajorversion ${PG_MAJOR_VERSION}" \ + --define "pginstdir /usr/pgsql-${PG_MAJOR_VERSION}" \ + --define "pg_duckdb_version ${PG_DUCKDB_VERSION}" \ + --define "pg_duckdb_buildnum ${PG_DUCKDB_BUILDNUM}" \ + --define "curl_version ${CURL_VERSION}" \ + ~/rpmbuild/SPECS/pg_duckdb.spec +} + +build() { + echo "Building RPM and SRPM..." + QA_RPATHS=$(( 0xffff )) rpmbuild -ba ~/rpmbuild/SPECS/pg_duckdb.spec \ + --define "pgmajorversion ${PG_MAJOR_VERSION}" \ + --define "pginstdir /usr/pgsql-${PG_MAJOR_VERSION}" \ + --define "pg_duckdb_version ${PG_DUCKDB_VERSION}" \ + --define "pg_duckdb_buildnum ${PG_DUCKDB_BUILDNUM}" \ + --define "curl_version ${CURL_VERSION}" +} + +post_build() { + echo "📤 Copying built RPMs to /output..." + mkdir -p /output + cp -v ~/rpmbuild/RPMS/*/*.rpm /output/ || echo "No binary RPMs found" + cp -v ~/rpmbuild/SRPMS/*.src.rpm /output/ || echo "No SRPM found" + + sign_rpms /output/*.rpm + validate_signatures /output/*.rpm +} diff --git a/packaging/pg_duckdb/common.sh b/packaging/pg_duckdb/common.sh new file mode 100644 index 0000000..505eaf0 --- /dev/null +++ b/packaging/pg_duckdb/common.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# common.sh - Common environment variables for the ColdFront pg_duckdb package. + +# Build once per PostgreSQL major version (fan-out across the matrix). +PER_PG_VERSION=true + +# Default PostgreSQL version and derived values +export PG_VERSION="${PG_VERSION:-17.7}" +export PG_MAJOR_VERSION="$(echo "$PG_VERSION" | cut -d. -f1)" + +# pg_duckdb pinned to the merged PR #1025 commit (DuckDB 1.5.4). No released +# pg_duckdb tag carries 1.5.x; this commit pins its duckdb submodule to v1.5.4. +# COMPONENT_BRANCH/COMPONENT_VERSION may override for a rebuild. +export PG_DUCKDB_REPO="https://github.com/duckdb/pg_duckdb" +export PG_DUCKDB_COMMIT="${COMPONENT_BRANCH:-c04e6a2dcf4e999abb921da1ba2f8335dad644e0}" +export PG_DUCKDB_VERSION="${COMPONENT_VERSION:-1.5.4}" +export PG_DUCKDB_BUILDNUM="${COMPONENT_BUILDNUM:-1}" + +# libcurl built from source and BUNDLED in the package. DuckDB 1.5.4 httpfs links +# CURLSSLOPT_AUTO_CLIENT_CERT (curl >= 7.77); el9 ships 7.76 and debian bullseye +# 7.74, so the system curl can't resolve pg_duckdb.so's symbols at load. 8.12.0 +# also fixes CVE-2025-0665. Installed into the PG libdir with a $ORIGIN RUNPATH +# on pg_duckdb.so so it finds the co-located copy without touching the OS libcurl. +export CURL_VERSION="${CURL_VERSION:-8.12.0}" + +# DEB only: move a pre-release pretag (e.g. BUILDNUM='rc1_1') into the upstream +# VERSION with a leading '~' (1.5.4~rc1, BUILDNUM=1) so '~' sorts pre-releases +# BELOW stable in dpkg/reprepro. Gated on apt-get so RPM keeps the pretag in +# Release (rpmvercmp already sorts rc1_1 below 1). The source pin is the commit, +# not the version, so this never affects what is built. +if command -v apt-get &>/dev/null; then + if [[ "$PG_DUCKDB_BUILDNUM" == *_* ]]; then + PG_DUCKDB_PRETAG="${PG_DUCKDB_BUILDNUM%%_*}" + export PG_DUCKDB_VERSION="${PG_DUCKDB_VERSION}~${PG_DUCKDB_PRETAG}" + PG_DUCKDB_BUILDNUM="${PG_DUCKDB_BUILDNUM#*_}" + fi +fi + +export REPO_TYPE="${REPO_TYPE:-daily}" diff --git a/packaging/pg_duckdb/deb/debian/control.in b/packaging/pg_duckdb/deb/debian/control.in new file mode 100644 index 0000000..ce6d2cd --- /dev/null +++ b/packaging/pg_duckdb/deb/debian/control.in @@ -0,0 +1,45 @@ +Source: pgedge-pg-duckdb +Section: database +Priority: optional +Maintainer: pgEdge Build Team +Build-Depends: + debhelper-compat (= 13), + pgedge-postgresql-all , + pgedge-postgresql-server-dev-PG_MAJOR_VERSION, + pgedge-postgresql-server-dev-all, + cmake, + ninja-build, + g++, + pkg-config, + libssl-dev, + liblz4-dev, + zlib1g-dev, + libicu-dev, + libcurl4-openssl-dev, + git, + wget, + ca-certificates, +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Homepage: https://github.com/duckdb/pg_duckdb + +Package: pgedge-postgresql-PG_MAJOR_VERSION-pg-duckdb +Architecture: any +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + pgedge-postgresql-PG_MAJOR_VERSION, + libcurl4, +Conflicts: + postgresql-PG_MAJOR_VERSION-pg-duckdb, +Provides: + postgresql-PG_MAJOR_VERSION-pg-duckdb, +Replaces: + postgresql-PG_MAJOR_VERSION-pg-duckdb, +Breaks: + postgresql-PG_MAJOR_VERSION-pg-duckdb, +Description: DuckDB analytics engine embedded in PostgreSQL + pg_duckdb embeds DuckDB's columnar-vectorized analytics engine into + PostgreSQL, enabling fast analytical queries and direct access to data + lakes and external file formats (Parquet, CSV, Iceberg, object storage) + from within Postgres. diff --git a/packaging/pg_duckdb/deb/debian/pgedge-postgresql-pg-duckdb.install b/packaging/pg_duckdb/deb/debian/pgedge-postgresql-pg-duckdb.install new file mode 100644 index 0000000..19f2a62 --- /dev/null +++ b/packaging/pg_duckdb/deb/debian/pgedge-postgresql-pg-duckdb.install @@ -0,0 +1 @@ +debian/tmp/sbom/* usr/lib/postgresql/PG_MAJOR_VERSION/sbom/ diff --git a/packaging/pg_duckdb/deb/debian/pgversions b/packaging/pg_duckdb/deb/debian/pgversions new file mode 100644 index 0000000..21244ca --- /dev/null +++ b/packaging/pg_duckdb/deb/debian/pgversions @@ -0,0 +1 @@ +15+ diff --git a/packaging/pg_duckdb/deb/debian/rules b/packaging/pg_duckdb/deb/debian/rules new file mode 100644 index 0000000..4376178 --- /dev/null +++ b/packaging/pg_duckdb/deb/debian/rules @@ -0,0 +1,40 @@ +#!/usr/bin/make -f + +# DuckDB is a large, memory-heavy C++ build driven by CMake+Ninja. +export DUCKDB_GEN = ninja +export CMAKE_BUILD_PARALLEL_LEVEL = 4 + +%: + dh $@ + +override_dh_auto_build: + # Cap parallelism — DuckDB can OOM at high -j on CI runners. with_llvm=no + # matches the proven ColdFront recipe (no JIT bitcode shipped). On bullseye + # the curl >= 7.77 found via CMAKE_PREFIX_PATH (build-deb.sh) is used here. + USE_PGXS=1 dh_auto_build -- -j4 with_llvm=no + +override_dh_auto_install: + USE_PGXS=1 dh_auto_install -- with_llvm=no + +override_dh_auto_test: + # pg_duckdb's `make check` drives a full regression harness that needs a + # running server and is not packaging-relevant; skip it during the build. + +override_dh_dwz: + # libduckdb.so is enormous; dwz bails with "Too many DIEs" and errors out. + # Skip dwz debug-info optimization (not packaging-relevant). + +override_dh_builddeb: + dh_builddeb -- -Zgzip + +execute_before_dh_install: + # --- SBOM generation and signing (per-format DEB key) --- + mkdir -p debian/tmp/sbom + syft dir:$(CURDIR) -o cyclonedx-json > debian/tmp/sbom/pg_duckdb-sbom.json || exit 1 + KEY_ID=$$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $$5}' | head -n 1); \ + gpg --armor --detach-sign --local-user "$$KEY_ID" \ + --output debian/tmp/sbom/pg_duckdb-sbom.json.asc \ + debian/tmp/sbom/pg_duckdb-sbom.json || exit 1 + +override_dh_installdocs: + dh_installdocs --all README.* diff --git a/packaging/pg_duckdb/deb/debian/source/format b/packaging/pg_duckdb/deb/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/packaging/pg_duckdb/deb/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/pg_duckdb/rpm/pg_duckdb.spec b/packaging/pg_duckdb/rpm/pg_duckdb.spec new file mode 100644 index 0000000..2ca0f89 --- /dev/null +++ b/packaging/pg_duckdb/rpm/pg_duckdb.spec @@ -0,0 +1,112 @@ +%global sname pg_duckdb +# Package name uses a hyphen (pg-duckdb) to match the DEB naming +# (pgedge-postgresql-N-pg-duckdb); sname keeps the upstream underscore for the +# installed artifacts (pg_duckdb.so, pg_duckdb.control, tarball dir). +%global pkgname pg-duckdb + +Name: pgedge-%{pkgname}_%{pgmajorversion} +Version: %{pg_duckdb_version} +Release: %{pg_duckdb_buildnum}%{?dist} +Summary: DuckDB analytics engine embedded in PostgreSQL +License: MIT +URL: https://github.com/duckdb/pg_duckdb +# Vendored tarball produced by build-rpm.sh that bundles the third_party/duckdb +# submodule (pinned by the pg_duckdb commit; GitHub auto-tarballs omit submodules). +Source0: %{sname}-%{version}.tar.gz +# curl source, built and used at BUILD TIME ONLY on el9 (system libcurl 7.76 is +# too old to compile the DuckDB 1.5.x httpfs, which needs the curl >= 7.77 macro +# CURLSSLOPT_AUTO_CLIENT_CERT). Not shipped: the built .so references only the +# libcurl.so.4 soname + standard symbols that el9's 7.76 already provides, so at +# runtime the native system libcurl resolves it. el10 builds against system curl. +Source1: curl-%{curl_version}.tar.gz + +BuildRequires: pgedge-postgresql%{pgmajorversion}-devel +BuildRequires: cmake +BuildRequires: ninja-build +BuildRequires: gcc-c++ +BuildRequires: make +BuildRequires: redhat-rpm-config +BuildRequires: openssl-devel +BuildRequires: lz4-devel +BuildRequires: zlib-devel +BuildRequires: libicu-devel +BuildRequires: pkgconf-pkg-config +BuildRequires: python3 +BuildRequires: git +# EL10+ compiles against the system libcurl (>= 7.77). On EL9 we build curl from +# Source1 instead, so libcurl-devel is intentionally NOT pulled there. +%if 0%{?rhel} >= 10 +BuildRequires: libcurl-devel +%endif +Requires: pgedge-postgresql%{pgmajorversion}-server +# Native system libcurl at runtime on every EL (the curl >= 7.77 need is +# compile-time only; the curl HTTP path is unused — ColdFront forces httplib). +Requires: libcurl +Requires: lz4-libs +Provides: %{pkgname}_%{pgmajorversion} + +%description +pg_duckdb embeds DuckDB's columnar-vectorized analytics engine into PostgreSQL, +enabling high-performance analytical queries and direct access to data lakes and +external file formats (Parquet, CSV, Iceberg, object storage) from within +PostgreSQL. + +%prep +%setup -q -n %{sname}-%{version} + +%build +# pgEdge propagates -fexcess-precision=standard into CXXFLAGS, which gcc rejects +# for C++. Strip it after PGXS is included (no-op if absent). +printf '\noverride CXXFLAGS := $(filter-out -fexcess-precision=standard,$(CXXFLAGS))\n' >> Makefile.global + +%if 0%{?rhel} && 0%{?rhel} < 10 +# --- EL9 only: build curl >= 7.77 from source for the build (NOT shipped) ------ +# Installed to a build-local prefix and exposed to find_package(CURL); we do NOT +# add it to any RUNPATH, so the produced .so resolves libcurl.so.4 from the +# system at runtime. +%global curl_prefix %{_builddir}/curl-install +mkdir -p %{curl_prefix} +tar xf %{SOURCE1} -C %{_builddir} +( cd %{_builddir}/curl-%{curl_version} \ + && ./configure --with-openssl --prefix=%{curl_prefix} --disable-static \ + --without-libpsl --without-libssh2 --without-nghttp2 --without-brotli --without-zstd \ + && %{__make} -j"$(nproc)" && %{__make} install ) +export CMAKE_PREFIX_PATH=%{curl_prefix}${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH} +export PKG_CONFIG_PATH=%{curl_prefix}/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} +export LIBRARY_PATH=%{curl_prefix}/lib${LIBRARY_PATH:+:$LIBRARY_PATH} +export CPATH=%{curl_prefix}/include${CPATH:+:$CPATH} +%endif + +# DuckDB is a large C++17 build driven by CMake+Ninja under the PGXS Makefile. +# with_llvm=no matches the proven ColdFront recipe (no JIT bitcode shipped). +# Parallelism is capped to limit peak memory (DuckDB can OOM at high -j). +export DUCKDB_GEN=ninja +export CMAKE_BUILD_PARALLEL_LEVEL=4 +USE_PGXS=1 PATH=%{pginstdir}/bin:$PATH %{__make} -j4 with_llvm=no + +syft dir:%{_builddir}/%{sname}-%{version} -o cyclonedx-json > %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1); export KEY_ID +gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +%install +%{__rm} -rf %{buildroot} +USE_PGXS=1 PATH=%{pginstdir}/bin:$PATH %{__make} install DESTDIR=%{buildroot} with_llvm=no +mkdir -p %{buildroot}/%{pginstdir}/sbom +install -p -m 0644 %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json %{buildroot}/%{pginstdir}/sbom/%{sname}-sbom.json +install -p -m 0644 %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{buildroot}/%{pginstdir}/sbom/%{sname}-sbom.json.asc + +%files +%doc README.md +%license LICENSE +# Dynamic build ships the extension and the bundled DuckDB shared library. +%{pginstdir}/lib/%{sname}.so +%{pginstdir}/lib/libduckdb.so +%{pginstdir}/share/extension/%{sname}.control +%{pginstdir}/share/extension/%{sname}*.sql +%{pginstdir}/sbom/%{sname}-sbom.json +%{pginstdir}/sbom/%{sname}-sbom.json.asc + +%changelog +* Mon Jun 30 2026 Muhammad Aqeel - 1.5.4-1 +- ColdFront build of pgEdge pg_duckdb (DuckDB 1.5.4, pg_duckdb PR #1025) From 192af5179b812e4a57eac3af7e3965479af4a5bd Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 13:38:00 +0500 Subject: [PATCH 02/12] coldfront: build-once duckdb-extensions (manylinux build and repackage per distro) and ship coldfront-bin config --- .github/workflows/release.yml | 303 ++++++++++++++++-- .../deb/debian/pgedge-coldfront.conffiles | 1 + packaging/coldfront-bin/deb/debian/rules | 7 + packaging/coldfront-bin/rpm/coldfront.spec | 12 +- .../coldfront-duckdb-extensions/build-deb.sh | 65 ++-- .../build-extensions.sh | 85 +++++ .../coldfront-duckdb-extensions/build-rpm.sh | 91 ++---- 7 files changed, 434 insertions(+), 130 deletions(-) create mode 100644 packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles create mode 100644 packaging/coldfront-duckdb-extensions/build-extensions.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62207bc..a4bbc9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,15 +5,21 @@ # Tag-driven. A pushed v* tag builds and publishes, in one GitHub Actions run: # 1. GoReleaser: the Go binaries (archiver, partitioner, compactor) + the # GitHub release, and uploads the linux tarballs as a workflow artifact. -# 2. Native RPM/DEB packages for three components, across PG 16/17/18 and +# 2. Native RPM/DEB packages for four components, across PG 16/17/18 and # el9/el10 + jammy/noble/bullseye/bookworm/trixie, amd64 + arm64: # * packaging/coldfront-bin — the Go tools (PG-independent, per-arch) # * packaging/coldfront — the coldfront PG extension (per PG major) # * packaging/pg_duckdb — pg_duckdb 1.5.4 (per PG major) # * packaging/coldfront-duckdb-extensions — patched iceberg/avro/azure/ -# postgres_scanner .duckdb_extension (PG-independent, per distro/arch) +# postgres_scanner .duckdb_extension. # pg_duckdb + duckdb-extensions keep their own version (1.5.4); coldfront-bin # + the extension track the release tag. +# +# The duckdb-extensions are BUILD-ONCE per arch: build-duckdb-ext compiles +# them a single time on manylinux_2_28 (glibc 2.28 baseline → portable to +# every supported distro), verify-duckdb-ext then LOADs all four in the real +# DuckDB engine on every distro as a gate, and the per-distro package-ext-* +# cells only repackage those verified binaries (no recompile). # 3. Push RPMs to dnf.pgedge.com and DEBs to apt.pgedge.com, back up to S3, # and publish a promotion manifest + Slack notification. # @@ -160,10 +166,19 @@ jobs: runs-on: ubuntu-latest needs: [determine-repo-type] outputs: + # Source-built components (pg_duckdb, coldfront, coldfront-bin). rpm_matrix: ${{ steps.merge.outputs.rpm_matrix }} deb_matrix: ${{ steps.merge.outputs.deb_matrix }} has_rpm: ${{ steps.merge.outputs.has_rpm }} has_deb: ${{ steps.merge.outputs.has_deb }} + # duckdb-extensions only (build-once → repackage per distro). + ext_rpm_matrix: ${{ steps.merge.outputs.ext_rpm_matrix }} + ext_deb_matrix: ${{ steps.merge.outputs.ext_deb_matrix }} + has_ext_rpm: ${{ steps.merge.outputs.has_ext_rpm }} + has_ext_deb: ${{ steps.merge.outputs.has_ext_deb }} + # All components merged — used by push + manifest to enumerate cells. + all_rpm_matrix: ${{ steps.merge.outputs.all_rpm_matrix }} + all_deb_matrix: ${{ steps.merge.outputs.all_deb_matrix }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -232,22 +247,140 @@ jobs: jq -c --arg c "$2" --arg v "$3" \ '[.include[] | . + {component_name:$c, component_version:$v}]' <<<"$1" } + emit() { # + echo "$1={\"include\":$2}" >> "$GITHUB_OUTPUT" + } + # Source-built components (their cells compile in-container). rpm_inc="$(jq -c -n \ --argjson a "$(tag_cells "$RPM_PGD" packaging/pg_duckdb '')" \ --argjson b "$(tag_cells "$RPM_CF" packaging/coldfront "$TAG_VERSION")" \ --argjson d "$(tag_cells "$RPM_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ - --argjson e "$(tag_cells "$RPM_DUX" packaging/coldfront-duckdb-extensions '')" \ - '$a + $b + $d + $e')" + '$a + $b + $d')" deb_inc="$(jq -c -n \ --argjson a "$(tag_cells "$DEB_PGD" packaging/pg_duckdb '')" \ --argjson b "$(tag_cells "$DEB_CF" packaging/coldfront "$TAG_VERSION")" \ --argjson d "$(tag_cells "$DEB_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ - --argjson e "$(tag_cells "$DEB_DUX" packaging/coldfront-duckdb-extensions '')" \ - '$a + $b + $d + $e')" - echo "rpm_matrix={\"include\":$rpm_inc}" >> "$GITHUB_OUTPUT" - echo "deb_matrix={\"include\":$deb_inc}" >> "$GITHUB_OUTPUT" - echo "has_rpm=$( [ "$(jq 'length' <<<"$rpm_inc")" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT" - echo "has_deb=$( [ "$(jq 'length' <<<"$deb_inc")" -gt 0 ] && echo true || echo false )" >> "$GITHUB_OUTPUT" + '$a + $b + $d')" + # duckdb-extensions: build-once, repackaged per distro/arch. + ext_rpm_inc="$(tag_cells "$RPM_DUX" packaging/coldfront-duckdb-extensions '')" + ext_deb_inc="$(tag_cells "$DEB_DUX" packaging/coldfront-duckdb-extensions '')" + emit rpm_matrix "$rpm_inc" + emit deb_matrix "$deb_inc" + emit ext_rpm_matrix "$ext_rpm_inc" + emit ext_deb_matrix "$ext_deb_inc" + emit all_rpm_matrix "$(jq -c -n --argjson x "$rpm_inc" --argjson y "$ext_rpm_inc" '$x + $y')" + emit all_deb_matrix "$(jq -c -n --argjson x "$deb_inc" --argjson y "$ext_deb_inc" '$x + $y')" + gt0() { [ "$(jq 'length' <<<"$1")" -gt 0 ] && echo true || echo false; } + { + echo "has_rpm=$(gt0 "$rpm_inc")" + echo "has_deb=$(gt0 "$deb_inc")" + echo "has_ext_rpm=$(gt0 "$ext_rpm_inc")" + echo "has_ext_deb=$(gt0 "$ext_deb_inc")" + } >> "$GITHUB_OUTPUT" + + # ========================================================================= + # BUILD-ONCE: compile the patched DuckDB extensions a single time per arch on + # manylinux_2_28 (glibc 2.28 baseline). The resulting .duckdb_extension + # binaries are portable to every supported distro; the per-distro package-ext-* + # cells repackage them without recompiling. + # ========================================================================= + build-duckdb-ext: + name: Build DuckDB extensions (${{ matrix.arch }}, manylinux) + needs: [determine-repo-type] + if: github.ref_type == 'tag' || github.event.inputs.simulate_tag != '' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Build extensions in manylinux_2_28 + env: + ML_ARCH: ${{ matrix.arch == 'arm64' && 'aarch64' || 'x86_64' }} + run: | + set -euo pipefail + mkdir -p ext-out + # Native-arch runner → native-arch manylinux container (no emulation). + # gcc-toolset-14 is pre-baked in manylinux_2_28; install the rest. + docker run --rm -v "$PWD":/src -w /src "quay.io/pypa/manylinux_2_28_${ML_ARCH}" bash -c ' + set -euo pipefail + dnf install -y --setopt=install_weak_deps=False \ + ninja-build perl-IPC-Cmd perl-Time-Piece ccache jq wget zip unzip tar \ + autoconf libtool kernel-headers cmake git make flex bison \ + gcc-toolset-14-libasan-devel gcc-toolset-14-libubsan-devel + export COMPONENT_NAME=packaging/coldfront-duckdb-extensions + # shellcheck disable=SC1091 + source packaging/coldfront-duckdb-extensions/common.sh + bash packaging/coldfront-duckdb-extensions/build-extensions.sh /src/ext-out + ' + ls -la ext-out/ + - name: Upload built extensions + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: duckdb-ext-${{ matrix.arch }} + path: ext-out/*.duckdb_extension + retention-days: 7 + + # ========================================================================= + # GATE: load all four extensions in the real DuckDB engine on EVERY supported + # distro (per arch). A clean LOAD proves every dependency — system libs and the + # DuckDB ABI — resolves. Packaging waits on this; if any distro fails to load, + # the extensions are not published. + # ========================================================================= + verify-duckdb-ext: + name: Verify extensions load (${{ matrix.arch }}) + needs: [build-duckdb-ext] + if: needs.build-duckdb-ext.result == 'success' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Resolve DUCKDB_VERSION + id: duckdb + run: | + set -euo pipefail + # shellcheck disable=SC1091 + source packaging/coldfront-duckdb-extensions/common.sh + echo "version=${DUCKDB_VERSION}" >> "$GITHUB_OUTPUT" + - name: Download built extensions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: duckdb-ext-${{ matrix.arch }} + path: ext + - name: Load-test on every distro + env: + DUCKDB_VERSION: ${{ steps.duckdb.outputs.version }} + CLI_ARCH: ${{ matrix.arch == 'arm64' && 'arm64' || 'amd64' }} + run: | + set -uo pipefail + CLI_URL="https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-${CLI_ARCH}.zip" + echo "Engine CLI: $CLI_URL" + pass=0; fail=0 + for img in almalinux:9 almalinux:10 ubuntu:jammy ubuntu:noble debian:bullseye debian:bookworm debian:trixie; do + echo "###### $img ######" + cid=$(docker run -d -v "$PWD/ext:/ext:ro" "$img" sleep 600) || { echo " run failed"; fail=$((fail+1)); continue; } + docker exec "$cid" bash -c ' + if command -v dnf >/dev/null 2>&1; then dnf install -y --setopt=install_weak_deps=False libstdc++ unzip tar >/dev/null 2>&1; + else export DEBIAN_FRONTEND=noninteractive; apt-get update >/dev/null 2>&1; apt-get install -y libstdc++6 unzip ca-certificates curl >/dev/null 2>&1; fi' + docker exec -e CLI_URL="$CLI_URL" "$cid" bash -c ' + cd /root + curl -sSL -o cli.zip "$CLI_URL" && unzip -o -q cli.zip && chmod +x ./duckdb || { echo " CLI fetch failed"; exit 3; } + # allow_unsigned_extensions is a STARTUP setting -> -unsigned flag. + # Load in dependency order (iceberg needs avro registered first). + out=$(./duckdb -unsigned -c "LOAD '"'"'/ext/avro.duckdb_extension'"'"'; LOAD '"'"'/ext/iceberg.duckdb_extension'"'"'; LOAD '"'"'/ext/azure.duckdb_extension'"'"'; LOAD '"'"'/ext/postgres_scanner.duckdb_extension'"'"'; SELECT '"'"'ALL_LOADED'"'"';" 2>&1) + if echo "$out" | grep -q ALL_LOADED; then echo " ==> all 4 loaded"; exit 0 + else echo " ==> LOAD FAILED"; echo "$out" | sed "s/^/ /" | head -10; exit 1; fi + ' && { echo " PASS: $img"; pass=$((pass+1)); } || { echo " FAIL: $img"; fail=$((fail+1)); } + docker rm -f "$cid" >/dev/null 2>&1 + done + echo "=== load-test (${CLI_ARCH}): $pass passed, $fail failed ===" + [ "$fail" -eq 0 ] # ========================================================================= # RPM build matrix — one cell per (component, PG major, EL major, arch). @@ -403,16 +536,145 @@ jobs: path: output/${{ steps.cell.outputs.distro }}-${{ matrix.arch }}/ retention-days: 7 + # ========================================================================= + # Repackage the build-once extensions into RPMs per (EL major, arch). Each + # cell just stages the verified prebuilt binaries and runs the packaging — + # no recompile. Gated on verify-duckdb-ext. + # ========================================================================= + package-ext-rpm: + name: RPM ext ${{ matrix.image }} ${{ matrix.arch }} + needs: [determine-repo-type, detect-matrix, verify-duckdb-ext] + if: | + always() && + needs.determine-repo-type.result == 'success' && + needs.detect-matrix.result == 'success' && + needs.verify-duckdb-ext.result == 'success' && + needs.detect-matrix.outputs.has_ext_rpm == 'true' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-matrix.outputs.ext_rpm_matrix) }} + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_BUILDNUM: ${{ needs.determine-repo-type.outputs.component-buildnum }} + COMPONENT_BRANCH: ${{ needs.determine-repo-type.outputs.effective-tag }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Derive os_version + id: cell + env: + IMAGE: ${{ matrix.image }} + run: echo "os_version=${IMAGE##*:}" >> "$GITHUB_OUTPUT" + - name: Stage prebuilt extensions (${{ matrix.arch }}) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: duckdb-ext-${{ matrix.arch }} + path: release-artifacts/duckdb-extensions + - name: Checkout pgedge-builder-action + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-builder-action.git" \ + ".github/actions/pgedge-builder-action" + - name: Build RPM + uses: ./.github/actions/pgedge-builder-action + with: + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + component_name: ${{ matrix.component_name }} + pg_version: ${{ matrix.pg_version }} + component_branch: ${{ env.COMPONENT_BRANCH }} + component_version: ${{ matrix.component_version }} + component_buildnum: ${{ env.COMPONENT_BUILDNUM }} + repo_type: ${{ env.REPO_TYPE }} + gpg_private_key: ${{ secrets.GPG_FIPS_RPM_PRIVATE_KEY }} + gpg_public_key: ${{ secrets.GPG_FIPS_RPM_PUBLIC_KEY }} + pgedge_builder_token: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + - name: Upload RPM cell artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: rpm__coldfront-duckdb-extensions__el-${{ steps.cell.outputs.os_version }}__${{ matrix.arch }} + path: output/el${{ steps.cell.outputs.os_version }}-${{ matrix.arch }}/ + retention-days: 7 + + # ========================================================================= + # Repackage the build-once extensions into DEBs per (distro, arch). + # ========================================================================= + package-ext-deb: + name: DEB ext ${{ matrix.image }} ${{ matrix.arch }} + needs: [determine-repo-type, detect-matrix, verify-duckdb-ext] + if: | + always() && + needs.determine-repo-type.result == 'success' && + needs.detect-matrix.result == 'success' && + needs.verify-duckdb-ext.result == 'success' && + needs.detect-matrix.outputs.has_ext_deb == 'true' + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-matrix.outputs.ext_deb_matrix) }} + env: + REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} + COMPONENT_BUILDNUM: ${{ needs.determine-repo-type.outputs.component-buildnum }} + COMPONENT_BRANCH: ${{ needs.determine-repo-type.outputs.effective-tag }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Derive distro + id: cell + env: + IMAGE: ${{ matrix.image }} + run: echo "distro=${IMAGE##*:}" >> "$GITHUB_OUTPUT" + - name: Stage prebuilt extensions (${{ matrix.arch }}) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: duckdb-ext-${{ matrix.arch }} + path: release-artifacts/duckdb-extensions + - name: Checkout pgedge-builder-action + env: + TOKEN: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + run: | + set -euo pipefail + mkdir -p .github/actions + git clone --depth 1 \ + "https://x-access-token:${TOKEN}@github.com/pgEdge/pgedge-builder-action.git" \ + ".github/actions/pgedge-builder-action" + - name: Build DEB + uses: ./.github/actions/pgedge-builder-action + with: + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + component_name: ${{ matrix.component_name }} + pg_version: ${{ matrix.pg_version }} + component_branch: ${{ env.COMPONENT_BRANCH }} + component_version: ${{ matrix.component_version }} + component_buildnum: ${{ env.COMPONENT_BUILDNUM }} + repo_type: ${{ env.REPO_TYPE }} + gpg_private_key: ${{ secrets.GPG_FIPS_DEB_PRIVATE_KEY }} + gpg_public_key: ${{ secrets.GPG_FIPS_DEB_PUBLIC_KEY }} + pgedge_builder_token: ${{ secrets.PGEDGE_BUILDER_TOKEN }} + - name: Upload DEB cell artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: deb__coldfront-duckdb-extensions__${{ steps.cell.outputs.distro }}__${{ matrix.arch }} + path: output/${{ steps.cell.outputs.distro }}-${{ matrix.arch }}/ + retention-days: 7 + # ========================================================================= # Push RPMs → dnf.pgedge.com (all components, batched per EL major) + S3 backup. # ========================================================================= push-dnf: name: Push DNF (all components, batched) - needs: [determine-repo-type, detect-matrix, package-rpm] + needs: [determine-repo-type, detect-matrix, package-rpm, package-ext-rpm] if: | always() && github.ref_type == 'tag' && - (needs.package-rpm.result == 'success' || - (github.event.inputs.force_push == 'true' && needs.package-rpm.result != 'cancelled')) + ((needs.package-rpm.result == 'success' && needs.package-ext-rpm.result == 'success') || + (github.event.inputs.force_push == 'true' && + needs.package-rpm.result != 'cancelled' && needs.package-ext-rpm.result != 'cancelled')) runs-on: ubuntu-latest concurrency: group: dnf-push-${{ needs.determine-repo-type.outputs.repo-type }} @@ -422,7 +684,7 @@ jobs: env: REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} COMPONENT_VERSION: ${{ needs.determine-repo-type.outputs.component-version }} - RPM_MATRIX: ${{ needs.detect-matrix.outputs.rpm_matrix }} + RPM_MATRIX: ${{ needs.detect-matrix.outputs.all_rpm_matrix }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -488,11 +750,12 @@ jobs: # ========================================================================= push-apt: name: Push APT (all components, batched) - needs: [determine-repo-type, detect-matrix, package-deb] + needs: [determine-repo-type, detect-matrix, package-deb, package-ext-deb] if: | always() && github.ref_type == 'tag' && - (needs.package-deb.result == 'success' || - (github.event.inputs.force_push == 'true' && needs.package-deb.result != 'cancelled')) + ((needs.package-deb.result == 'success' && needs.package-ext-deb.result == 'success') || + (github.event.inputs.force_push == 'true' && + needs.package-deb.result != 'cancelled' && needs.package-ext-deb.result != 'cancelled')) runs-on: ubuntu-latest concurrency: group: apt-push-${{ needs.determine-repo-type.outputs.repo-type }} @@ -502,7 +765,7 @@ jobs: env: REPO_TYPE: ${{ needs.determine-repo-type.outputs.repo-type }} COMPONENT_VERSION: ${{ needs.determine-repo-type.outputs.component-version }} - DEB_MATRIX: ${{ needs.detect-matrix.outputs.deb_matrix }} + DEB_MATRIX: ${{ needs.detect-matrix.outputs.all_deb_matrix }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -591,8 +854,8 @@ jobs: PUSH_APT_RESULT: ${{ needs.push-apt.result }} DNF_TS: ${{ needs.push-dnf.outputs.timestamp }} APT_TS: ${{ needs.push-apt.outputs.timestamp }} - RPM_MATRIX: ${{ needs.detect-matrix.outputs.rpm_matrix }} - DEB_MATRIX: ${{ needs.detect-matrix.outputs.deb_matrix }} + RPM_MATRIX: ${{ needs.detect-matrix.outputs.all_rpm_matrix }} + DEB_MATRIX: ${{ needs.detect-matrix.outputs.all_deb_matrix }} run: | set -euo pipefail mkdir -p aggregated diff --git a/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles new file mode 100644 index 0000000..79ff6ba --- /dev/null +++ b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles @@ -0,0 +1 @@ +/etc/coldfront/config.yaml diff --git a/packaging/coldfront-bin/deb/debian/rules b/packaging/coldfront-bin/deb/debian/rules index b22ee34..64bdffc 100644 --- a/packaging/coldfront-bin/deb/debian/rules +++ b/packaging/coldfront-bin/deb/debian/rules @@ -22,3 +22,10 @@ execute_before_dh_install: override_dh_installdocs: dh_installdocs --all README.* + +execute_after_dh_install: + # Ship the example deployment config as the default config file, renamed to + # config.yaml. The tools take `-config `; point them at this. It lands + # under /etc and debian/pgedge-coldfront.conffiles marks it a conffile, so an + # admin's edited copy survives upgrades. + install -D -m 0644 config.example.yaml debian/pgedge-coldfront/etc/coldfront/config.yaml diff --git a/packaging/coldfront-bin/rpm/coldfront.spec b/packaging/coldfront-bin/rpm/coldfront.spec index 12ad681..767a1b2 100644 --- a/packaging/coldfront-bin/rpm/coldfront.spec +++ b/packaging/coldfront-bin/rpm/coldfront.spec @@ -1,5 +1,9 @@ %global sname coldfront +# Prebuilt, fully-stripped static Go binaries — there is no debug info or source +# to extract, so disable the (empty) debuginfo/debugsource subpackages. +%global debug_package %{nil} + Name: pgedge-%{sname} Version: %{coldfront_version} Release: %{coldfront_buildnum}%{?dist} @@ -36,6 +40,10 @@ gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}- install -D -m 0755 archiver %{buildroot}%{_bindir}/archiver install -D -m 0755 partitioner %{buildroot}%{_bindir}/partitioner install -D -m 0755 compactor %{buildroot}%{_bindir}/compactor +# Ship the example deployment config as the default config file. The tools take +# `-config `; point them at this. Installed %config(noreplace) so an admin's +# edited copy survives upgrades (a new default lands as config.yaml.rpmnew). +install -D -m 0644 config.example.yaml %{buildroot}%{_sysconfdir}/coldfront/config.yaml install -D -m 0644 %{sname}-sbom.json %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc @@ -45,9 +53,11 @@ install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname %{_bindir}/archiver %{_bindir}/partitioner %{_bindir}/compactor +%dir %{_sysconfdir}/coldfront +%config(noreplace) %{_sysconfdir}/coldfront/config.yaml %{_datadir}/pgedge-%{sname}/%{sname}-sbom.json %{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc %changelog -* Mon Jun 30 2026 Muhammad Aqeel - 1.0.0-1 +* Tue Jun 30 2026 Muhammad Aqeel - 1.0.0-1 - Initial build of the pgEdge ColdFront tools (archiver, partitioner, compactor) diff --git a/packaging/coldfront-duckdb-extensions/build-deb.sh b/packaging/coldfront-duckdb-extensions/build-deb.sh index fe702c6..51b4b61 100644 --- a/packaging/coldfront-duckdb-extensions/build-deb.sh +++ b/packaging/coldfront-duckdb-extensions/build-deb.sh @@ -5,14 +5,15 @@ CWD="$(pwd)" export DEBIAN_FRONTEND=noninteractive ARCH=$(dpkg --print-architecture) # amd64 | arm64 (== GoReleaser/DuckDB arch) case "$ARCH" in - amd64) DUCKDB_PLATFORM=linux_amd64; VCPKG_TRIPLET=x64-linux ;; - arm64) DUCKDB_PLATFORM=linux_arm64; VCPKG_TRIPLET=arm64-linux ;; + amd64) DUCKDB_PLATFORM=linux_amd64 ;; + arm64) DUCKDB_PLATFORM=linux_arm64 ;; *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; esac -BUILD_ROOT="/tmp/cf-duckdb-ext-build" -ICE="${BUILD_ROOT}/duckdb-iceberg" SRC_DIR="/tmp/pg_deb_build/coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}" +# CI stages the build-once artifacts here (per-arch, from build-duckdb-ext); when +# present we just repackage them — no compile in the per-distro cell. +PREBUILT="${PREBUILT:-${CWD}/release-artifacts/duckdb-extensions}" prepare() { @@ -21,50 +22,22 @@ prepare() { # This function is for debugging purpose if you have your own keys. GH workflow does not need it. #import_gpg_keys - echo "Installing duckdb-iceberg build toolchain..." - sudo apt-get install -y \ - build-essential cmake ninja-build git ccache jq wget zip unzip \ - tar autoconf libtool flex bison pkg-config perl - - echo "Cloning + patching duckdb-iceberg (against DuckDB v${DUCKDB_VERSION})..." - rm -rf "$BUILD_ROOT"; mkdir -p "$BUILD_ROOT" - git clone --filter=blob:none --no-checkout "${ICEBERG_REPO}" "$ICE" - git -C "$ICE" fetch --depth 80 origin "${ICEBERG_BRANCH}" - git -C "$ICE" checkout "${ICEBERG_REF}" - git -C "$ICE" submodule update --init --recursive --depth 1 --jobs 8 - rm -rf "$ICE/duckdb" - git clone --depth 1 --branch "v${DUCKDB_VERSION}" --recurse-submodules "${DUCKDB_REPO}" "$ICE/duckdb" - git clone "${VCPKG_REPO}" "${BUILD_ROOT}/vcpkg" - "${BUILD_ROOT}/vcpkg/bootstrap-vcpkg.sh" -disableMetrics - - cp "${CWD}/docker/iceberg-azure-extension-config-v15.cmake" "$ICE/extension_config.cmake" - for p in iceberg-bakery-aware-commit-refresh-v15 \ - iceberg-manifest-list-format-version-v15 \ - iceberg-manifest-content-v15 \ - iceberg-data-file-format-v15; do - git -C "$ICE" apply --check "${CWD}/docker/${p}.patch" - git -C "$ICE" apply "${CWD}/docker/${p}.patch" - done + echo "Staging extensions + docs + debian packaging..." + rm -rf "$SRC_DIR"; mkdir -p "$SRC_DIR" - echo "Building extensions (make release)..." - ( - cd "$ICE" - export VCPKG_TOOLCHAIN_PATH="${BUILD_ROOT}/vcpkg/scripts/buildsystems/vcpkg.cmake" - export VCPKG_ROOT="${BUILD_ROOT}/vcpkg" - export VCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" VCPKG_HOST_TRIPLET="$VCPKG_TRIPLET" - export USE_MERGED_VCPKG_MANIFEST=1 - export EXT_CONFIG="${ICE}/extension_config.cmake" - export OVERRIDE_GIT_DESCRIBE="v${DUCKDB_VERSION}" - # Cap parallelism to bound peak memory (DuckDB can OOM at high -j). - export CMAKE_BUILD_PARALLEL_LEVEL=4 - make -j4 release - ) + if compgen -G "${PREBUILT}/*.duckdb_extension" >/dev/null; then + # ---- build-once path: reuse the manylinux-built, load-tested binaries ---- + echo "Using prebuilt extensions from ${PREBUILT} (build-once)" + cp "${PREBUILT}"/*.duckdb_extension "$SRC_DIR/" + else + # ---- local / fallback path: build from source in this container ---- + echo "No prebuilt extensions staged — building from source in-cell." + sudo apt-get install -y \ + build-essential cmake ninja-build git ccache jq wget zip unzip \ + tar autoconf libtool flex bison pkg-config perl + bash "${CWD}/${COMPONENT_NAME}/build-extensions.sh" "$SRC_DIR" + fi - echo "Staging built extensions + docs + debian packaging..." - rm -rf "$SRC_DIR"; mkdir -p "$SRC_DIR" - for e in iceberg avro azure postgres_scanner; do - cp "${ICE}/build/release/extension/${e}/${e}.duckdb_extension" "$SRC_DIR/" - done cp "${COMPONENT_NAME}/config/coldfront-duckdb-extensions.conf.sample" "$SRC_DIR/" cp "${CWD}/LICENSE.md" "${CWD}/README.md" "$SRC_DIR/" cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" diff --git a/packaging/coldfront-duckdb-extensions/build-extensions.sh b/packaging/coldfront-duckdb-extensions/build-extensions.sh new file mode 100644 index 0000000..e6ae6d7 --- /dev/null +++ b/packaging/coldfront-duckdb-extensions/build-extensions.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# build-extensions.sh — build the 4 patched ColdFront DuckDB extensions ONCE and +# stage them into . Arch-aware (linux_amd64 / linux_arm64), toolchain- +# agnostic: the CALLER installs the compiler/build tools (a modern C++ toolchain — +# gcc >= 14 — plus cmake/ninja/perl/vcpkg prerequisites). It is invoked: +# * by release.yml's build-duckdb-ext job, inside a manylinux_2_28 container, so +# the single binary is portable across every supported glibc (build-once), and +# * by build-rpm.sh / build-deb.sh as a LOCAL fallback when no prebuilt artifact +# has been staged. +# +# Usage: build-extensions.sh +# Requires the component pins in the environment (ICEBERG_REPO/REF/BRANCH, +# DUCKDB_VERSION/REPO, VCPKG_REPO); sources common.sh if they are unset. +set -euo pipefail + +OUT_DIR="${1:?usage: build-extensions.sh }" +CWD="$(pwd)" # repo root (build.sh cwd) +: "${COMPONENT_NAME:=packaging/coldfront-duckdb-extensions}" + +# Pull in the build pins if we were invoked standalone (CI manylinux job). +if [ -z "${DUCKDB_VERSION:-}" ]; then + # shellcheck disable=SC1090 + source "${CWD}/${COMPONENT_NAME}/common.sh" +fi + +ARCH=$(uname -m) +case "$ARCH" in + x86_64) DUCKDB_PLATFORM=linux_amd64; VCPKG_TRIPLET=x64-linux ;; + aarch64) DUCKDB_PLATFORM=linux_arm64; VCPKG_TRIPLET=arm64-linux ;; + *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; +esac + +BUILD_ROOT="/tmp/cf-duckdb-ext-build" +ICE="${BUILD_ROOT}/duckdb-iceberg" + +echo "Cloning + patching duckdb-iceberg (DuckDB v${DUCKDB_VERSION}, ${DUCKDB_PLATFORM})..." +rm -rf "$BUILD_ROOT"; mkdir -p "$BUILD_ROOT" +git clone --filter=blob:none --no-checkout "${ICEBERG_REPO}" "$ICE" +git -C "$ICE" fetch --depth 80 origin "${ICEBERG_BRANCH}" +git -C "$ICE" checkout "${ICEBERG_REF}" +git -C "$ICE" submodule update --init --recursive --depth 1 --jobs 8 +# Pin the duckdb submodule to the engine version pg_duckdb links. +rm -rf "$ICE/duckdb" +git clone --depth 1 --branch "v${DUCKDB_VERSION}" --recurse-submodules "${DUCKDB_REPO}" "$ICE/duckdb" +git clone "${VCPKG_REPO}" "${BUILD_ROOT}/vcpkg" +"${BUILD_ROOT}/vcpkg/bootstrap-vcpkg.sh" -disableMetrics + +# extension set (avro/azure/postgres_scanner refs) + the 4 ColdFront patches live +# in the repo's docker/ dir; git apply --check fails loudly on patch rot. +cp "${CWD}/docker/iceberg-azure-extension-config-v15.cmake" "$ICE/extension_config.cmake" +for p in iceberg-bakery-aware-commit-refresh-v15 \ + iceberg-manifest-list-format-version-v15 \ + iceberg-manifest-content-v15 \ + iceberg-data-file-format-v15; do + git -C "$ICE" apply --check "${CWD}/docker/${p}.patch" + git -C "$ICE" apply "${CWD}/docker/${p}.patch" +done + +echo "Building extensions (make release)..." +( + cd "$ICE" + # Activate gcc-toolset-14 if present (manylinux_2_28 / EL9); no-op elsewhere. + # shellcheck disable=SC1090 + source /opt/rh/gcc-toolset-*/enable 2>/dev/null || true + echo "-- compiler: $(gcc --version | head -1) --" + export VCPKG_TOOLCHAIN_PATH="${BUILD_ROOT}/vcpkg/scripts/buildsystems/vcpkg.cmake" + export VCPKG_ROOT="${BUILD_ROOT}/vcpkg" + export VCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" VCPKG_HOST_TRIPLET="$VCPKG_TRIPLET" + export USE_MERGED_VCPKG_MANIFEST=1 + export EXT_CONFIG="${ICE}/extension_config.cmake" + # DuckDB stamps this as the engine version in the extension metadata footer; + # it MUST match the loading engine (pg_duckdb's bundled DuckDB) exactly. + export OVERRIDE_GIT_DESCRIBE="v${DUCKDB_VERSION}" + # Cap parallelism to bound peak memory (DuckDB can OOM at high -j). + export CMAKE_BUILD_PARALLEL_LEVEL=4 + make -j4 release +) + +echo "Staging built extensions → ${OUT_DIR}" +mkdir -p "$OUT_DIR" +for e in iceberg avro azure postgres_scanner; do + cp "${ICE}/build/release/extension/${e}/${e}.duckdb_extension" "$OUT_DIR/" +done +echo "Done. Built for ${DUCKDB_PLATFORM} (DuckDB v${DUCKDB_VERSION}):" +ls -la "$OUT_DIR"/*.duckdb_extension diff --git a/packaging/coldfront-duckdb-extensions/build-rpm.sh b/packaging/coldfront-duckdb-extensions/build-rpm.sh index 80bc0e7..2022afa 100644 --- a/packaging/coldfront-duckdb-extensions/build-rpm.sh +++ b/packaging/coldfront-duckdb-extensions/build-rpm.sh @@ -5,80 +5,45 @@ RHEL="$(rpm --eval %rhel)" CWD="$(pwd)" ARCH=$(uname -m) case "$ARCH" in - x86_64) DUCKDB_PLATFORM=linux_amd64; VCPKG_TRIPLET=x64-linux ;; - aarch64) DUCKDB_PLATFORM=linux_arm64; VCPKG_TRIPLET=arm64-linux ;; + x86_64) DUCKDB_PLATFORM=linux_amd64 ;; + aarch64) DUCKDB_PLATFORM=linux_arm64 ;; *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; esac -BUILD_ROOT="/tmp/cf-duckdb-ext-build" -ICE="${BUILD_ROOT}/duckdb-iceberg" STAGE="/tmp/coldfront-duckdb-extensions-${DUCKDB_EXT_VERSION}" +# CI stages the build-once artifacts here (per-arch, from build-duckdb-ext); when +# present we just repackage them — no compile in the per-distro cell. +PREBUILT="${PREBUILT:-${CWD}/release-artifacts/duckdb-extensions}" prepare() { setup_dnf_build_env - echo "Installing duckdb-iceberg build toolchain..." - # DuckDB 1.5.4's extensions (e.g. postgres_scanner) need a modern C++ compiler: - # EL9's native gcc 11 rejects them, so use gcc-toolset-14 (gcc 14, as the docker - # recipe does); EL10 ships gcc 14 natively. libasan/libubsan: the - # extension_configuration phase builds a Debug binary that links ASan. - if [ "${RHEL}" -lt 10 ]; then - dnf install -y gcc-toolset-14-gcc-c++ gcc-toolset-14-libasan-devel gcc-toolset-14-libubsan-devel + echo "Staging extensions + docs for packaging..." + rm -rf "$STAGE"; mkdir -p "$STAGE" + + if compgen -G "${PREBUILT}/*.duckdb_extension" >/dev/null; then + # ---- build-once path: reuse the manylinux-built, load-tested binaries ---- + echo "Using prebuilt extensions from ${PREBUILT} (build-once)" + cp "${PREBUILT}"/*.duckdb_extension "$STAGE/" else - dnf install -y gcc gcc-c++ libasan libubsan + # ---- local / fallback path: build from source in this container ---- + echo "No prebuilt extensions staged — building from source in-cell." + # DuckDB 1.5.4's extensions (e.g. postgres_scanner) need a modern C++ compiler: + # EL9's native gcc 11 rejects them, so use gcc-toolset-14 (gcc 14, as the docker + # recipe does); EL10 ships gcc 14 natively. libasan/libubsan: the + # extension_configuration phase builds a Debug binary that links ASan. + if [ "${RHEL}" -lt 10 ]; then + dnf install -y gcc-toolset-14-gcc-c++ gcc-toolset-14-libasan-devel gcc-toolset-14-libubsan-devel + else + dnf install -y gcc gcc-c++ libasan libubsan + fi + # perl-core: openssl-from-source (vcpkg) Configure needs FindBin + other core + # perl modules that EL's minimal perl splits out; perl-core pulls the full set. + dnf install -y make ninja-build perl-core perl-FindBin perl-IPC-Cmd perl-Time-Piece \ + ccache jq wget zip unzip tar autoconf libtool kernel-headers cmake git flex bison + bash "${CWD}/${COMPONENT_NAME}/build-extensions.sh" "$STAGE" fi - # perl-core: openssl-from-source (vcpkg) Configure needs FindBin + other core - # perl modules that EL's minimal perl splits out; perl-core pulls the full set. - dnf install -y make ninja-build perl-core perl-FindBin perl-IPC-Cmd perl-Time-Piece \ - ccache jq wget zip unzip tar autoconf libtool kernel-headers cmake git flex bison - - echo "Cloning + patching duckdb-iceberg (against DuckDB v${DUCKDB_VERSION})..." - rm -rf "$BUILD_ROOT"; mkdir -p "$BUILD_ROOT" - git clone --filter=blob:none --no-checkout "${ICEBERG_REPO}" "$ICE" - git -C "$ICE" fetch --depth 80 origin "${ICEBERG_BRANCH}" - git -C "$ICE" checkout "${ICEBERG_REF}" - git -C "$ICE" submodule update --init --recursive --depth 1 --jobs 8 - # Pin the duckdb submodule to the engine version pg_duckdb links. - rm -rf "$ICE/duckdb" - git clone --depth 1 --branch "v${DUCKDB_VERSION}" --recurse-submodules "${DUCKDB_REPO}" "$ICE/duckdb" - git clone "${VCPKG_REPO}" "${BUILD_ROOT}/vcpkg" - "${BUILD_ROOT}/vcpkg/bootstrap-vcpkg.sh" -disableMetrics - # extension set (avro/azure/postgres_scanner refs) + the 4 ColdFront patches - # live in the repo's docker/ dir; git apply --check fails loudly on patch rot. - cp "${CWD}/docker/iceberg-azure-extension-config-v15.cmake" "$ICE/extension_config.cmake" - for p in iceberg-bakery-aware-commit-refresh-v15 \ - iceberg-manifest-list-format-version-v15 \ - iceberg-manifest-content-v15 \ - iceberg-data-file-format-v15; do - git -C "$ICE" apply --check "${CWD}/docker/${p}.patch" - git -C "$ICE" apply "${CWD}/docker/${p}.patch" - done - - echo "Building extensions (make release)..." - ( - cd "$ICE" - # Activate gcc-toolset-14 on EL9 (gcc 14 for the C++ extensions); no-op on - # EL10 / where the toolset is absent (native gcc used). - # shellcheck disable=SC1090 - source /opt/rh/gcc-toolset-*/enable 2>/dev/null || true - export VCPKG_TOOLCHAIN_PATH="${BUILD_ROOT}/vcpkg/scripts/buildsystems/vcpkg.cmake" - export VCPKG_ROOT="${BUILD_ROOT}/vcpkg" - export VCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" VCPKG_HOST_TRIPLET="$VCPKG_TRIPLET" - export USE_MERGED_VCPKG_MANIFEST=1 - export EXT_CONFIG="${ICE}/extension_config.cmake" - export OVERRIDE_GIT_DESCRIBE="v${DUCKDB_VERSION}" - # Cap parallelism to bound peak memory (DuckDB can OOM at high -j); matches - # pg_duckdb and keeps a single cell within a GitHub runner / the Docker VM. - export CMAKE_BUILD_PARALLEL_LEVEL=4 - make -j4 release - ) - - echo "Staging built extensions + docs for packaging..." - rm -rf "$STAGE"; mkdir -p "$STAGE" - for e in iceberg avro azure postgres_scanner; do - cp "${ICE}/build/release/extension/${e}/${e}.duckdb_extension" "$STAGE/" - done cp "${COMPONENT_NAME}/config/coldfront-duckdb-extensions.conf.sample" "$STAGE/" cp "${CWD}/LICENSE.md" "${CWD}/README.md" "$STAGE/" cp "${COMPONENT_NAME}/rpm/coldfront-duckdb-extensions.spec" ~/rpmbuild/SPECS/ From 0ad0ec7fea1f88ff6ed4d8f099dcd5ee83d605ca Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 14:59:25 +0500 Subject: [PATCH 03/12] coldfront: commit common-functions.sh into common directory --- common/build.sh | 6 +- common/common-functions.sh | 266 +++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 2 deletions(-) create mode 100755 common/common-functions.sh diff --git a/common/build.sh b/common/build.sh index 4c8ec6f..42ea7e3 100644 --- a/common/build.sh +++ b/common/build.sh @@ -8,8 +8,10 @@ set -euo pipefail COMPONENT_NAME=$1 source "$(dirname "$0")/../${COMPONENT_NAME}/common.sh" -# common-functions.sh is copied into common/ by the workflow (builder-action) -# from the shared private repo; not committed here. +# common-functions.sh is a committed copy of the shared pgEdge build helpers +# (from pgedge-enterprise-packages/common/). The builder-action mounts the repo +# and runs this script but does not supply it, so it must live in the repo — +# same pattern as ai-dba-workbench's packaging/scripts/common-functions.sh. COMMON_FILE="$(dirname "$0")/common-functions.sh" if [ -f "$COMMON_FILE" ]; then source "$COMMON_FILE" diff --git a/common/common-functions.sh b/common/common-functions.sh new file mode 100755 index 0000000..171b397 --- /dev/null +++ b/common/common-functions.sh @@ -0,0 +1,266 @@ +#!/bin/bash + +install_syft(){ + + # Pin the installer to a tagged ref (not mutable main) AND pin the installed + # version, so release builds are reproducible and not exposed to upstream + # changes on syft's main branch. Override SYFT_VERSION to bump. + SYFT_VERSION="${SYFT_VERSION:-v1.45.1}" + echo "Installing syft ${SYFT_VERSION}..." + curl -sSfL "https://raw.githubusercontent.com/anchore/syft/${SYFT_VERSION}/install.sh" | sudo sh -s -- -b /usr/local/bin "${SYFT_VERSION}" +} + +setup_dnf_build_env(){ + + echo "Installing required packages..." + dnf groupinstall "Development Tools" -y + dnf install -y rpm-build rpmdevtools yum-utils tar wget git gnupg2 sudo + + echo "📦 Enabling additional repositories..." + dnf install -y epel-release + if [ "$RHEL" = "8" ]; then + dnf config-manager --set-enabled powertools + else + dnf config-manager --set-enabled crb + fi + + echo "Configuring pgEdge repository..." + configure_pgedge_dnf_repo $REPO_TYPE + + echo "Setting up RPM build environment..." + rpmdev-setuptree + + install_syft +} + +setup_apt_build_env(){ + + echo "Installing build tools and dependencies..." + sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime + + sudo apt-get update + sudo apt-get install -y devscripts build-essential pkg-config fakeroot git curl \ + ca-certificates debhelper dpkg-dev gnupg2 wget sudo lsb-release + + echo "Configuring pgEdge repository..." + configure_pgedge_apt_repo $REPO_TYPE + + install_syft +} + +rename_ddeb_packages(){ + # Rename any *.ddeb (debug-symbol packages) to *.deb. Uses find -print0 so it + # is safe under `set -euo pipefail` when there are NO .ddeb files (no grep + # non-zero exit) and tolerates filenames with spaces. + local build_dir="$1" f + find "$build_dir" -maxdepth 1 -type f -name '*.ddeb' -print0 | while IFS= read -r -d '' f; do + mv -- "$f" "${f%.ddeb}.deb" + done +} + +configure_pgedge_dnf_repo() { + local REPO_TYPE="${1:-daily}" # "daily" or "staging" + + sudo dnf install -y https://dnf.pgedge.com/reporpm/pgedge-release-latest.noarch.rpm + sudo sed -i "s|release|$REPO_TYPE|g" /etc/yum.repos.d/pgedge.repo + + echo "Repo configured at /etc/yum.repos.d/pgedge.repo" +} + +configure_pgedge_apt_repo(){ + local REPO_TYPE="${1:-daily}" # "daily" or "staging" + local REPO_PATH="repodeb" + + curl -sSL https://apt.pgedge.com/${REPO_PATH}/pgedge-release_latest_all.deb -o /tmp/pgedge-release.deb && sudo dpkg -i /tmp/pgedge-release.deb && rm -f /tmp/pgedge-release.deb || true + sed -i "s|release|$REPO_TYPE|g" /etc/apt/sources.list.d/pgedge.sources + apt-get update + + echo "Repo configured at /etc/apt/sources.list.d/pgedge.sources" +} + +detect_os_type(){ + if command -v dnf &>/dev/null || command -v yum &>/dev/null; then + echo "Detected RPM-based system" + source "$(dirname "$0")/../${COMPONENT_NAME}/build-rpm.sh" + elif command -v apt-get &>/dev/null; then + echo "Detected Debian-based system" + source "$(dirname "$0")/../${COMPONENT_NAME}/build-deb.sh" + else + echo "Unsupported platform: No known package manager found" >&2 + exit 1 + fi +} + +import_gpg_keys() { + if ! command -v rpm &>/dev/null || ! command -v gpg &>/dev/null; then + echo "Installing rpm or gpg" + if command -v dnf &>/dev/null; then + sudo dnf install -y rpm gnupg2 + elif command -v apt-get &>/dev/null; then + sudo apt-get install -y rpm gnupg2 + fi + if [ $? -ne 0 ]; then + echo "Error: Failed to install rpm or gnupg2" + return 1 + fi + fi + + PRI_FILE=$(dirname "$0")/public.key + PUB_FILE=$(dirname "$0")/private.key + + GPG_PUBLIC_KEY=$(cat $PRI_FILE) + GPG_PRIVATE_KEY=$(cat $PUB_FILE) + rm -f $PRI_FILE $PUB_FILE + + [ -z "$GPG_PUBLIC_KEY" ] && { echo "Error: GPG_PUBLIC_KEY is unset"; return 1; } + [ -z "$GPG_PRIVATE_KEY" ] && { echo "Error: GPG_PRIVATE_KEY is unset"; return 1; } + + PUBLIC_KEY_FILE=$(mktemp) + echo "$GPG_PUBLIC_KEY" > "$PUBLIC_KEY_FILE" + + gpg --import "$PUBLIC_KEY_FILE" || { + echo "Error: Failed to import public key" + rm -f "$PUBLIC_KEY_FILE" + return 1 + } + + rpm --import "$PUBLIC_KEY_FILE" || { + echo "Error: Failed to import public key to RPM" + rm -f "$PUBLIC_KEY_FILE" + return 1 + } + + PRIVATE_KEY_FILE=$(mktemp) + echo "$GPG_PRIVATE_KEY" > "$PRIVATE_KEY_FILE" + gpg --import "$PRIVATE_KEY_FILE" || { + echo "Error: Failed to import private key" + rm -f "$PRIVATE_KEY_FILE" + rm -f "$PUBLIC_KEY_FILE" + return 1 + } + rm -f "$PRIVATE_KEY_FILE" + rm -f "$PUBLIC_KEY_FILE" + return 0 +} + +sign_rpms() { + # Check if at least one file is provided + if [ $# -eq 0 ]; then + echo "Error: No files provided to sign." + return 1 + fi + + # Check if rpmsign and gpg are installed, install if not + if ! command -v rpmsign &>/dev/null; then + echo "rpmsign not found. Installing rpm-sign" + if command -v sudo &>/dev/null; then + sudo dnf install -y rpm-sign + else + dnf install -y rpm-sign + fi + if [ $? -ne 0 ]; then + echo "Error: Failed to install rpm-sign" + return 1 + fi + fi + + # Get the key ID of the imported private key + KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1) + if [ -z "$KEY_ID" ]; then + echo "Error: No private key found after import." + rm -f "$PRIVATE_KEY_FILE" + rm -rf "$GNUPGHOME" + return 1 + fi + + echo "=======================Signing RPMs=======================" + # Sign each RPM file + for file in "$@"; do + # Ensure the file has /output/ prefix if relative + if [[ ! "$file" = /* ]]; then + file="/output/$file" + fi + + if [ ! -f "$file" ]; then + echo "Error: File '$file' does not exist." + continue + fi + + # Check if the file is an RPM + if ! file "$file" | grep -q "RPM"; then + echo "Error: File '$file' is not an RPM file." + continue + fi + + # Sign the RPM using rpmsign, using passphrase if provided + rpmsign --define "_gpg_name $KEY_ID" --addsign "$file" >/dev/null 2>&1 + + if [ $? -eq 0 ]; then + echo "Successfully signed '$file'." + else + echo "Error: Failed to sign '$file'." + fi + done + echo "=======================Signing Completes==================" + + # Clean up + rm -f "$PRIVATE_KEY_FILE" +} + +validate_signatures() { + + # Check if files are provided + if [ $# -eq 0 ]; then + echo "Error: No files provided to validate." + return 1 + fi + + # Install dependencies + if ! command -v rpm &>/dev/null; then + echo "Installing rpm" + if command -v sudo &>/dev/null; then + sudo dnf install -y rpm + else + dnf install -y rpm + fi + if [ $? -ne 0 ]; then + echo "Error: Failed to install rpm" + return 1 + fi + fi + + # Validate each RPM + local all_valid=0 + echo "=======================Starting validation=======================" + for file in "$@"; do + if [[ ! "$file" = /* ]]; then + file="/output/$file" + fi + + if [ ! -f "$file" ]; then + echo "Error: File '$file' does not exist." + all_valid=1 + continue + fi + + if ! file "$file" | grep -q "RPM"; then + echo "Error: File '$file' is not an RPM file." + all_valid=1 + continue + fi + + CHECKSIG_OUTPUT=$(rpm --checksig "$file" 2>&1) + echo "$CHECKSIG_OUTPUT" + if echo "$CHECKSIG_OUTPUT" | grep -q "digests signatures OK"; then + echo "Signature for '$file' is valid." + else + echo "Error: Signature for '$file' is invalid or missing." + all_valid=1 + fi + done + echo "=======================Validation completes======================" + # Clean up + rm -f "$PUBLIC_KEY_FILE" + + return $all_valid +} From 1efcc23b9b65b3d16d4b9cda34be4a77013987cf Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 15:19:42 +0500 Subject: [PATCH 04/12] pg_duckdb: pin upstream commit via PG_DUCKDB_COMMIT, not COMPONENT_BRANCH --- packaging/pg_duckdb/common.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/pg_duckdb/common.sh b/packaging/pg_duckdb/common.sh index 505eaf0..b4df875 100644 --- a/packaging/pg_duckdb/common.sh +++ b/packaging/pg_duckdb/common.sh @@ -10,9 +10,11 @@ export PG_MAJOR_VERSION="$(echo "$PG_VERSION" | cut -d. -f1)" # pg_duckdb pinned to the merged PR #1025 commit (DuckDB 1.5.4). No released # pg_duckdb tag carries 1.5.x; this commit pins its duckdb submodule to v1.5.4. -# COMPONENT_BRANCH/COMPONENT_VERSION may override for a rebuild. +# This is an UPSTREAM pin, independent of the ColdFront release tag — do NOT +# derive it from COMPONENT_BRANCH (the builder-action always sets that to the +# ColdFront tag, which is not a pg_duckdb ref). Override only via PG_DUCKDB_COMMIT. export PG_DUCKDB_REPO="https://github.com/duckdb/pg_duckdb" -export PG_DUCKDB_COMMIT="${COMPONENT_BRANCH:-c04e6a2dcf4e999abb921da1ba2f8335dad644e0}" +export PG_DUCKDB_COMMIT="${PG_DUCKDB_COMMIT:-c04e6a2dcf4e999abb921da1ba2f8335dad644e0}" export PG_DUCKDB_VERSION="${COMPONENT_VERSION:-1.5.4}" export PG_DUCKDB_BUILDNUM="${COMPONENT_BUILDNUM:-1}" From c224739eac3110950de6b984ca61eeb571c2a9b5 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 16:49:33 +0500 Subject: [PATCH 05/12] release.yml: free ~25GB runner disk before pg_duckdb cells (DuckDB LTO link was hitting No space left on device) + a note to start using /mnt in future if required. --- .github/workflows/release.yml | 42 +++++++++++++++++++++++++++++++++++ packaging/pg_duckdb/common.sh | 7 ++++++ 2 files changed, 49 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4bbc9b..be4254f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -414,6 +414,27 @@ jobs: set -euo pipefail echo "os_version=${IMAGE##*:}" >> "$GITHUB_OUTPUT" echo "safe_comp=${COMP##*/}" >> "$GITHUB_OUTPUT" + - name: Free up disk space (pg_duckdb compiles DuckDB from source) + if: matrix.component_name == 'packaging/pg_duckdb' + run: | + # pg_duckdb builds all of DuckDB + extensions with LTO — the link + # step is disk-heavy and once hit "No space left on device" on the + # ~14GB-free hosted runner. Reclaiming preinstalled bloat below + # takes / to ~40GB, comfortably enough. + # ESCALATION (if this ever still runs out): relocate Docker storage + # to the runner's larger /mnt (~70GB) disk before the build — the + # DuckDB compile runs inside the builder-action container, so moving + # Docker's data-root moves the build: + # sudo systemctl stop docker + # sudo mkdir -p /mnt/docker + # echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json + # sudo systemctl start docker + echo "before: $(df -h / | awk 'NR==2{print $4" free"}')" + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost \ + /usr/local/lib/node_modules /usr/share/swift "${AGENT_TOOLSDIRECTORY:-}" || true + sudo docker image prune -af >/dev/null 2>&1 || true + echo "after: $(df -h / | awk 'NR==2{print $4" free"}')" - name: Stage binaries tarball (coldfront-bin only) if: matrix.component_name == 'packaging/coldfront-bin' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -491,6 +512,27 @@ jobs: set -euo pipefail echo "distro=${IMAGE##*:}" >> "$GITHUB_OUTPUT" echo "safe_comp=${COMP##*/}" >> "$GITHUB_OUTPUT" + - name: Free up disk space (pg_duckdb compiles DuckDB from source) + if: matrix.component_name == 'packaging/pg_duckdb' + run: | + # pg_duckdb builds all of DuckDB + extensions with LTO — the link + # step is disk-heavy and once hit "No space left on device" on the + # ~14GB-free hosted runner. Reclaiming preinstalled bloat below + # takes / to ~40GB, comfortably enough. + # ESCALATION (if this ever still runs out): relocate Docker storage + # to the runner's larger /mnt (~70GB) disk before the build — the + # DuckDB compile runs inside the builder-action container, so moving + # Docker's data-root moves the build: + # sudo systemctl stop docker + # sudo mkdir -p /mnt/docker + # echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json + # sudo systemctl start docker + echo "before: $(df -h / | awk 'NR==2{print $4" free"}')" + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost \ + /usr/local/lib/node_modules /usr/share/swift "${AGENT_TOOLSDIRECTORY:-}" || true + sudo docker image prune -af >/dev/null 2>&1 || true + echo "after: $(df -h / | awk 'NR==2{print $4" free"}')" - name: Stage binaries tarball (coldfront-bin only) if: matrix.component_name == 'packaging/coldfront-bin' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 diff --git a/packaging/pg_duckdb/common.sh b/packaging/pg_duckdb/common.sh index b4df875..39b647d 100644 --- a/packaging/pg_duckdb/common.sh +++ b/packaging/pg_duckdb/common.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash # common.sh - Common environment variables for the ColdFront pg_duckdb package. +# +# NOTE (disk): this package compiles all of DuckDB + extensions with LTO and is +# the heaviest build here — it can exhaust a hosted runner's disk. The release +# workflow frees ~25-30GB before pg_duckdb cells; if that ever still hits "No +# space left on device", the next lever is to move Docker's storage to the +# runner's /mnt (~70GB) disk. See the "Free up disk space" step in +# .github/workflows/release.yml for the ready-to-use snippet. # Build once per PostgreSQL major version (fan-out across the matrix). PER_PG_VERSION=true From 2c0ca9d8f71a7d12640861db7e2a03079cf0addf Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 2 Jul 2026 18:17:58 +0500 Subject: [PATCH 06/12] =?UTF-8?q?coldfront:=20require=20coldfront-duckdb-e?= =?UTF-8?q?xtensions=20(RPM=20+=20DEB)=20=E2=80=94=20cold=20tier=20needs?= =?UTF-8?q?=20the=20patched=20extensions=20at=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packaging/coldfront/deb/debian/control.in | 1 + packaging/coldfront/rpm/coldfront.spec | 1 + 2 files changed, 2 insertions(+) diff --git a/packaging/coldfront/deb/debian/control.in b/packaging/coldfront/deb/debian/control.in index e36a9e1..0c16e5c 100644 --- a/packaging/coldfront/deb/debian/control.in +++ b/packaging/coldfront/deb/debian/control.in @@ -18,6 +18,7 @@ Depends: ${shlibs:Depends}, pgedge-postgresql-PG_MAJOR_VERSION, pgedge-postgresql-PG_MAJOR_VERSION-pg-duckdb, + pgedge-coldfront-duckdb-extensions, Description: Transparent tiered storage for PostgreSQL (hot PG + cold Iceberg) ColdFront routes DML on tiered views across a hot tier (PostgreSQL) and a cold tier (Apache Iceberg over object storage), with cold reads/writes served diff --git a/packaging/coldfront/rpm/coldfront.spec b/packaging/coldfront/rpm/coldfront.spec index 3032b6f..b8c7129 100644 --- a/packaging/coldfront/rpm/coldfront.spec +++ b/packaging/coldfront/rpm/coldfront.spec @@ -19,6 +19,7 @@ Requires: pgedge-postgresql%{pgmajorversion}-server # coldfront drives DuckDB through pg_duckdb (shared_preload_libraries + # duckdb.raw_query); it is non-functional without it. Requires: pgedge-pg-duckdb_%{pgmajorversion} +Requires: pgedge-coldfront-duckdb-extensions Provides: %{sname}_%{pgmajorversion} %description From ceca1fe2212621f4cd0d43f1cbcbc2a8117d77f5 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Fri, 3 Jul 2026 15:00:51 +0500 Subject: [PATCH 07/12] Fix backup paths in release manifest and add backup paths to Slack notification --- .github/workflows/release.yml | 49 +++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be4254f..28215d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -901,21 +901,36 @@ jobs: run: | set -euo pipefail mkdir -p aggregated - cells='[]' + # Build cells entirely in jq (NOT a bash `read` loop): several + # components carry an empty component_version, and reading @tsv + # with IFS=tab collapses that empty field — shifting every later + # column and corrupting os_version/arch/backup_path. jq is immune. + # backup_path mirrors pgedge-backup-artifacts exactly: + # ////// + # where os_version is the EL major (dnf) or distro codename (apt); + # the arch is a subdir INSIDE that prefix, so it's not in the path. + rpm_cells='[]'; deb_cells='[]'; dnf_paths='[]'; apt_paths='[]' if [ "${PUSH_DNF_RESULT}" = "success" ] && [ -n "${DNF_TS}" ]; then - while IFS=$'\t' read -r comp cver image os arch; do - path="${REPO_TYPE}/${COMPONENT_NAME}/${CV}/${DNF_TS}/dnf/${os}/" - cell="$(jq -nc --arg fam rpm --arg comp "$comp" --arg cv "$cver" --arg img "$image" --arg ver "$os" --arg arch "$arch" --arg bp "$path" --arg ts "$DNF_TS" --arg rt "$REPO_TYPE" '{family:$fam,component:$comp,component_version:$cv,image:$img,arch:$arch,os_version:$ver,backup_path:$bp,timestamp:$ts,repo_type:$rt}')" - cells="$(jq -c --argjson c "$cell" '. + [$c]' <<<"$cells")" - done < <(echo "$RPM_MATRIX" | jq -r '.include[] | [.component_name, .component_version, .image, (.image|split(":")[1]), .arch] | @tsv') + rpm_cells="$(echo "$RPM_MATRIX" | jq -c \ + --arg rt "$REPO_TYPE" --arg cn "$COMPONENT_NAME" --arg cv "$CV" --arg ts "$DNF_TS" ' + [ .include[] | (.image | split(":")[1]) as $os | { + family:"rpm", component:.component_name, component_version:.component_version, + image:.image, arch:.arch, os_version:$os, + backup_path:($rt+"/"+$cn+"/"+$cv+"/"+$ts+"/dnf/"+$os+"/"), + timestamp:$ts, repo_type:$rt } ]')" + dnf_paths="$(jq -c '[.[].backup_path] | unique' <<<"$rpm_cells")" fi if [ "${PUSH_APT_RESULT}" = "success" ] && [ -n "${APT_TS}" ]; then - while IFS=$'\t' read -r comp cver image distro arch; do - path="${REPO_TYPE}/${COMPONENT_NAME}/${CV}/${APT_TS}/apt/${distro}/" - cell="$(jq -nc --arg fam deb --arg comp "$comp" --arg cv "$cver" --arg img "$image" --arg distro "$distro" --arg arch "$arch" --arg bp "$path" --arg ts "$APT_TS" --arg rt "$REPO_TYPE" '{family:$fam,component:$comp,component_version:$cv,image:$img,arch:$arch,distro:$distro,backup_path:$bp,timestamp:$ts,repo_type:$rt}')" - cells="$(jq -c --argjson c "$cell" '. + [$c]' <<<"$cells")" - done < <(echo "$DEB_MATRIX" | jq -r '.include[] | [.component_name, .component_version, .image, (.image|split(":")[1]), .arch] | @tsv') + deb_cells="$(echo "$DEB_MATRIX" | jq -c \ + --arg rt "$REPO_TYPE" --arg cn "$COMPONENT_NAME" --arg cv "$CV" --arg ts "$APT_TS" ' + [ .include[] | (.image | split(":")[1]) as $distro | { + family:"deb", component:.component_name, component_version:.component_version, + image:.image, arch:.arch, distro:$distro, + backup_path:($rt+"/"+$cn+"/"+$cv+"/"+$ts+"/apt/"+$distro+"/"), + timestamp:$ts, repo_type:$rt } ]')" + apt_paths="$(jq -c '[.[].backup_path] | unique' <<<"$deb_cells")" fi + cells="$(jq -c -n --argjson r "$rpm_cells" --argjson d "$deb_cells" '$r + $d')" jq -n \ --arg run_id "${GITHUB_RUN_ID}" --arg run_attempt "${GITHUB_RUN_ATTEMPT}" \ --arg repo "${GITHUB_REPOSITORY}" --arg tag "${EFFECTIVE_TAG}" \ @@ -925,7 +940,8 @@ jobs: --arg dnf_timestamp "${DNF_TS}" --arg apt_timestamp "${APT_TS}" \ --arg push_dnf_outcome "${PUSH_DNF_RESULT}" --arg push_apt_outcome "${PUSH_APT_RESULT}" \ --argjson cells "${cells}" \ - '{run_id:$run_id, run_attempt:$run_attempt, repo:$repo, tag:$tag, simulated:$simulated, commit_sha:$commit_sha, component_name:$component_name, component_version:$component_version, component_buildnum:$component_buildnum, repo_type:$repo_type, timestamps:{dnf:$dnf_timestamp, apt:$apt_timestamp}, push_results:{dnf:$push_dnf_outcome, apt:$push_apt_outcome}, cells:$cells}' \ + --argjson dnf_paths "${dnf_paths}" --argjson apt_paths "${apt_paths}" \ + '{run_id:$run_id, run_attempt:$run_attempt, repo:$repo, tag:$tag, simulated:$simulated, commit_sha:$commit_sha, component_name:$component_name, component_version:$component_version, component_buildnum:$component_buildnum, repo_type:$repo_type, timestamps:{dnf:$dnf_timestamp, apt:$apt_timestamp}, push_results:{dnf:$push_dnf_outcome, apt:$push_apt_outcome}, backups:{dnf:$dnf_paths, apt:$apt_paths}, cells:$cells}' \ > aggregated/manifest.json cat aggregated/manifest.json { echo "manifest_json<<__EOF__"; jq -c '.' aggregated/manifest.json; echo "__EOF__"; } >> "$GITHUB_OUTPUT" @@ -963,7 +979,13 @@ jobs: if [ -f aggregated/manifest.json ]; then a=$(jq -r '(.cells // []) | map(.arch) | unique | join(", ")' aggregated/manifest.json); [ -n "$a" ] && archs="$a"; fi fields=$(jq -nc --arg channel "${REPO_TYPE}" --arg version "${CV}" --arg buildnum "${CBN}" --arg archs "${archs}" --arg actor "${ACTOR}" --arg unix_ts "${unix_ts}" --arg fallback_ts "${fallback_ts}" '[{title:"Channel",value:("`"+$channel+"`")},{title:"Version",value:("`"+$version+"`")},{title:"Build #",value:("`"+$buildnum+"`")},{title:"Archs",value:("`"+$archs+"`")},{title:"Triggered by",value:$actor},{title:"When",value:("")}]') result_fields=$(jq -nc --arg dnf "${PUSH_DNF_RESULT}" --arg apt "${PUSH_APT_RESULT}" '[{title:"Push DNF",value:("`"+$dnf+"`")},{title:"Push APT",value:("`"+$apt+"`")}]') - { echo "status=${status}"; echo "subject<<__S__"; echo "${subject}"; echo "__S__"; echo "fields<<__F__"; echo "${fields}"; echo "__F__"; echo "result_fields<<__RF__"; echo "${result_fields}"; echo "__RF__"; } >> "$GITHUB_OUTPUT" + # Backup paths as code blocks (one per OS major / distro; arch is a + # subdir inside). Publisher elides a section whose lines[] is empty, + # so a skipped push simply shows nothing. + dnf_paths=$(jq -c '.backups.dnf // []' aggregated/manifest.json) + apt_paths=$(jq -c '.backups.apt // []' aggregated/manifest.json) + code_sections=$(jq -nc --argjson dnf "${dnf_paths}" --argjson apt "${apt_paths}" '[{title:"DNF backup paths",lines:$dnf},{title:"APT backup paths",lines:$apt}]') + { echo "status=${status}"; echo "subject<<__S__"; echo "${subject}"; echo "__S__"; echo "fields<<__F__"; echo "${fields}"; echo "__F__"; echo "result_fields<<__RF__"; echo "${result_fields}"; echo "__RF__"; echo "code_sections<<__CS__"; echo "${code_sections}"; echo "__CS__"; } >> "$GITHUB_OUTPUT" - name: Publish manifest + Slack notification uses: ./.github/actions/pgedge-build-publisher with: @@ -979,6 +1001,7 @@ jobs: subject: ${{ steps.compose_slack.outputs.subject }} fields_json: ${{ steps.compose_slack.outputs.fields }} result_fields_json: ${{ steps.compose_slack.outputs.result_fields }} + code_sections_json: ${{ steps.compose_slack.outputs.code_sections }} tagline: "pgEdge ColdFront release pipeline" aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From ebdf5cc5c45e528198ec162180bc08634fa21a54 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Fri, 3 Jul 2026 15:34:51 +0500 Subject: [PATCH 08/12] Install coldfront config.yaml to /etc/pgedge/coldfront instead of /etc/coldfront --- .../coldfront-bin/deb/debian/pgedge-coldfront.conffiles | 2 +- packaging/coldfront-bin/deb/debian/rules | 2 +- packaging/coldfront-bin/rpm/coldfront.spec | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles index 79ff6ba..1a75ecb 100644 --- a/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles +++ b/packaging/coldfront-bin/deb/debian/pgedge-coldfront.conffiles @@ -1 +1 @@ -/etc/coldfront/config.yaml +/etc/pgedge/coldfront/config.yaml diff --git a/packaging/coldfront-bin/deb/debian/rules b/packaging/coldfront-bin/deb/debian/rules index 64bdffc..a1f5241 100644 --- a/packaging/coldfront-bin/deb/debian/rules +++ b/packaging/coldfront-bin/deb/debian/rules @@ -28,4 +28,4 @@ execute_after_dh_install: # config.yaml. The tools take `-config `; point them at this. It lands # under /etc and debian/pgedge-coldfront.conffiles marks it a conffile, so an # admin's edited copy survives upgrades. - install -D -m 0644 config.example.yaml debian/pgedge-coldfront/etc/coldfront/config.yaml + install -D -m 0644 config.example.yaml debian/pgedge-coldfront/etc/pgedge/coldfront/config.yaml diff --git a/packaging/coldfront-bin/rpm/coldfront.spec b/packaging/coldfront-bin/rpm/coldfront.spec index 767a1b2..2b6cedb 100644 --- a/packaging/coldfront-bin/rpm/coldfront.spec +++ b/packaging/coldfront-bin/rpm/coldfront.spec @@ -43,7 +43,7 @@ install -D -m 0755 compactor %{buildroot}%{_bindir}/compactor # Ship the example deployment config as the default config file. The tools take # `-config `; point them at this. Installed %config(noreplace) so an admin's # edited copy survives upgrades (a new default lands as config.yaml.rpmnew). -install -D -m 0644 config.example.yaml %{buildroot}%{_sysconfdir}/coldfront/config.yaml +install -D -m 0644 config.example.yaml %{buildroot}%{_sysconfdir}/pgedge/coldfront/config.yaml install -D -m 0644 %{sname}-sbom.json %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc @@ -53,8 +53,9 @@ install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname %{_bindir}/archiver %{_bindir}/partitioner %{_bindir}/compactor -%dir %{_sysconfdir}/coldfront -%config(noreplace) %{_sysconfdir}/coldfront/config.yaml +%dir %{_sysconfdir}/pgedge +%dir %{_sysconfdir}/pgedge/coldfront +%config(noreplace) %{_sysconfdir}/pgedge/coldfront/config.yaml %{_datadir}/pgedge-%{sname}/%{sname}-sbom.json %{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc From 1470320c1a57671439d80c5ba2f29d97178b13ff Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Mon, 6 Jul 2026 16:10:01 +0500 Subject: [PATCH 09/12] Add lakekeeper native RPM/DEB packaging and wire it into release.yml --- .github/workflows/release.yml | 38 ++++--- packaging/lakekeeper/README.md | 62 ++++++++++++ packaging/lakekeeper/build-deb.sh | 72 ++++++++++++++ packaging/lakekeeper/build-rpm.sh | 56 +++++++++++ packaging/lakekeeper/common.sh | 29 ++++++ packaging/lakekeeper/common/lakekeeper.env | 32 ++++++ .../lakekeeper/common/lakekeeper.service | 29 ++++++ packaging/lakekeeper/deb/debian/control | 29 ++++++ .../deb/debian/pgedge-lakekeeper.conffiles | 1 + .../deb/debian/pgedge-lakekeeper.docs | 3 + .../deb/debian/pgedge-lakekeeper.install | 2 + packaging/lakekeeper/deb/debian/postinst | 23 +++++ packaging/lakekeeper/deb/debian/rules | 38 +++++++ packaging/lakekeeper/deb/debian/source/format | 1 + packaging/lakekeeper/rpm/lakekeeper.spec | 98 +++++++++++++++++++ 15 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 packaging/lakekeeper/README.md create mode 100755 packaging/lakekeeper/build-deb.sh create mode 100755 packaging/lakekeeper/build-rpm.sh create mode 100755 packaging/lakekeeper/common.sh create mode 100644 packaging/lakekeeper/common/lakekeeper.env create mode 100644 packaging/lakekeeper/common/lakekeeper.service create mode 100644 packaging/lakekeeper/deb/debian/control create mode 100644 packaging/lakekeeper/deb/debian/pgedge-lakekeeper.conffiles create mode 100644 packaging/lakekeeper/deb/debian/pgedge-lakekeeper.docs create mode 100644 packaging/lakekeeper/deb/debian/pgedge-lakekeeper.install create mode 100755 packaging/lakekeeper/deb/debian/postinst create mode 100644 packaging/lakekeeper/deb/debian/rules create mode 100644 packaging/lakekeeper/deb/debian/source/format create mode 100644 packaging/lakekeeper/rpm/lakekeeper.spec diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28215d2..307dd11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,15 +5,17 @@ # Tag-driven. A pushed v* tag builds and publishes, in one GitHub Actions run: # 1. GoReleaser: the Go binaries (archiver, partitioner, compactor) + the # GitHub release, and uploads the linux tarballs as a workflow artifact. -# 2. Native RPM/DEB packages for four components, across PG 16/17/18 and -# el9/el10 + jammy/noble/bullseye/bookworm/trixie, amd64 + arm64: +# 2. Native RPM/DEB packages for five components, across PG 16/17/18 and +# el9/el10 + jammy/noble/bookworm/trixie, amd64 + arm64: # * packaging/coldfront-bin — the Go tools (PG-independent, per-arch) # * packaging/coldfront — the coldfront PG extension (per PG major) # * packaging/pg_duckdb — pg_duckdb 1.5.4 (per PG major) # * packaging/coldfront-duckdb-extensions — patched iceberg/avro/azure/ # postgres_scanner .duckdb_extension. -# pg_duckdb + duckdb-extensions keep their own version (1.5.4); coldfront-bin -# + the extension track the release tag. +# * packaging/lakekeeper — Apache Iceberg REST Catalog, prebuilt binary +# (PG-independent, per-arch; needs glibc >= 2.34). +# pg_duckdb + duckdb-extensions keep their own version (1.5.4), lakekeeper +# its own (0.13.1); coldfront-bin + the extension track the release tag. # # The duckdb-extensions are BUILD-ONCE per arch: build-duckdb-ext compiles # them a single time on manylinux_2_28 (glibc 2.28 baseline → portable to @@ -199,7 +201,7 @@ jobs: pg_versions: '16,17,18' archs: '["amd64","arm64"]' rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' - name: Detect — coldfront extension (PG 16/17/18) id: coldfront uses: ./.github/actions/pgedge-detect-build-matrix @@ -208,7 +210,7 @@ jobs: pg_versions: '16,17,18' archs: '["amd64","arm64"]' rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' - name: Detect — coldfront binaries (PG-independent) id: bin uses: ./.github/actions/pgedge-detect-build-matrix @@ -216,7 +218,7 @@ jobs: component_name: packaging/coldfront-bin archs: '["amd64","arm64"]' rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' - name: Detect — coldfront duckdb-extensions (PG-independent) id: duckdbext uses: ./.github/actions/pgedge-detect-build-matrix @@ -224,7 +226,15 @@ jobs: component_name: packaging/coldfront-duckdb-extensions archs: '["amd64","arm64"]' rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bullseye","debian:bookworm","debian:trixie"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + - name: Detect — lakekeeper (PG-independent; prebuilt binary, needs glibc >= 2.34) + id: lakekeeper + uses: ./.github/actions/pgedge-detect-build-matrix + with: + component_name: packaging/lakekeeper + archs: '["amd64","arm64"]' + rpm_images: '["almalinux:9","almalinux:10"]' + deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' - name: Merge component matrices id: merge env: @@ -237,6 +247,8 @@ jobs: DEB_BIN: ${{ steps.bin.outputs.deb_matrix }} RPM_DUX: ${{ steps.duckdbext.outputs.rpm_matrix }} DEB_DUX: ${{ steps.duckdbext.outputs.deb_matrix }} + RPM_LK: ${{ steps.lakekeeper.outputs.rpm_matrix }} + DEB_LK: ${{ steps.lakekeeper.outputs.deb_matrix }} run: | set -euo pipefail # Tag a component's cells with component_name + component_version. @@ -251,16 +263,20 @@ jobs: echo "$1={\"include\":$2}" >> "$GITHUB_OUTPUT" } # Source-built components (their cells compile in-container). + # lakekeeper keeps its own version (empty -> common.sh default, + # 0.13.1), like pg_duckdb + duckdb-extensions. rpm_inc="$(jq -c -n \ --argjson a "$(tag_cells "$RPM_PGD" packaging/pg_duckdb '')" \ --argjson b "$(tag_cells "$RPM_CF" packaging/coldfront "$TAG_VERSION")" \ --argjson d "$(tag_cells "$RPM_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ - '$a + $b + $d')" + --argjson e "$(tag_cells "$RPM_LK" packaging/lakekeeper '')" \ + '$a + $b + $d + $e')" deb_inc="$(jq -c -n \ --argjson a "$(tag_cells "$DEB_PGD" packaging/pg_duckdb '')" \ --argjson b "$(tag_cells "$DEB_CF" packaging/coldfront "$TAG_VERSION")" \ --argjson d "$(tag_cells "$DEB_BIN" packaging/coldfront-bin "$TAG_VERSION")" \ - '$a + $b + $d')" + --argjson e "$(tag_cells "$DEB_LK" packaging/lakekeeper '')" \ + '$a + $b + $d + $e')" # duckdb-extensions: build-once, repackaged per distro/arch. ext_rpm_inc="$(tag_cells "$RPM_DUX" packaging/coldfront-duckdb-extensions '')" ext_deb_inc="$(tag_cells "$DEB_DUX" packaging/coldfront-duckdb-extensions '')" @@ -362,7 +378,7 @@ jobs: CLI_URL="https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-${CLI_ARCH}.zip" echo "Engine CLI: $CLI_URL" pass=0; fail=0 - for img in almalinux:9 almalinux:10 ubuntu:jammy ubuntu:noble debian:bullseye debian:bookworm debian:trixie; do + for img in almalinux:9 almalinux:10 ubuntu:jammy ubuntu:noble debian:bookworm debian:trixie; do echo "###### $img ######" cid=$(docker run -d -v "$PWD/ext:/ext:ro" "$img" sleep 600) || { echo " run failed"; fail=$((fail+1)); continue; } docker exec "$cid" bash -c ' diff --git a/packaging/lakekeeper/README.md b/packaging/lakekeeper/README.md new file mode 100644 index 0000000..d49f1ed --- /dev/null +++ b/packaging/lakekeeper/README.md @@ -0,0 +1,62 @@ +# pgedge-lakekeeper + +[Lakekeeper](https://lakekeeper.io) is an implementation of the Apache Iceberg +REST Catalog. This package installs the prebuilt `lakekeeper` binary and a +systemd service (`lakekeeper.service`). + +The service is **not enabled or started automatically** — Lakekeeper needs an +external PostgreSQL database and some required configuration before it can run. + +## 1. Provide a PostgreSQL database + +Lakekeeper stores all catalog state in PostgreSQL **15 or newer**. Create a +database and a role, for example: + +```sql +CREATE ROLE lakekeeper LOGIN PASSWORD 'change-me'; +CREATE DATABASE lakekeeper OWNER lakekeeper; +``` + +The one-time migration (step 3) creates the extensions Lakekeeper needs +(`uuid-ossp`, `pgcrypto`, `pg_trgm`, `btree_gin`, `btree_gist`), so the role +must be able to `CREATE EXTENSION` — or create them ahead of time as a +superuser. + +## 2. Configure `/etc/lakekeeper/lakekeeper.env` + +Edit the environment file and set at least the two required variables: + +- `LAKEKEEPER__PG_DATABASE_URL_WRITE` — the PostgreSQL connection string. +- `LAKEKEEPER__PG_ENCRYPTION_KEY` — a strong random secret that encrypts stored + credentials. Generate one with `openssl rand -base64 32`. **Keep it stable and + back it up**: it must be identical across nodes sharing the same catalog, and + losing it makes stored secrets unrecoverable. + +Optional settings (read replica, listen address/port, authorization backend) +are documented inline in the file. If no authorization backend is configured +the catalog is **open** — configure authentication/authorization before +exposing it in production. + +## 3. Run the one-time database migration + +Lakekeeper must migrate the database before the first `serve`: + +```bash +# Load the same settings the service uses, then migrate as the lakekeeper user. +set -a; . /etc/lakekeeper/lakekeeper.env; set +a +sudo -E -u lakekeeper /usr/bin/lakekeeper migrate +``` + +Alternatively, uncomment `ExecStartPre=/usr/bin/lakekeeper migrate` in +`/usr/lib/systemd/system/lakekeeper.service` to run the idempotent migration +automatically on every start. + +## 4. Start the service + +```bash +sudo systemctl enable --now lakekeeper +sudo systemctl status lakekeeper +``` + +Lakekeeper listens on `0.0.0.0:8181` by default. Logs go to the journal +(`journalctl -u lakekeeper`). diff --git a/packaging/lakekeeper/build-deb.sh b/packaging/lakekeeper/build-deb.sh new file mode 100755 index 0000000..56b75b4 --- /dev/null +++ b/packaging/lakekeeper/build-deb.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUILD_DIR="/tmp/pg_deb_build" +SRC_DIR="${BUILD_DIR}/lakekeeper-${LAKEKEEPER_VERSION}" +CWD="$(pwd)" + +export DEBIAN_FRONTEND=noninteractive + +# Map the dpkg arch to the upstream target-triple prefix. +DEB_ARCH=$(dpkg --print-architecture) # amd64 | arm64 +case "$DEB_ARCH" in + amd64) UNAME_ARCH=x86_64 ;; + arm64) UNAME_ARCH=aarch64 ;; + *) echo "unsupported arch: $DEB_ARCH" >&2; exit 1 ;; +esac + +RELEASE_URL="https://github.com/lakekeeper/lakekeeper/releases/download/v${LAKEKEEPER_VERSION}" +REPO_RAW="https://raw.githubusercontent.com/lakekeeper/lakekeeper/v${LAKEKEEPER_VERSION}" + +prepare() { + setup_apt_build_env + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys + + echo "Downloading lakekeeper ${LAKEKEEPER_VERSION} binary (${UNAME_ARCH})..." + rm -rf "$BUILD_DIR" + mkdir -p "$SRC_DIR" + local tarball="lakekeeper-${UNAME_ARCH}-unknown-linux-gnu.tar.gz" + wget -q "${RELEASE_URL}/${tarball}" -O "${BUILD_DIR}/${tarball}" + tar -C "$SRC_DIR" -xzf "${BUILD_DIR}/${tarball}" + + # LICENSE + NOTICE are not inside the tarball; fetch them from the repo tag. + echo "Fetching LICENSE + NOTICE from the ${LAKEKEEPER_VERSION} tag..." + wget -q "${REPO_RAW}/LICENSE" -O "${SRC_DIR}/LICENSE" + wget -q "${REPO_RAW}/NOTICE" -O "${SRC_DIR}/NOTICE" + cp "${CWD}/${COMPONENT_NAME}/README.md" "${SRC_DIR}/README.md" + + echo "Staging Debian packaging..." + cp -rp "${CWD}/${COMPONENT_NAME}/deb/debian" "$SRC_DIR/" + cp "${CWD}/${COMPONENT_NAME}/common/lakekeeper.service" "$SRC_DIR/debian/lakekeeper.service" + cp "${CWD}/${COMPONENT_NAME}/common/lakekeeper.env" "$SRC_DIR/debian/lakekeeper.env" + + echo "Installing build dependencies..." + cd "$SRC_DIR" + sudo apt-get update + sudo apt-get build-dep -y . +} + +build() { + cd "$SRC_DIR" + echo "Building Debian package..." + DISTRO=$(lsb_release -cs) + rm -f debian/changelog +cat > debian/changelog < $(date -R) +EOF + + dpkg-buildpackage -us -uc -b +} + +post_build() { + echo "Copying .deb packages to output..." + sudo mkdir -p "/output" + rename_ddeb_packages "$BUILD_DIR" + sudo cp "$BUILD_DIR"/*.deb "/output" || echo "No .deb packages found." +} diff --git a/packaging/lakekeeper/build-rpm.sh b/packaging/lakekeeper/build-rpm.sh new file mode 100755 index 0000000..92f7c09 --- /dev/null +++ b/packaging/lakekeeper/build-rpm.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -euo pipefail + +# setup_dnf_build_env (common-functions.sh) reads $RHEL under `set -u`. +RHEL="$(rpm --eval %rhel)" + +# uname arch matches the upstream target-triple prefix (x86_64 / aarch64). +ARCH=$(uname -m) +case "$ARCH" in + x86_64|aarch64) : ;; + *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; +esac + +RELEASE_URL="https://github.com/lakekeeper/lakekeeper/releases/download/v${LAKEKEEPER_VERSION}" +REPO_RAW="https://raw.githubusercontent.com/lakekeeper/lakekeeper/v${LAKEKEEPER_VERSION}" + +prepare() { + setup_dnf_build_env + + echo "Copying packaging files..." + cp "${COMPONENT_NAME}/rpm/lakekeeper.spec" ~/rpmbuild/SPECS/ + cp "${COMPONENT_NAME}/common/lakekeeper.service" ~/rpmbuild/SOURCES/ + cp "${COMPONENT_NAME}/common/lakekeeper.env" ~/rpmbuild/SOURCES/ + cp "${COMPONENT_NAME}/README.md" ~/rpmbuild/SOURCES/ + + local tarball="lakekeeper-${ARCH}-unknown-linux-gnu.tar.gz" + echo "Downloading lakekeeper ${LAKEKEEPER_VERSION} binary (${ARCH})..." + wget -q "${RELEASE_URL}/${tarball}" -O ~/rpmbuild/SOURCES/"${tarball}" + + # LICENSE + NOTICE are not shipped inside the release tarball; fetch them from + # the repo at the matching tag for %license/%doc. + echo "Fetching LICENSE + NOTICE from the ${LAKEKEEPER_VERSION} tag..." + wget -q "${REPO_RAW}/LICENSE" -O ~/rpmbuild/SOURCES/LICENSE + wget -q "${REPO_RAW}/NOTICE" -O ~/rpmbuild/SOURCES/NOTICE + + # This function is for debugging purpose if you have your own keys. GH workflow does not need it. + #import_gpg_keys +} + +build() { + echo "Building RPM and SRPM..." + QA_RPATHS=$(( 0xffff )) rpmbuild -ba ~/rpmbuild/SPECS/lakekeeper.spec \ + --define "lakekeeper_version ${LAKEKEEPER_VERSION}" \ + --define "lakekeeper_buildnum ${LAKEKEEPER_BUILDNUM}" \ + --define "lk_arch ${ARCH}" +} + +post_build() { + echo "📤 Copying built RPMs to /output..." + mkdir -p /output + cp -v ~/rpmbuild/RPMS/*/*.rpm /output/ || echo "No binary RPMs found" + cp -v ~/rpmbuild/SRPMS/*.src.rpm /output/ || echo "No SRPM found" + + sign_rpms /output/*.rpm + validate_signatures /output/*.rpm +} diff --git a/packaging/lakekeeper/common.sh b/packaging/lakekeeper/common.sh new file mode 100755 index 0000000..ee7858e --- /dev/null +++ b/packaging/lakekeeper/common.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# common.sh - Lakekeeper (Apache Iceberg REST Catalog) package. +# +# PG-INDEPENDENT: packaged from the upstream PREBUILT release binary +# (github.com/lakekeeper/lakekeeper), NOT compiled here. One build per arch (no +# PG-major fan-out). The build scripts auto-detect the arch from the build +# container and download the matching release tarball. +# +# The upstream binaries are glibc-DYNAMIC and require GLIBC >= 2.34, so this +# package targets el9/el10 + jammy/noble/bookworm/trixie. Debian bullseye +# (glibc 2.31) CANNOT run these prebuilt binaries and is intentionally omitted. + +export LAKEKEEPER_VERSION="${COMPONENT_VERSION:-0.13.1}" +export LAKEKEEPER_BUILDNUM="${COMPONENT_BUILDNUM:-1}" + +# DEB only: move a pre-release pretag (e.g. BUILDNUM='rc1_1') into the upstream +# VERSION with a leading '~' (0.13.1~rc1, BUILDNUM=1) so '~' sorts pre-releases +# BELOW stable in dpkg/reprepro. Gated on apt-get so RPM keeps the pretag in +# Release (rpmvercmp already sorts rc1_1 below stable). +if command -v apt-get &>/dev/null; then + if [[ "$LAKEKEEPER_BUILDNUM" == *_* ]]; then + LAKEKEEPER_PRETAG="${LAKEKEEPER_BUILDNUM%%_*}" + export LAKEKEEPER_VERSION="${LAKEKEEPER_VERSION}~${LAKEKEEPER_PRETAG}" + LAKEKEEPER_BUILDNUM="${LAKEKEEPER_BUILDNUM#*_}" + fi +fi +export LAKEKEEPER_BUILDNUM + +export REPO_TYPE="${REPO_TYPE:-daily}" diff --git a/packaging/lakekeeper/common/lakekeeper.env b/packaging/lakekeeper/common/lakekeeper.env new file mode 100644 index 0000000..7cd9430 --- /dev/null +++ b/packaging/lakekeeper/common/lakekeeper.env @@ -0,0 +1,32 @@ +# Lakekeeper configuration — systemd EnvironmentFile for pgedge-lakekeeper. +# +# Lakekeeper is configured entirely through LAKEKEEPER__* environment variables. +# Uncomment and set the values below, run the one-time database migration, then +# start the service. See /usr/share/doc/pgedge-lakekeeper/README.md for details. +# +# ============================ REQUIRED ============================ +# The service will not start until these are set. +# +# PostgreSQL connection for the catalog backend (PostgreSQL 15+). The target +# database must have these extensions available: uuid-ossp, pgcrypto, pg_trgm, +# btree_gin, btree_gist (the one-time migration creates them). +#LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://lakekeeper:CHANGEME@127.0.0.1:5432/lakekeeper +# +# Key that encrypts secrets stored in the catalog. Generate a strong random +# value, e.g. openssl rand -base64 32 +# KEEP IT STABLE and BACK IT UP: it must be identical across all nodes sharing +# the same catalog database, and losing it makes stored secrets unrecoverable. +#LAKEKEEPER__PG_ENCRYPTION_KEY=CHANGEME +# +# ============================ OPTIONAL ============================ +# +# Read-replica connection (defaults to the write URL if unset). +#LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://lakekeeper:CHANGEME@127.0.0.1:5432/lakekeeper +# +# HTTP listen address / port (defaults: 0.0.0.0 and 8181). +#LAKEKEEPER__BIND_IP=0.0.0.0 +#LAKEKEEPER__LISTEN_PORT=8181 +# +# Authorization backend. If unset, the catalog is OPEN (no authorization). +# Configure authentication/authorization before exposing this in production. +#LAKEKEEPER__AUTHZ_BACKEND=allowall diff --git a/packaging/lakekeeper/common/lakekeeper.service b/packaging/lakekeeper/common/lakekeeper.service new file mode 100644 index 0000000..a97cb89 --- /dev/null +++ b/packaging/lakekeeper/common/lakekeeper.service @@ -0,0 +1,29 @@ +[Unit] +Description=Lakekeeper - Apache Iceberg REST Catalog +Documentation=https://docs.lakekeeper.io +After=network-online.target +Wants=network-online.target + +[Service] +Type=exec +User=lakekeeper +Group=lakekeeper +EnvironmentFile=/etc/lakekeeper/lakekeeper.env +# Lakekeeper stores all state in PostgreSQL; the database must be migrated once +# before the first start (`lakekeeper migrate`) — see the README. To run the +# (idempotent) migration automatically on every start instead, uncomment: +#ExecStartPre=/usr/bin/lakekeeper migrate +ExecStart=/usr/bin/lakekeeper serve +Restart=on-failure +RestartSec=5s +LimitNOFILE=65536 + +# Hardening — lakekeeper only reads its env file and talks to PostgreSQL over +# the network; it writes nothing to the local filesystem. +NoNewPrivileges=true +ProtectSystem=full +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/packaging/lakekeeper/deb/debian/control b/packaging/lakekeeper/deb/debian/control new file mode 100644 index 0000000..0dfa886 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/control @@ -0,0 +1,29 @@ +Source: pgedge-lakekeeper +Section: database +Priority: optional +Maintainer: pgEdge Build Team +Build-Depends: + debhelper-compat (= 13), + systemd, +Standards-Version: 4.7.0 +Rules-Requires-Root: no +Homepage: https://lakekeeper.io + +Package: pgedge-lakekeeper +Architecture: any +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + systemd, + adduser, +Conflicts: lakekeeper +Provides: lakekeeper +Description: Lakekeeper - Apache Iceberg REST Catalog + Lakekeeper is an implementation of the Apache Iceberg REST Catalog. It stores + catalog state in PostgreSQL and serves the Iceberg REST API to query engines + and data tools. + . + This package installs the prebuilt lakekeeper binary and a systemd service. + Lakekeeper requires an external PostgreSQL 15+ database and is not enabled or + started automatically: configure /etc/lakekeeper/lakekeeper.env and run the + one-time database migration first. diff --git a/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.conffiles b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.conffiles new file mode 100644 index 0000000..6db21e1 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.conffiles @@ -0,0 +1 @@ +/etc/lakekeeper/lakekeeper.env diff --git a/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.docs b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.docs new file mode 100644 index 0000000..902232f --- /dev/null +++ b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.docs @@ -0,0 +1,3 @@ +README.md +NOTICE +LICENSE diff --git a/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.install b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.install new file mode 100644 index 0000000..ffe8ec0 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/pgedge-lakekeeper.install @@ -0,0 +1,2 @@ +lakekeeper usr/bin/ +debian/tmp/sbom/* usr/share/pgedge-lakekeeper/ diff --git a/packaging/lakekeeper/deb/debian/postinst b/packaging/lakekeeper/deb/debian/postinst new file mode 100755 index 0000000..119ca93 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/postinst @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +# Create the lakekeeper group and system user. +if ! getent group lakekeeper >/dev/null; then + addgroup --system lakekeeper +fi + +if ! getent passwd lakekeeper >/dev/null; then + adduser --system --ingroup lakekeeper --home /var/lib/lakekeeper \ + --no-create-home --shell /usr/sbin/nologin \ + --disabled-password --gecos "Lakekeeper Iceberg REST Catalog" lakekeeper +fi + +# The env file carries secrets (DB URL, encryption key) — restrict it to +# root:lakekeeper now that the group exists. +if [ -f /etc/lakekeeper/lakekeeper.env ]; then + chgrp lakekeeper /etc/lakekeeper/lakekeeper.env 2>/dev/null || true + chmod 0640 /etc/lakekeeper/lakekeeper.env || true +fi + +#DEBHELPER# +exit 0 diff --git a/packaging/lakekeeper/deb/debian/rules b/packaging/lakekeeper/deb/debian/rules new file mode 100644 index 0000000..d1f4b43 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/rules @@ -0,0 +1,38 @@ +#!/usr/bin/make -f + +export DH_VERBOSE = 1 + +%: + dh $@ + +# Prebuilt binary — nothing to compile or test. +override_dh_auto_build: + +override_dh_auto_test: + +override_dh_builddeb: + dh_builddeb -- -Zgzip + +# ~155MB prebuilt Rust binary: strip in place with no -dbgsym package, and skip +# dwz (its DIE handling chokes on a binary this size). +override_dh_dwz: + +override_dh_strip: + dh_strip --no-automatic-dbgsym + +override_dh_installsystemd: + dh_installsystemd --no-enable --no-start --name=lakekeeper + +execute_before_dh_install: + # --- SBOM generation and signing (per-format DEB key) --- + mkdir -p debian/tmp/sbom + syft dir:$(CURDIR) -o cyclonedx-json > debian/tmp/sbom/lakekeeper-sbom.json || exit 1 + KEY_ID=$$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $$5}' | head -n 1); \ + gpg --armor --detach-sign --local-user "$$KEY_ID" \ + --output debian/tmp/sbom/lakekeeper-sbom.json.asc \ + debian/tmp/sbom/lakekeeper-sbom.json || exit 1 + +execute_after_dh_install: + # The env file carries secrets (DB URL, encryption key); install it 0640 and + # let postinst set the group to lakekeeper. + install -D -m 0640 debian/lakekeeper.env debian/pgedge-lakekeeper/etc/lakekeeper/lakekeeper.env diff --git a/packaging/lakekeeper/deb/debian/source/format b/packaging/lakekeeper/deb/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/packaging/lakekeeper/deb/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/lakekeeper/rpm/lakekeeper.spec b/packaging/lakekeeper/rpm/lakekeeper.spec new file mode 100644 index 0000000..b9b6fa6 --- /dev/null +++ b/packaging/lakekeeper/rpm/lakekeeper.spec @@ -0,0 +1,98 @@ +%global sname lakekeeper + +# Prebuilt binary — no debug info / source to extract, so disable the (empty) +# debuginfo/debugsource subpackages. The binary is stripped in %%install. +%global debug_package %{nil} + +Name: pgedge-%{sname} +Version: %{lakekeeper_version} +Release: %{lakekeeper_buildnum}%{?dist} +Summary: Lakekeeper - Apache Iceberg REST Catalog + +License: Apache-2.0 +URL: https://lakekeeper.io + +# Upstream prebuilt release binary (glibc/gnu target), staged into SOURCES by +# build-rpm.sh. Arch-specific; nothing is compiled here. +Source0: lakekeeper-%{lk_arch}-unknown-linux-gnu.tar.gz +Source1: lakekeeper.service +Source2: lakekeeper.env +Source3: LICENSE +Source4: NOTICE +Source5: README.md + +BuildArch: %{_arch} + +BuildRequires: systemd-rpm-macros +Requires(pre): shadow-utils +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +Requires: systemd + +# Own the upstream name so this can't be co-installed with a community build. +Conflicts: lakekeeper +Provides: lakekeeper + +%description +Lakekeeper is an implementation of the Apache Iceberg REST Catalog. It stores +catalog state in PostgreSQL and serves the Iceberg REST API to query engines +and data tools. + +This package installs the prebuilt lakekeeper binary and a systemd service. +Lakekeeper requires an external PostgreSQL 15+ database and is NOT enabled or +started automatically: configure /etc/lakekeeper/lakekeeper.env and run the +one-time database migration first (see the bundled README). + +%prep +# The release tarball contains only the `lakekeeper` binary (no wrapping +# directory); -c creates one and extracts into it. +%setup -q -c -n %{sname}-%{version} +cp -p %{SOURCE3} %{SOURCE4} %{SOURCE5} . + +%build +syft dir:%{_builddir}/%{sname}-%{version} -o cyclonedx-json > %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '/^sec/{print $5}' | head -n 1); export KEY_ID +gpg --armor --detach-sign --local-user "$KEY_ID" --output %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json.asc %{_builddir}/%{sname}-%{version}/%{sname}-sbom.json || exit 1 + +%install +install -D -m 0755 lakekeeper %{buildroot}%{_bindir}/lakekeeper +# Prebuilt binary ships unstripped (~155 MB) — strip to shrink the package. +strip %{buildroot}%{_bindir}/lakekeeper || : +install -D -m 0644 %{SOURCE1} %{buildroot}%{_unitdir}/lakekeeper.service +install -D -m 0640 %{SOURCE2} %{buildroot}%{_sysconfdir}/lakekeeper/lakekeeper.env +install -D -m 0644 %{sname}-sbom.json %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +install -D -m 0644 %{sname}-sbom.json.asc %{buildroot}%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%pre +# Create the lakekeeper group and system user. +getent group lakekeeper >/dev/null || groupadd -r lakekeeper +getent passwd lakekeeper >/dev/null || \ + useradd -r -g lakekeeper -d /var/lib/lakekeeper -s /sbin/nologin \ + -c "Lakekeeper Iceberg REST Catalog" lakekeeper +exit 0 + +%post +%systemd_post lakekeeper.service + +%preun +%systemd_preun lakekeeper.service + +%postun +%systemd_postun_with_restart lakekeeper.service + +%files +%license LICENSE +%doc NOTICE README.md +%{_bindir}/lakekeeper +%{_unitdir}/lakekeeper.service +%dir %{_sysconfdir}/lakekeeper +%attr(0640, root, lakekeeper) %config(noreplace) %{_sysconfdir}/lakekeeper/lakekeeper.env +%dir %{_datadir}/pgedge-%{sname} +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json +%{_datadir}/pgedge-%{sname}/%{sname}-sbom.json.asc + +%changelog +* Mon Jul 06 2026 pgEdge Build Team - 0.13.1-1 +- Initial package of Lakekeeper (Apache Iceberg REST Catalog) from the upstream prebuilt binary From 8f70d8878138f52057ff3a3d55c694fa9dc9af09 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Mon, 6 Jul 2026 17:03:30 +0500 Subject: [PATCH 10/12] Add Ubuntu resolute target and centralize coldfront build distros into env.RPM_IMAGES/DEB_IMAGES --- .github/workflows/release.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 307dd11..3e8c722 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,11 @@ env: # Umbrella name for the backup/manifest grouping; per-component identity # lives in the manifest cells[]. COMPONENT_NAME: ${{ github.event.repository.name }} + # Single source of truth for the build/target distros — consumed by every + # detect-matrix step (rpm_images / deb_images inputs) AND the duckdb-ext + # load-test loop. Add/remove a distro in ONE place. + RPM_IMAGES: '["almalinux:9","almalinux:10"]' + DEB_IMAGES: '["ubuntu:jammy","ubuntu:noble","ubuntu:resolute","debian:bookworm","debian:trixie"]' jobs: # ========================================================================= @@ -200,8 +205,8 @@ jobs: component_name: packaging/pg_duckdb pg_versions: '16,17,18' archs: '["amd64","arm64"]' - rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + rpm_images: ${{ env.RPM_IMAGES }} + deb_images: ${{ env.DEB_IMAGES }} - name: Detect — coldfront extension (PG 16/17/18) id: coldfront uses: ./.github/actions/pgedge-detect-build-matrix @@ -209,32 +214,32 @@ jobs: component_name: packaging/coldfront pg_versions: '16,17,18' archs: '["amd64","arm64"]' - rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + rpm_images: ${{ env.RPM_IMAGES }} + deb_images: ${{ env.DEB_IMAGES }} - name: Detect — coldfront binaries (PG-independent) id: bin uses: ./.github/actions/pgedge-detect-build-matrix with: component_name: packaging/coldfront-bin archs: '["amd64","arm64"]' - rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + rpm_images: ${{ env.RPM_IMAGES }} + deb_images: ${{ env.DEB_IMAGES }} - name: Detect — coldfront duckdb-extensions (PG-independent) id: duckdbext uses: ./.github/actions/pgedge-detect-build-matrix with: component_name: packaging/coldfront-duckdb-extensions archs: '["amd64","arm64"]' - rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + rpm_images: ${{ env.RPM_IMAGES }} + deb_images: ${{ env.DEB_IMAGES }} - name: Detect — lakekeeper (PG-independent; prebuilt binary, needs glibc >= 2.34) id: lakekeeper uses: ./.github/actions/pgedge-detect-build-matrix with: component_name: packaging/lakekeeper archs: '["amd64","arm64"]' - rpm_images: '["almalinux:9","almalinux:10"]' - deb_images: '["ubuntu:jammy","ubuntu:noble","debian:bookworm","debian:trixie"]' + rpm_images: ${{ env.RPM_IMAGES }} + deb_images: ${{ env.DEB_IMAGES }} - name: Merge component matrices id: merge env: @@ -378,7 +383,7 @@ jobs: CLI_URL="https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-${CLI_ARCH}.zip" echo "Engine CLI: $CLI_URL" pass=0; fail=0 - for img in almalinux:9 almalinux:10 ubuntu:jammy ubuntu:noble debian:bookworm debian:trixie; do + for img in $(jq -r '.[]' <<<"$RPM_IMAGES") $(jq -r '.[]' <<<"$DEB_IMAGES"); do echo "###### $img ######" cid=$(docker run -d -v "$PWD/ext:/ext:ro" "$img" sleep 600) || { echo " run failed"; fail=$((fail+1)); continue; } docker exec "$cid" bash -c ' From f50a6e49e085134be02a626b61639237ecbf02fb Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Tue, 7 Jul 2026 17:40:02 +0500 Subject: [PATCH 11/12] Fix lakekeeper DEB build: strip ~pretag from version for the release download URL --- packaging/lakekeeper/build-deb.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packaging/lakekeeper/build-deb.sh b/packaging/lakekeeper/build-deb.sh index 56b75b4..e51ca35 100755 --- a/packaging/lakekeeper/build-deb.sh +++ b/packaging/lakekeeper/build-deb.sh @@ -15,8 +15,11 @@ case "$DEB_ARCH" in *) echo "unsupported arch: $DEB_ARCH" >&2; exit 1 ;; esac -RELEASE_URL="https://github.com/lakekeeper/lakekeeper/releases/download/v${LAKEKEEPER_VERSION}" -REPO_RAW="https://raw.githubusercontent.com/lakekeeper/lakekeeper/v${LAKEKEEPER_VERSION}" +# Strip any '~pretag' for the release tag/URL: LAKEKEEPER_VERSION carries it for +# the deb package version (e.g. 0.13.1~test1), but the GitHub release tag is the +# clean upstream version (v0.13.1). +RELEASE_URL="https://github.com/lakekeeper/lakekeeper/releases/download/v${LAKEKEEPER_VERSION%%~*}" +REPO_RAW="https://raw.githubusercontent.com/lakekeeper/lakekeeper/v${LAKEKEEPER_VERSION%%~*}" prepare() { setup_apt_build_env @@ -24,7 +27,7 @@ prepare() { # This function is for debugging purpose if you have your own keys. GH workflow does not need it. #import_gpg_keys - echo "Downloading lakekeeper ${LAKEKEEPER_VERSION} binary (${UNAME_ARCH})..." + echo "Downloading lakekeeper ${LAKEKEEPER_VERSION%%~*} binary (${UNAME_ARCH})..." rm -rf "$BUILD_DIR" mkdir -p "$SRC_DIR" local tarball="lakekeeper-${UNAME_ARCH}-unknown-linux-gnu.tar.gz" From 590a90f26e4505790877b77320f6902987cb5cdd Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Wed, 8 Jul 2026 21:05:44 +0500 Subject: [PATCH 12/12] Harden syft install: download and verify release tarball instead of curl | sudo sh --- common/common-functions.sh | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/common/common-functions.sh b/common/common-functions.sh index 171b397..83414fb 100755 --- a/common/common-functions.sh +++ b/common/common-functions.sh @@ -2,12 +2,27 @@ install_syft(){ - # Pin the installer to a tagged ref (not mutable main) AND pin the installed - # version, so release builds are reproducible and not exposed to upstream - # changes on syft's main branch. Override SYFT_VERSION to bump. + # Install syft from its pinned release tarball and verify it against the + # published checksums before installing — avoids piping a remote installer + # script straight into a root shell. Override SYFT_VERSION to bump. SYFT_VERSION="${SYFT_VERSION:-v1.45.1}" - echo "Installing syft ${SYFT_VERSION}..." - curl -sSfL "https://raw.githubusercontent.com/anchore/syft/${SYFT_VERSION}/install.sh" | sudo sh -s -- -b /usr/local/bin "${SYFT_VERSION}" + local ver="${SYFT_VERSION#v}" arch + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64) arch=arm64 ;; + *) echo "unsupported arch for syft: $(uname -m)" >&2; return 1 ;; + esac + local tgz="syft_${ver}_linux_${arch}.tar.gz" + local base="https://github.com/anchore/syft/releases/download/v${ver}" + local tmp; tmp="$(mktemp -d)" + echo "Installing syft ${SYFT_VERSION} (${arch})..." + curl -sSfL "${base}/${tgz}" -o "${tmp}/${tgz}" + curl -sSfL "${base}/syft_${ver}_checksums.txt" -o "${tmp}/checksums.txt" + ( cd "${tmp}" && grep " ${tgz}\$" checksums.txt | sha256sum -c - ) + tar -xzf "${tmp}/${tgz}" -C "${tmp}" syft + sudo install -m 0755 "${tmp}/syft" /usr/local/bin/syft + rm -rf "${tmp}" + syft version } setup_dnf_build_env(){