From 2462e2729f7c0630fc3c9f580de85987806048e1 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 13 Oct 2026 13:15:46 +0000 Subject: [PATCH 1/4] Add riscv64 wheel build for aimrocks 0.5.2 RocksDB 6.29.fb and its zlib/bzip2/lz4/zstd/snappy dependencies are built from source with PORTABLE=1, mirroring upstream's own docker/build-*.sh. PORTABLE=1 disables every x86-only compiler probe in rocksdb's build_detect_platform, and util/crc32c.cc's hardware CRC32 paths are all gated on HAVE_SSE42/HAVE_ARM64_CRC/HAVE_POWER8, none of which get defined on riscv64, so it falls back to the portable table-based implementation upstream already ships non-x86 (aarch64) wheels with. --- .github/workflows/build-aimrocks.yml | 167 +++++++++++++++++++++++++++ docs/packages/aimrocks.yaml | 5 + 2 files changed, 172 insertions(+) create mode 100644 .github/workflows/build-aimrocks.yml create mode 100644 docs/packages/aimrocks.yaml diff --git a/.github/workflows/build-aimrocks.yml b/.github/workflows/build-aimrocks.yml new file mode 100644 index 000000000000..8b132b94bcbb --- /dev/null +++ b/.github/workflows/build-aimrocks.yml @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `deps`/`rocksdb`/`wheels` targets of +# https://github.com/aimhubio/aimrocks/blob/v0.5.2/Dockerfile and the +# docker/build-*.sh scripts it runs. +name: Build aimrocks wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/aimrocks.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-aimrocks.yml' + - 'docs/packages/aimrocks.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-aimrocks.yml' + - 'docs/packages/aimrocks.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + # Versions upstream's own docker/build-*.sh pin. + ZLIB_VERSION: '1.2.11' + BZIP2_VERSION: '1.0.8' + LZ4_VERSION: '1.9.3' + ZSTD_VERSION: '1.1.3' + SNAPPY_VERSION: '1.1.8' + ROCKSDB_VERSION: '6.29.fb' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: aimrocks + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build aimrocks ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: + - "cp312" + - "cp313" + - "cp314" + - "cp314t" + + env: + AIMROCKS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout aimrocks ${{ env.AIMROCKS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: aimhubio/aimrocks + ref: v${{ env.AIMROCKS_VERSION }} + persist-credentials: false + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT_PASS_LINUX: >- + ZLIB_VERSION BZIP2_VERSION LZ4_VERSION ZSTD_VERSION SNAPPY_VERSION + ROCKSDB_VERSION + # Reproduces upstream's docker/build-{zlib,bzip2,lz4,zstd,snappy,rocksdb}.sh: + # each dep installs into /opt/deps with its default (mostly-static) `make + # install`/cmake, then the `rm -f *.so*` before the rocksdb build (matching + # upstream's own `rm -rf lib/lib*.so*` in build-rocksdb.sh) forces rocksdb to + # link every codec in statically, so the only shared object produced is + # librocksdb.so itself. PORTABLE=1 (upstream's own flag, used unconditionally + # there) disables -march=native and every x86 SSE4.2/PCLMUL/AVX2 probe in + # rocksdb's build_detect_platform; util/crc32c.cc's hardware paths are gated on + # HAVE_SSE42/HAVE_ARM64_CRC/HAVE_POWER8, none of which get defined here, so it + # falls back to the portable table-based CRC32C upstream already ships + # non-x86 wheels (aarch64) with. + CIBW_BEFORE_ALL_LINUX: >- + mkdir -p /opt/deps/lib /opt/deps/include && + curl -sSLf "https://www.zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/zlib-${ZLIB_VERSION} && ./configure && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make install prefix=/opt/deps) && + curl -sSLf "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/bzip2-${BZIP2_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make install PREFIX=/opt/deps) && + curl -sSLf "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/lz4-${LZ4_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make PREFIX=/opt/deps install) && + curl -sSLf "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" | tar -xz -C /tmp && + (cd /tmp/zstd-${ZSTD_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make PREFIX=/opt/deps install) && + curl -sSLf "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" | tar -xz -C /tmp && + cmake -S /tmp/snappy-${SNAPPY_VERSION} -B /tmp/snappy-${SNAPPY_VERSION}/build + -DCMAKE_INSTALL_PREFIX=/opt/deps -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 && + cmake --build /tmp/snappy-${SNAPPY_VERSION}/build --parallel --target install && + rm -f /opt/deps/lib/lib*.so* && + curl -sSLf "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" | tar -xz -C /tmp && + LIBRARY_PATH=/opt/deps/lib PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} + shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I/opt/deps/include" + EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 -j4 && + strip --strip-debug /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so && + cp /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so /opt/deps/lib/ && + PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} PREFIX=/opt/deps DEBUG_LEVEL=0 install-headers && + cp /tmp/zlib-${ZLIB_VERSION}/README {project}/LICENSE.zlib && + cp /tmp/bzip2-${BZIP2_VERSION}/LICENSE {project}/LICENSE.bzip2 && + cp /tmp/lz4-${LZ4_VERSION}/lib/LICENSE {project}/LICENSE.lz4 && + cp /tmp/zstd-${ZSTD_VERSION}/LICENSE {project}/LICENSE.zstd && + cp /tmp/snappy-${SNAPPY_VERSION}/COPYING {project}/LICENSE.snappy && + cp /tmp/rocksdb-${ROCKSDB_VERSION}/LICENSE.Apache {project}/LICENSE.rocksdb + CIBW_ENVIRONMENT: AIM_DEP_DIR=/opt/deps + CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair --strip -w {dest_dir} {wheel} + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_SOURCES: tests + CIBW_TEST_COMMAND: pytest tests + + - name: Check the extension and licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("aimrocks/lib_rocksdb") and ".so" in n for n in names), whl + assert any(n.endswith("licenses/LICENSE") for n in names), whl + for dep in ("zlib", "bzip2", "lz4", "zstd", "snappy", "rocksdb"): + assert any(n.endswith(f"licenses/LICENSE.{dep}") for n in names), (whl, dep) + print(whl, "ok") + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: aimrocks-${{ env.AIMROCKS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish aimrocks ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: aimrocks-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/aimrocks.yaml b/docs/packages/aimrocks.yaml new file mode 100644 index 000000000000..ba449ab81244 --- /dev/null +++ b/docs/packages/aimrocks.yaml @@ -0,0 +1,5 @@ +package-name: aimrocks +source-code: https://github.com/aimhubio/aimrocks +license: Apache License 2.0 +versions: +- version: 0.5.2 From aadc6e773d86b2d8f4cdd788c7fa9ba5166afd9e Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 16 Oct 2026 01:05:03 +0000 Subject: [PATCH 2/4] aimrocks: fix riscv64 build, disable rocksdb -Werror for GCC 14 All four manylinux_riscv64 matrix legs failed identically at the same step: rocksdb 6.29.fb's db/db_impl/db_impl.cc fails to compile under manylinux_2_39_riscv64's GCC 14.3.1 with `error: redundant move in return statement [-Werror=redundant-move]` on `return std::move(io_s);`. The code predates this diagnostic; upstream's own (older) build image never hits it. rocksdb's Makefile already gates -Werror behind `ifndef DISABLE_WARNING_AS_ERROR`, so pass DISABLE_WARNING_AS_ERROR=1 to the shared_lib build instead of patching or disabling warnings project-wide. --- .github/workflows/build-aimrocks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-aimrocks.yml b/.github/workflows/build-aimrocks.yml index 8b132b94bcbb..51b6c676e860 100644 --- a/.github/workflows/build-aimrocks.yml +++ b/.github/workflows/build-aimrocks.yml @@ -94,7 +94,11 @@ jobs: # rocksdb's build_detect_platform; util/crc32c.cc's hardware paths are gated on # HAVE_SSE42/HAVE_ARM64_CRC/HAVE_POWER8, none of which get defined here, so it # falls back to the portable table-based CRC32C upstream already ships - # non-x86 wheels (aarch64) with. + # non-x86 wheels (aarch64) with. DISABLE_WARNING_AS_ERROR=1 is rocksdb's own + # Makefile knob (gated by `ifndef DISABLE_WARNING_AS_ERROR`) for a compiler + # newer than 6.29.fb's code anticipated: manylinux_2_39_riscv64's GCC 14.3.1 + # flags `return std::move(io_s);` in db/db_impl/db_impl.cc as + # -Werror=redundant-move, which upstream's own (older) build image never hit. CIBW_BEFORE_ALL_LINUX: >- mkdir -p /opt/deps/lib /opt/deps/include && curl -sSLf "https://www.zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" | tar -xz -C /tmp && @@ -114,7 +118,7 @@ jobs: curl -sSLf "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" | tar -xz -C /tmp && LIBRARY_PATH=/opt/deps/lib PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I/opt/deps/include" - EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 -j4 && + EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 DISABLE_WARNING_AS_ERROR=1 -j4 && strip --strip-debug /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so && cp /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so /opt/deps/lib/ && PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} PREFIX=/opt/deps DEBUG_LEVEL=0 install-headers && From 558fadae8436dbcf2a967624f0907980d2f7937e Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 23 Oct 2026 18:05:55 +0000 Subject: [PATCH 3/4] aimrocks: fix rocksdb 6.29.fb build with GCC 14 on riscv64 Force-include for rocksdb: GCC 14's libstdc++ no longer pulls it in transitively and table/block_based/data_block_hash_index.h needs uint8_t/uint16_t/uint32_t. Pass PPC_LIBC_IS_GNU=0 so rocksdb's Makefile leaves out the optional range-lock manager, whose toku_time.h has no riscv64 timer and stops with '#error No timer implementation for this platform'. aimrocks does not use TransactionDB or NewRangeLockManager. --- .github/workflows/build-aimrocks.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-aimrocks.yml b/.github/workflows/build-aimrocks.yml index 51b6c676e860..f3a7173b2cd1 100644 --- a/.github/workflows/build-aimrocks.yml +++ b/.github/workflows/build-aimrocks.yml @@ -99,6 +99,13 @@ jobs: # newer than 6.29.fb's code anticipated: manylinux_2_39_riscv64's GCC 14.3.1 # flags `return std::move(io_s);` in db/db_impl/db_impl.cc as # -Werror=redundant-move, which upstream's own (older) build image never hit. + # `-include cstdint` is for the same GCC 14: libstdc++ (since 13) no longer pulls + # in transitively, which table/block_based/data_block_hash_index.h + # relies on for uint8_t/uint16_t/uint32_t. + # PPC_LIBC_IS_GNU=0 is rocksdb's own Makefile switch for leaving out the optional + # range-lock manager (RANGE_TREE_SOURCES): its toku_time.h has no riscv64 timer and + # hits `#error No timer implementation for this platform`. aimrocks never binds + # TransactionDB or NewRangeLockManager. CIBW_BEFORE_ALL_LINUX: >- mkdir -p /opt/deps/lib /opt/deps/include && curl -sSLf "https://www.zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" | tar -xz -C /tmp && @@ -117,8 +124,8 @@ jobs: rm -f /opt/deps/lib/lib*.so* && curl -sSLf "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" | tar -xz -C /tmp && LIBRARY_PATH=/opt/deps/lib PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} - shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I/opt/deps/include" - EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 DISABLE_WARNING_AS_ERROR=1 -j4 && + shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I/opt/deps/include -include cstdint" + EXTRA_CFLAGS=-fPIC USE_RTTI=0 DEBUG_LEVEL=0 DISABLE_WARNING_AS_ERROR=1 PPC_LIBC_IS_GNU=0 -j4 && strip --strip-debug /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so && cp /tmp/rocksdb-${ROCKSDB_VERSION}/librocksdb.so /opt/deps/lib/ && PORTABLE=1 make -C /tmp/rocksdb-${ROCKSDB_VERSION} PREFIX=/opt/deps DEBUG_LEVEL=0 install-headers && From 9eb640fda0b940ce4b0beee879d0620a6bbfde02 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 17:52:11 +0000 Subject: [PATCH 4/4] aimrocks: fix snappy linking on riscv64 (GNUInstallDirs lib64 default) cmake's GNUInstallDirs module defaults the install libdir to lib64 for a 64-bit build on this manylinux_2_39_riscv64 (RHEL-family) image, so snappy's cmake install was putting libsnappy.a in /opt/deps/lib64 while zlib/bzip2/lz4/zstd's plain Makefiles install into /opt/deps/lib. rocksdb's shared_lib link step only sets LIBRARY_PATH=/opt/deps/lib, so the linker couldn't find it: /usr/bin/ld: cannot find -lsnappy: No such file or directory identical across all four matrix legs (cp312/cp313/cp314/cp314t), since none of them reach the link step with a different environment. Pin snappy's cmake configure to -DCMAKE_INSTALL_LIBDIR=lib so it installs next to the other codecs. --- .github/workflows/build-aimrocks.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-aimrocks.yml b/.github/workflows/build-aimrocks.yml index f3a7173b2cd1..a9812c661c26 100644 --- a/.github/workflows/build-aimrocks.yml +++ b/.github/workflows/build-aimrocks.yml @@ -106,6 +106,12 @@ jobs: # range-lock manager (RANGE_TREE_SOURCES): its toku_time.h has no riscv64 timer and # hits `#error No timer implementation for this platform`. aimrocks never binds # TransactionDB or NewRangeLockManager. + # snappy's -DCMAKE_INSTALL_LIBDIR=lib pins its GNUInstallDirs libdir: on this + # manylinux_2_39_riscv64 (RHEL-family) image, GNUInstallDirs otherwise defaults a + # 64-bit install to lib64, so `cmake --install` puts libsnappy.a in + # /opt/deps/lib64 while zlib/bzip2/lz4/zstd's plain Makefiles install into + # /opt/deps/lib. rocksdb's link step only sets LIBRARY_PATH=/opt/deps/lib, so the + # linker can't find it: `/usr/bin/ld: cannot find -lsnappy`. CIBW_BEFORE_ALL_LINUX: >- mkdir -p /opt/deps/lib /opt/deps/include && curl -sSLf "https://www.zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" | tar -xz -C /tmp && @@ -118,7 +124,8 @@ jobs: (cd /tmp/zstd-${ZSTD_VERSION} && make CFLAGS=-fPIC CXXFLAGS=-fPIC && make PREFIX=/opt/deps install) && curl -sSLf "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" | tar -xz -C /tmp && cmake -S /tmp/snappy-${SNAPPY_VERSION} -B /tmp/snappy-${SNAPPY_VERSION}/build - -DCMAKE_INSTALL_PREFIX=/opt/deps -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_INSTALL_PREFIX=/opt/deps -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 && cmake --build /tmp/snappy-${SNAPPY_VERSION}/build --parallel --target install && rm -f /opt/deps/lib/lib*.so* &&