ladybug: Add version 0.19.1

.github/workflows/build-ladybug.yml

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: 2026 The RISE Project
+# SPDX-License-Identifier: MIT
+---
+# This workflow is based on: https://github.com/LadybugDB/ladybug/blob/v0.19.1/.github/workflows/python-wheel-workflow.yml
+# and the Linux compat leg of https://github.com/LadybugDB/ladybug/blob/v0.19.1/.github/workflows/precompiled-bin-workflow.yml
+name: Build ladybug wheels (riscv64)
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version glob to (re)build; empty builds every version of docs/packages/ladybug.yaml not released yet'
+        required: false
+        default: ''
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-ladybug.yml'
+      - 'docs/packages/ladybug.yaml'
+      - 'patches/ladybug/**'
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-ladybug.yml'
+      - 'docs/packages/ladybug.yaml'
+      - 'patches/ladybug/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read  # to fetch code (actions/checkout)
+
+env:
+  MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
+
+jobs:
+  setup:
+    uses: $/.github/workflows/_setup.yml
+    with:
+      package: ladybug
+      version: ${{ inputs.version }}
+
+  package_python_sdist:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    name: Package ladybug ${{ matrix.version }} sdist
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+
+    env:
+      LADYBUG_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout ladybug v${{ env.LADYBUG_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: LadybugDB/ladybug
+          ref: v${{ env.LADYBUG_VERSION }}
+          persist-credentials: false
+
+      - name: Update submodules
+        run: git submodule update --init --recursive tools/python_api
+
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
+        with:
+          python-version: '3.12'
+          activate-environment: true
+          enable-cache: false
+
+      - name: Package Python sdist
+        working-directory: scripts/pip-package
+        run: |
+          uv pip install setuptools
+          python package_tar.py
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: ladybug-${{ env.LADYBUG_VERSION }}-sdist
+          path: scripts/pip-package/*.tar.gz
+          if-no-files-found: error
+
+  build_precompiled_bin:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    name: Build liblbug ${{ matrix.version }} manylinux_riscv64
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 1440
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+
+    env:
+      LADYBUG_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout ladybug v${{ env.LADYBUG_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: LadybugDB/ladybug
+          ref: v${{ env.LADYBUG_VERSION }}
+          persist-credentials: false
+
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          path: python-wheels
+          persist-credentials: false
+
+      - name: Apply patches
+        run: git apply --exclude='tools/python_api/*' python-wheels/patches/ladybug/${{ env.LADYBUG_VERSION }}/*.patch
+
+      # Rocky 10's openssl-devel is OpenSSL 3 and its default GCC 14 replaces gcc-toolset-13.
+      - name: Build
+        run: |
+          docker run --rm -v "$(pwd):/work" -w /work "${MANYLINUX_RISCV64_IMAGE}" bash -c '
+            set -euxo pipefail
+            dnf install -y ninja-build openssl-devel
+            make GEN=Ninja
+            make install
+            mkdir precompiled-liblbug
+            cp install/include/lbug.h install/include/lbug.hpp precompiled-liblbug/
+            cp -L install/lib*/liblbug.a precompiled-liblbug/
+          '
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: ladybug-${{ env.LADYBUG_VERSION }}-liblbug-static
+          path: precompiled-liblbug/
+          if-no-files-found: error
+
+  build_wheels:
+    needs: [setup, package_python_sdist, build_precompiled_bin]
+    if: needs.setup.outputs.versions != '[]'
+    name: Build ladybug ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 720
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+        python: ["cp312", "cp313", "cp314", "cp314t"]
+
+    env:
+      LADYBUG_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout ladybug v${{ env.LADYBUG_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: LadybugDB/ladybug
+          ref: v${{ env.LADYBUG_VERSION }}
+          persist-credentials: false
+
+      - name: Update submodules
+        run: git submodule update --init --depth 1 dataset tools/python_api
+
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          path: python-wheels
+          persist-credentials: false
+
+      - name: Apply test patches
+        run: git apply --include='tools/python_api/test/*' python-wheels/patches/ladybug/${{ env.LADYBUG_VERSION }}/*.patch
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: ladybug-${{ env.LADYBUG_VERSION }}-sdist
+          path: scripts/pip-package/
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: ladybug-${{ env.LADYBUG_VERSION }}-liblbug-static
+          path: precompiled-liblbug/
+
+      - name: Prepare wheel source tree
+        run: |
+          mkdir -p scripts/pip-package/cibw-source
+          tar -xzf scripts/pip-package/ladybug-${{ env.LADYBUG_VERSION }}.tar.gz -C scripts/pip-package/cibw-source
+
+      # The wheel statically links the vendored third_party libraries but upstream ships only ladybug's own LICENSE.
+      - name: Add the vendored libraries' licences
+        working-directory: scripts/pip-package/cibw-source/sdist
+        run: |
+          for f in ladybug-source/third_party/*/LICENSE*; do
+            cp "$f" "LICENSE.$(basename "$(dirname "$f")")"
+          done
+          ls LICENSE*
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074  # v4.2.0
+        with:
+          package-dir: scripts/pip-package/cibw-source/sdist
+          output-dir: wheelhouse/
+          only: ${{ matrix.python }}-manylinux_riscv64
+        env:
+          CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
+          CIBW_BEFORE_ALL_LINUX: dnf install -y openssl-devel pkg-config
+          CIBW_ENVIRONMENT_LINUX: >-
+            LBUG_API_PRECOMPILED_LIB_PATH=/project/precompiled-liblbug/liblbug.a
+            PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libssl.so.3 --exclude libcrypto.so.3 -w {dest_dir} {wheel}
+          # Not PIP_ONLY_BINARY: torch-geometric is pure Python.
+          CIBW_TEST_ENVIRONMENT: LBUG_PYTHON_BACKEND=pybind PIP_PREFER_BINARY=1
+          CIBW_TEST_SOURCES: dataset tools/python_api/test
+          # Upstream's [dev] test deps; pyarrow 25 is the only riscv64 build, and cp314t has no
+          # pandas/polars, so it takes the pytz that test_datatype.py imports and pandas brings in.
+          CIBW_TEST_REQUIRES: >-
+            pytest pytest-asyncio~=1.0 networkx~=3.0 numpy~=2.0 pyarrow torch torch-geometric>=2.5.0
+            ${{ matrix.python != 'cp314t' && 'pandas~=2.2 polars~=1.30' || 'pytz' }}
+          # test_fsm.py fails the same way with upstream's own x86_64 0.19.1 wheel. The deselected
+          # test_json.py tests INSTALL the json extension from upstream's server, which has no riscv64 build.
+          CIBW_TEST_COMMAND: >-
+            python -m pytest -vv tools/python_api/test
+            --ignore=tools/python_api/test/test_fsm.py
+            --deselect=tools/python_api/test/test_json.py::test_to_json_string_param_roundtrip
+            --deselect=tools/python_api/test/test_json.py::test_to_json_python_param_with_mixed_nested_list
+            --deselect=tools/python_api/test/test_json.py::test_get_as_df_json_extract
+            --deselect=tools/python_api/test/test_json.py::test_get_as_df_json_list
+            ${{ matrix.python == 'cp314t' && '--ignore=tools/python_api/test/test_arrow.py --ignore=tools/python_api/test/test_arrow_memory_backed_table.py --ignore=tools/python_api/test/test_df.py --ignore=tools/python_api/test/test_networkx.py --ignore=tools/python_api/test/test_scan_pandas.py --ignore=tools/python_api/test/test_scan_pandas_pyarrow.py --ignore=tools/python_api/test/test_scan_polars.py --ignore=tools/python_api/test/test_udf.py -k "not test_get_as_df_json"' || '' }}
+
+      - name: Check the extension module and licences made it into the wheel
+        run: |
+          python3 - wheelhouse/*.whl <<'EOF'
+          import sys, zipfile
+          for whl in sys.argv[1:]:
+              names = zipfile.ZipFile(whl).namelist()
+              assert any(n.startswith("ladybug/_lbug.") and n.endswith(".so") for n in names), f"no compiled extension in {whl}"
+              licences = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""}
+              assert {"LICENSE", "LICENSE.zstd", "LICENSE.re2", "LICENSE.pybind11"} <= licences, licences
+              print(whl, sorted(licences))
+          EOF
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: ladybug-${{ env.LADYBUG_VERSION }}-${{ matrix.python }}-manylinux_riscv64
+          path: wheelhouse/*.whl
+          if-no-files-found: error
+
+  publish:
+    name: Publish ladybug ${{ matrix.version }}
+    needs: [setup, build_wheels]
+    if: needs.setup.outputs.versions != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+    permissions:
+      contents: write
+      pull-requests: write
+    uses: $/.github/workflows/_publish-wheel.yml
+    secrets:
+      app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
+    with:
+      artifact-pattern: ladybug-${{ matrix.version }}-*-manylinux_riscv64

docs/packages/ladybug.yaml

@@ -0,0 +1,11 @@
+package-name: ladybug
+source-code: https://github.com/LadybugDB/ladybug
+license: MIT
+versions:
+- version: 0.19.1
+  patched: true
+  comment: >-
+    When the default 8TB buffer-manager reservation does not fit in the process address space
+    (riscv64 Sv39 gives 256GB), the database falls back to the largest power-of-two region that
+    does, which caps its maximum size accordingly. `INSTALL` looks for linux_riscv64 extension
+    builds, which upstream does not publish, instead of downloading the x86-64 ones.

patches/ladybug/0.19.1/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch

@@ -0,0 +1,48 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 25 Sep 2026 00:00:00 +0000
+Subject: [PATCH] storage: shrink the VMRegion reservation until it fits
+
+Upstream-Status: To upstream [not yet submitted; python-wheels does not open issues/PRs on third-party repos]
+
+Every Database reserves its buffer-manager region with one mmap of
+max_db_size bytes, and max_db_size defaults to 1 << 43 (8TB). riscv64
+Sv39 gives a process 256GB of user address space (the T-Head C910/C920
+cores the riscv64 runners use only implement Sv39), so the reservation
+fails with ENOMEM and every Database() created with the default
+settings throws "Mmap for size 8796093022208 failed." The same happens
+under a 39-bit VA arm64 kernel, or on x86-64 under
+`ulimit -v 268435456`. Still present in v0.20.4.
+
+On ENOMEM, halve the reservation until it fits. Where the full region
+fits nothing changes; elsewhere the database is capped at the largest
+power-of-two region the address space can hold, the limit
+max_db_size already expresses.
+---
+diff --git a/src/storage/buffer_manager/vm_region.cpp b/src/storage/buffer_manager/vm_region.cpp
+index 429bc61..d102eef 100644
+--- a/src/storage/buffer_manager/vm_region.cpp
++++ b/src/storage/buffer_manager/vm_region.cpp
+@@ -13,6 +13,8 @@
+ #else
+ #include <sys/mman.h>
+ #include <unistd.h>
++
++#include <cerrno>
+ #endif
+
+ #include "common/assert.h"
+@@ -61,6 +63,13 @@ VMRegion::VMRegion(PageSizeClass pageSizeClass, uint64_t maxRegionSize) : numFra
+     // backed by any file, and its content are initialized to zero.
+     region = static_cast<uint8_t*>(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE,
+         MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */));
++    // The default 8TB region does not fit in a smaller user address space (256GB under riscv64
++    // Sv39, 512GB under a 39-bit VA arm64 kernel), so halve the reservation until it does.
++    while (region == MAP_FAILED && errno == ENOMEM && maxNumFrameGroups > 1) {
++        maxNumFrameGroups /= 2;
++        region = static_cast<uint8_t*>(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE,
++            MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */));
++    }
+     if (region == MAP_FAILED) {
+         throw BufferManagerException(
+             "Mmap for size " + std::to_string(getMaxRegionSize()) + " failed.");

patches/ladybug/0.19.1/0002-test-repeat-the-interrupt-until-the-query-stops.patch

@@ -0,0 +1,41 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Sat, 27 Sep 2026 00:00:00 +0000
+Subject: [PATCH] test: repeat the interrupt until the query stops
+
+Upstream-Status: To upstream [not yet submitted; python-wheels does not open issues/PRs on third-party repos]
+
+test_connection_interrupt starts a long query on a thread, sleeps 5s,
+calls conn.interrupt() once and expects the thread to end within 100s.
+Binding folds each RANGE(1, 1000000) into a million-element list
+literal, and ClientContext::executeNoLock() calls resetActiveQuery(),
+which clears the interrupted flag, only once compilation is done. On
+the riscv64 runners compiling the query takes longer than 5s, so the
+interrupt lands during compilation, is wiped, and the query keeps
+running. The fixture teardown's close() then waits on it for about 4
+hours, and the query thread segfaults freeing its FactorizedTable
+after the database is gone. The same loss reproduces on x86-64 with
+upstream's 0.19.1 wheel when the sleep is shorter than the compile.
+Still present on ladybug-python main.
+
+Re-issue the interrupt every second until the thread ends, within the
+same 100s budget. On a fast machine the first interrupt still sticks
+and the test behaves as before.
+---
+diff --git a/tools/python_api/test/test_connection.py b/tools/python_api/test/test_connection.py
+index dcc8ee5..aeb2d54 100644
+--- a/tools/python_api/test/test_connection.py
++++ b/tools/python_api/test/test_connection.py
+@@ -46,6 +46,10 @@ def test_connection_interrupt(conn_db_readwrite: ConnDB) -> None:
+     execute_thread = threading.Thread(target=run_long_query, args=(conn,))
+     execute_thread.start()
+     time.sleep(5)
+-    conn.interrupt()
+-    execute_thread.join(timeout=100)
++    # Compiling this query can outlast the sleep on a slow machine, and the interrupt flag is
++    # cleared when execution starts, so an interrupt sent during compilation is lost; repeat it.
++    deadline = time.monotonic() + 100
++    while execute_thread.is_alive() and time.monotonic() < deadline:
++        conn.interrupt()
++        execute_thread.join(timeout=1)
+     assert not execute_thread.is_alive()

patches/ladybug/0.19.1/0003-extension-report-riscv64-as-its-own-platform.patch

@@ -0,0 +1,33 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Sun, 27 Sep 2026 00:00:00 +0000
+Subject: [PATCH] extension: report riscv64 as its own platform
+
+Upstream-Status: To upstream [not yet submitted; python-wheels does not open issues/PRs on third-party repos]
+
+getArch() starts from "amd64" and only overrides it for x86 and arm64,
+so on riscv64 getPlatform() returns "linux_amd64". INSTALL then
+downloads the x86-64 build of an extension from
+extension.ladybugdb.com into ~/.lbdb/extension/<ver>/linux_amd64/,
+and LOAD fails with "cannot open shared object file: No such file or
+directory", which is how glibc's dlopen reports an ELF for another
+machine. Seen in test_json.py's INSTALL json; LOAD json on the riscv64
+runners. Still present on main.
+
+Return "riscv64" there, so INSTALL looks for linux_riscv64 builds
+(upstream publishes none yet) and the extension cache is keyed by the
+right platform.
+---
+diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp
+index e87004f..e9e2891 100644
+--- a/src/extension/extension.cpp
++++ b/src/extension/extension.cpp
+@@ -144,6 +144,8 @@ std::string getArch() {
+     arch = "x86";
+ #elif defined(__aarch64__) || defined(__ARM_ARCH_ISA_A64)
+     arch = "arm64";
++#elif defined(__riscv) && __riscv_xlen == 64
++    arch = "riscv64";
+ #endif
+     return arch;
+ }