feat(index): share IVF partition scans across batch vector queries

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "9.0.0-beta.16"
+current_version = "12.0.0-beta.11"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -80,6 +80,11 @@ filename = "Cargo.toml"
 search = 'lance-index = {{ version = "={current_version}"'
 replace = 'lance-index = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-index-core = {{ version = "={current_version}"'
+replace = 'lance-index-core = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-io = {{ version = "={current_version}"'

.cargo/config.toml

@@ -22,6 +22,14 @@ lto = "thin"
 codegen-units = 16
 
 [target.x86_64-unknown-linux-gnu]
+# The default target is haswell.  This is an old enough target
+# that portability is high but a new enough target that we gain
+# some of the most common SIMD optimizations.
+#
+# On certain paths we use explicit SIMD and runtime dispatch to
+# opt-in to even higher CPU targets.  The target specified here
+# is for all code that is NOT using explicit SIMD (still the large
+# majority of code).
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
 [target.aarch64-apple-darwin]

.coderabbit.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: en-US
+early_access: false
+reviews:
+  profile: quiet
+  high_level_summary: false
+  poem: false
+  review_status: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - WIP
+      - Draft
+    drafts: false
+    base_branches:
+      - main

.gitattributes

@@ -1,3 +1,4 @@
 notebooks/** linguist-vendored
 Cargo.lock linguist-generated
-**/Cargo.lock linguist-generated
+**/Cargo.lock linguist-generated
+rust/lance-file/test_data/**/*.lance binary

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,79 @@
+name: Bug report
+description: Report incorrect, unexpected, or crashing behavior.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to file a bug report! Please search
+        [existing issues](https://github.com/lance-format/lance/issues) first to
+        avoid filing a duplicate.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: >-
+        A minimal, self-contained code snippet or sequence of steps that
+        reproduces the problem. The more we can copy-paste and run, the faster
+        we can fix it.
+      placeholder: |
+        1. Write a dataset with `...`
+        2. Scan with filter `...`
+        3. Observe `...`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: false
+  - type: input
+    id: lance-version
+    attributes:
+      label: Lance version
+      description: >-
+        Output of `python -c "import lance; print(lance.__version__)"`, or the
+        `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`.
+      placeholder: "e.g. 0.40.0"
+    validations:
+      required: true
+  - type: dropdown
+    id: language
+    attributes:
+      label: Language binding
+      multiple: true
+      options:
+        - Python
+        - Rust
+        - Java
+        - Other / not sure
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...).
+      placeholder: "e.g. Ubuntu 22.04, x86_64, S3"
+    validations:
+      required: false
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs / traceback
+      description: >-
+        Any relevant log output or stack trace. This will be automatically
+        formatted as code, so no need for backticks.
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question / usage help
+    url: https://discord.gg/lance
+    about: >-
+      For questions and general discussion, please ask in the Lance Discord
+      rather than opening an issue.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest a new capability or an improvement to an existing one.
+labels: feature
+---
+
+<!--
+Describe what you'd like Lance to do and why. There's no required format —
+mention the problem or use case, and a rough idea of the solution if you have
+one. Please search existing issues first to avoid duplicates.
+-->

.github/ISSUE_TEMPLATE/performance_issue.md

@@ -0,0 +1,12 @@
+---
+name: Performance issue
+about: Report slow operations, high memory use, or a performance regression.
+labels: performance
+---
+
+<!--
+Describe what's slow or memory-hungry. There's no required format, but concrete
+numbers (timings, memory, dataset shape) and a way to reproduce help a lot. If
+it's a regression, note the last version that performed well. Please search
+existing issues first to avoid duplicates.
+-->

.github/dependabot.yml

@@ -1,45 +1,86 @@
 version: 2
 updates:
+  # Rust — one grouped version-bump PR per directory each week, and security
+  # advisories that land together in a directory batched into a single PR
+  # instead of one PR per advisory.
   - package-ecosystem: "cargo"
-    directory: "/"
+    directories:
+      - "/"
+      - "/python"
+      - "/java/lance-jni"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
       cargo:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      cargo-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
-  - package-ecosystem: "cargo"
+  # Python (uv) — same weekly-grouped + grouped-security treatment.
+  - package-ecosystem: "uv"
     directory: "/python"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      cargo:
+      uv:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      uv-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
+  # Standalone Rust lockfiles for dev/test fixtures. Dependabot security updates
+  # scan these regardless of config, so we group them; open-pull-requests-limit: 0
+  # disables the (unwanted) weekly version bumps for these crates.
   - package-ecosystem: "cargo"
-    directory: "/java/lance-jni"
-    versioning-strategy: lockfile-only
+    directories:
+      - "/memtest"
+      - "/test_data/fri_straddle_pre_6610/datagen"
+    open-pull-requests-limit: 0
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      cargo:
+      cargo-fixtures-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
-  - package-ecosystem: "uv"
-    directory: "/python"
-    versioning-strategy: lockfile-only
+  # Java (Maven). Security updates fire regardless of config; group them and
+  # skip version bumps (limit 0) — we bump Java deps manually.
+  - package-ecosystem: "maven"
+    directory: "/java"
+    open-pull-requests-limit: 0
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      uv:
+      maven-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Python benchmark scripts (pip / requirements.txt). Group security updates
+  # and skip version bumps (limit 0).
+  - package-ecosystem: "pip"
+    directories:
+      - "/benchmarks/*"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      pip-security:
+        applies-to: security-updates
         patterns:
           - "*"

.github/labeler-area.yml

@@ -29,12 +29,40 @@ A-encoding:
           - "rust/lance-io/**"
           - "rust/lance-file/**"
 
-# On-disk format: the proto definitions and the format spec docs. Format docs
+# On-disk format: persisted proto definitions and the format spec docs. Keep the
+# explicit proto lists here in sync. Execution-plan schemas (ann,
+# filtered_read, and table_identifier) are intentionally excluded. Format docs
 # are excluded from A-docs (see below) so format changes get A-format only.
 A-format:
   - changed-files:
       - any-glob-to-any-file:
-          - "protos/**"
+          - "protos/encodings_v2_0.proto"
+          - "protos/encodings_v2_1.proto"
+          - "protos/file.proto"
+          - "protos/file2.proto"
+          - "protos/index.proto"
+          - "protos/index_old.proto"
+          - "protos/rowids.proto"
+          - "protos/table.proto"
+          - "protos/transaction.proto"
+          - "docs/src/format/**"
+
+# Drives the format-spec vote gate (.github/workflows/format-vote-gate.yml):
+# any change to persisted proto definitions or the spec docs requires a PMC
+# vote. Execution-only wire schemas such as filtered_read.proto are not format
+# changes. A PMC member waives a trivial edit with the `format-waived` label.
+format-change:
+  - changed-files:
+      - any-glob-to-any-file:
+          - "protos/encodings_v2_0.proto"
+          - "protos/encodings_v2_1.proto"
+          - "protos/file.proto"
+          - "protos/file2.proto"
+          - "protos/index.proto"
+          - "protos/index_old.proto"
+          - "protos/rowids.proto"
+          - "protos/table.proto"
+          - "protos/transaction.proto"
           - "docs/src/format/**"
 
 # Lockfiles are intentionally not excluded: a pure dependency bump gets both

.github/labeler-issues.yml

@@ -0,0 +1,28 @@
+version: 1
+# Never remove labels a template or a human already applied.
+appendOnly: true
+# Content-based labels for issues, applied by srvaroa/labeler via
+# .github/workflows/issue-labeler.yml. This covers issues that bypass the
+# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`,
+# the REST API, or an agent, which never see the web-UI template chooser.
+#
+# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]",
+# "perf -"), which many people type by hand; a modest body keyword match is a
+# secondary fallback. Rules are OR'd (one label per matching entry), so an
+# unmatched issue gets no label and is left for human / LLM triage rather than
+# guessed at. appendOnly means matches stack harmlessly with template labels.
+labels:
+# --- Primary signal: a leading bug/feature/perf marker in the title ---
+- label: bug
+  title: "(?i)^\\s*[\\[(]?\\s*bug\\b"
+- label: feature
+  title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b"
+- label: performance
+  title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b"
+# --- Secondary fallback: keywords anywhere in the body ---
+- label: bug
+  body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)"
+- label: performance
+  body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)"
+- label: feature
+  body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)"

.github/nextest/daily-code-coverage.toml

@@ -0,0 +1,3 @@
+[profile.daily-code-coverage]
+slow-timeout = { period = "60s", terminate-after = 30, grace-period = "10s" }
+global-timeout = "5h30m"