diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
index 366ba36df..dcddcb6d9 100644
--- a/.github/workflows/unit_test.yml
+++ b/.github/workflows/unit_test.yml
@@ -49,4 +49,70 @@ jobs:
# for unit test
cd $WORK
echo Testing code at $PWD ...
+ OFFLINE_BUILD=true CI=true make tidy test
+
+ enterprise-test:
+ timeout-minutes: 60
+ runs-on: ubuntu-latest
+ env:
+ GO_VERSION: 1.23.4
+ steps:
+ - name: Checkout current repository
+ uses: actions/checkout@v4
+ with:
+ path: framework
+
+ - name: Set up go toolchain
+ uses: actions/setup-go@v5
+ with:
+ go-version: ${{ env.GO_VERSION }}
+ check-latest: false
+ cache: true
+
+ # Enterprise plugins overlay (private repo): prefer the branch
+ # matching this PR's branch name, fall back to the default branch
+ # when it does not exist there; the license module is a build
+ # dependency of the overlay.
+ - name: Checkout enterprise plugins overlay
+ env:
+ CI_GH_TOKEN: ${{ secrets.CI_TOKEN || secrets.DOCS_DEPLOYMENT_TOKEN }}
+ OVERLAY_REF: ${{ github.head_ref }}
+ run: |
+ set -e
+ REPO="https://x-access-token:${CI_GH_TOKEN}@github.com/infinilabs/framework-enterprise-plugins.git"
+ if [ -n "$OVERLAY_REF" ] && git ls-remote --exit-code --heads "$REPO" "refs/heads/${OVERLAY_REF}" >/dev/null 2>&1; then
+ git clone --depth 1 --branch "${OVERLAY_REF}" "$REPO" framework/plugins/enterprise
+ else
+ git clone --depth 1 "$REPO" framework/plugins/enterprise
+ fi
+ git -C framework/plugins/enterprise log --oneline -1
+
+ - name: Checkout license module
+ uses: actions/checkout@v4
+ with:
+ token: ${{ secrets.CI_TOKEN || secrets.DOCS_DEPLOYMENT_TOKEN }}
+ repository: infinilabs/license
+ ref: main
+ path: license
+
+ - name: Check go toolchain
+ run: go version
+
+ - name: Unit test with enterprise overlay
+ env:
+ GOFLAGS: -tags=ci
+ run: |
+ echo Home path is $HOME
+ export WORKBASE=$HOME/go/src/infini.sh
+ export WORK=$WORKBASE/framework
+
+ # for test workspace
+ mkdir -p $HOME/go/src/
+ ln -s $GITHUB_WORKSPACE $WORKBASE
+
+ # the overlay's umbrella packages need the private infini.sh/license module (kept out of go.mod)
+ cd $WORK
+ go mod edit -replace infini.sh/license=../license
+
+ echo "Testing code at $PWD (with enterprise overlay) ..."
OFFLINE_BUILD=true CI=true make tidy test
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index c8d9a0b3a..c8f7b8aea 100755
--- a/.gitignore
+++ b/.gitignore
@@ -62,3 +62,4 @@ generated_*.go
# keystore runtime artifacts
core/elastic/.keystore/
+.mimosa/
diff --git a/README.md b/README.md
index 857d7142b..7bb0c592a 100755
--- a/README.md
+++ b/README.md
@@ -1,5 +1,22 @@
# INFINI Framework
+## Pipeline Terminology (glossary)
+
+The data-processing subsystems use a fixed vocabulary. The short version:
+
+| Term | Meaning |
+|------|---------|
+| **message** | A queue's unit of storage — opaque bytes. Queues never interpret payloads. |
+| **batch** | The pipeline's native input: a slice of messages, the *batch scope* all historical processors operate on. |
+| **record** | A typed, decoded event inside a record host's sub-chain — the *record scope*; mutable in place, never leaks downstream. |
+| **record host** | A splitter processor that turns a batch into records and back: `for_each`, `lane_consumer`, or a custom host. |
+| **codec** | Pluggable byte↔record converter (`otel` default, `coco_document`, ...). |
+| **namespace** | A record family (`document`, `attachment`) used in compile-time contract checks. |
+| **queue** | Durable at-least-once **byte** channel; cross-process safe. |
+| **lane** | In-process at-most-once **record** channel (zero serialization, pointer hand-off). |
+
+Full definitions and the queue-vs-lane selection guide live in the docs: [Glossary](docs/content.en/docs/references/glossary.md), [Pipeline Record Processing](docs/content.en/docs/references/pipeline_record.md), [Lane](docs/content.en/docs/references/lane.md).
+
## Requirements
- Go 1.21+
diff --git a/core/config/config_test.go b/core/config/config_test.go
index 3f6d990a1..e02ebbf4e 100755
--- a/core/config/config_test.go
+++ b/core/config/config_test.go
@@ -971,3 +971,47 @@ func TestLoadFileExpandsConfigTemplatesPlain(t *testing.T) {
assert.Equal(t, out.Entry[0].Name, "my_entry_test_entry")
assert.Equal(t, out.Entry[0].Network.Binding, "127.0.0.1:8002")
}
+
+// A processor entry with an empty body (`- echo: {}`) must keep its key
+// through LoadPath + Unpack. Empty dicts used to reify to nil, which made
+// the KEY vanish from map[string]interface{} targets and broke every
+// config-file-loaded pipeline (managed instances saw "each processor must
+// have exactly one action, but found 0 actions").
+func TestLoadPathKeepsEmptyProcessorKeys(t *testing.T) {
+ dir := t.TempDir()
+ yml := "pipeline:\n" +
+ " - name: t\n" +
+ " auto_start: false\n" +
+ " processor:\n" +
+ " - echo: {}\n" +
+ " - node_stamp:\n" +
+ " field: x\n"
+ if err := os.WriteFile(filepath.Join(dir, "p.yml"), []byte(yml), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ cfg, err := LoadPath(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ child, err := cfg.Child("pipeline", -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ type rawPipeline struct {
+ Processor []map[string]interface{} `config:"processor"`
+ }
+ list := []rawPipeline{}
+ if err := child.Unpack(&list); err != nil {
+ t.Fatal(err)
+ }
+ if len(list) != 1 || len(list[0].Processor) != 2 {
+ t.Fatalf("unexpected unpack result: %#v", list)
+ }
+ if _, ok := list[0].Processor[0]["echo"]; !ok {
+ t.Fatalf("empty processor body lost its key: %#v", list[0].Processor[0])
+ }
+ if _, ok := list[0].Processor[1]["node_stamp"]; !ok {
+ t.Fatalf("node_stamp lost its key: %#v", list[0].Processor[1])
+ }
+}
diff --git a/core/elastic/cluster_secrets.go b/core/elastic/cluster_secrets.go
index 156f9a671..57ad61d49 100644
--- a/core/elastic/cluster_secrets.go
+++ b/core/elastic/cluster_secrets.go
@@ -107,22 +107,25 @@ func HydrateClusterSecrets(cfg *ElasticsearchConfig) {
v, ok := loadClusterSecret(ClusterBasicAuthPasswordKey(cfg.ID))
if ok {
if v == "" {
- log.Warnf("cluster [%s]'s basic auth password is empty: %v", cfg.ID)
+ log.Warnf("cluster [%s]'s basic auth password is empty", cfg.ID)
}
cfg.BasicAuth.Password = ucfg.SecretString(v)
} else {
- log.Warnf("cluster [%s]'s basic auth password not found in the keystore: %v", cfg.ID)
+ log.Warnf("cluster [%s]'s basic auth password not found in the keystore", cfg.ID)
}
}
- if !isRealSecret(cfg.Token.Get()) {
+ // Empty token = the cluster never had one (no-auth) — nothing to
+ // hydrate and nothing to warn about. Only a MASKED token (record says a
+ // token exists, value stashed elsewhere) requires the keystore lookup.
+ if tok := cfg.Token.Get(); tok != "" && !isRealSecret(tok) {
v, ok := loadClusterSecret(ClusterTokenKey(cfg.ID))
if ok {
if v == "" {
- log.Warnf("cluster [%s]'s token is empty: %v", cfg.ID)
+ log.Warnf("cluster [%s]'s token is empty", cfg.ID)
}
cfg.Token = ucfg.SecretString(v)
} else {
- log.Warnf("cluster [%s]'s token not found in the keystore: %v", cfg.ID)
+ log.Warnf("cluster [%s]'s token not found in the keystore", cfg.ID)
}
}
}
@@ -171,6 +174,13 @@ func isRealSecret(s string) bool {
return s != "" && s != ucfg.SecretShadowText
}
+// IsRealSecret is the exported form of isRealSecret, for app-side code
+// building credential payloads from ORM-loaded cluster records (deciding
+// whether a hydrated value is safe to ship).
+func IsRealSecret(s string) bool {
+ return isRealSecret(s)
+}
+
func asMapStr(v interface{}) map[string]interface{} {
switch m := v.(type) {
case map[string]interface{}:
diff --git a/core/elastic/cluster_secrets_test.go b/core/elastic/cluster_secrets_test.go
index 33836208c..1b35bf822 100644
--- a/core/elastic/cluster_secrets_test.go
+++ b/core/elastic/cluster_secrets_test.go
@@ -9,8 +9,8 @@ import (
"os"
"testing"
- "infini.sh/framework/core/model"
"infini.sh/framework/core/keystore"
+ "infini.sh/framework/core/model"
ucfg "infini.sh/framework/lib/go-ucfg"
)
diff --git a/core/env/version.go b/core/env/version.go
index 5fef4a4ef..dd9651194 100644
--- a/core/env/version.go
+++ b/core/env/version.go
@@ -28,13 +28,17 @@
package env
type Version struct {
- VersionNumber string `json:"number,omitempty"`
- FrameworkBuildCommitHash string `json:"framework_hash,omitempty"`
- FrameworkVendorBuildCommitHash string `json:"vendor_hash,omitempty"`
- BuildCommitHash string `json:"build_hash,omitempty"`
- BuildDate string `json:"build_date,omitempty"`
- BuildNumber string `json:"build_number,omitempty"`
- EolDate string `json:"eol_date,omitempty"`
+ VersionNumber string `json:"number,omitempty" elastic_mapping:"number: { type: keyword }"`
+ FrameworkBuildCommitHash string `json:"framework_hash,omitempty" elastic_mapping:"framework_hash: { type: keyword }"`
+ FrameworkVendorBuildCommitHash string `json:"vendor_hash,omitempty" elastic_mapping:"vendor_hash: { type: keyword }"`
+ BuildCommitHash string `json:"build_hash,omitempty" elastic_mapping:"build_hash: { type: keyword }"`
+ // BuildDate/EolDate must map to keyword, never a dynamic date: binaries
+ // built without version ldflags report placeholders ("N/A"), which a
+ // date mapping rejects with document_parsing_exception — killing
+ // instance heartbeats on indices whose mapping predates this pin.
+ BuildDate string `json:"build_date,omitempty" elastic_mapping:"build_date: { type: keyword }"`
+ BuildNumber string `json:"build_number,omitempty" elastic_mapping:"build_number: { type: keyword }"`
+ EolDate string `json:"eol_date,omitempty" elastic_mapping:"eol_date: { type: keyword }"`
}
func (env *Env) GetVersionInfo() Version {
diff --git a/core/event/event.go b/core/event/event.go
index 87a89f322..9a3949231 100644
--- a/core/event/event.go
+++ b/core/event/event.go
@@ -47,6 +47,13 @@ type Event struct {
Meta util.MapStr `json:"-"`
Private interface{} `json:"-"` // for beats private use
TimeSeries bool `json:"-"` // true if the event contains timeseries data
+
+ // pooled links to the record-pool wrapper while the event came from
+ // AcquireEvent; managed by pool.go, never serialized. Held as
+ // interface{} on purpose: a typed *pooledEvent back-reference would
+ // create a recursive type (Event -> pooledEvent -> Event) and blow
+ // the reflection walkers (GetTagsByTagName) with a stack overflow.
+ pooled interface{}
}
type EventMetadata struct {
diff --git a/core/event/pool.go b/core/event/pool.go
new file mode 100644
index 000000000..2217fecf7
--- /dev/null
+++ b/core/event/pool.go
@@ -0,0 +1,138 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello#infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+/* ©INFINI, All Rights Reserved.
+ * mail: contact#infini.ltd */
+
+package event
+
+import (
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// The record pool recycles *Event across record-mode batches: decode-heavy
+// hosts (for_each) otherwise allocate a fresh Event, its field maps and the
+// whole decoded tree per record only to drop them after encode.
+//
+// Contract:
+// - AcquireEvent returns a zeroed event; Decode-style producers write into
+// it as if it were freshly allocated.
+// - ReleaseEvent returns the event to the pool. It must be called only
+// after the record has been fully encoded (or dropped) AND no downstream
+// processor retains a reference — the host owns the record lifetime.
+// - ReleaseEvent is a no-op on events that did not come from the pool
+// (codec- or clone-allocated), so a host can release every record
+// unconditionally.
+// - A double release is rejected atomically (second call returns false)
+// instead of pooling the same event twice.
+//
+// Reset policy: scalar fields are zeroed; the hot maps (Fields, Metadata
+// labels, Meta) are cleared in place — their bucket arrays are reused —
+// unless they grew past maxRecycledEntries, in which case they are dropped
+// so one giant record cannot pin memory in the pool forever.
+
+// maxRecycledEntries caps in-place map reuse; bigger maps are released to
+// the GC instead of living in the pool.
+const maxRecycledEntries = 128
+
+type pooledEvent struct {
+ live atomic.Bool
+ ev Event
+}
+
+var recordPool = sync.Pool{New: func() interface{} { return &pooledEvent{} }}
+
+var (
+ poolAcquires atomic.Int64
+ poolReleases atomic.Int64
+)
+
+// AcquireEvent returns an event from the record pool (zeroed).
+func AcquireEvent() *Event {
+ pe := recordPool.Get().(*pooledEvent)
+ pe.live.Store(true)
+ pe.ev.pooled = pe
+ poolAcquires.Add(1)
+ return &pe.ev
+}
+
+// ReleaseEvent recycles a pooled event; returns true when the event went
+// back to the pool, false for non-pooled events or a rejected double
+// release.
+func ReleaseEvent(e *Event) bool {
+ if e == nil {
+ return false
+ }
+ pe, ok := e.pooled.(*pooledEvent)
+ if !ok {
+ return false
+ }
+ if !pe.live.CompareAndSwap(true, false) {
+ return false
+ }
+ e.resetForPool()
+ e.pooled = nil
+ poolReleases.Add(1)
+ recordPool.Put(pe)
+ return true
+}
+
+// PoolStats reports cumulative acquire/release counts; the difference is
+// the number of pooled events currently in use (leak detection).
+func PoolStats() (acquired, released int64) {
+ return poolAcquires.Load(), poolReleases.Load()
+}
+
+// resetForPool clears an event for reuse; see the pool doc for the
+// map-retention policy.
+func (e *Event) resetForPool() {
+ e.Fields = recycleMap(e.Fields)
+ e.Metadata.Labels = recycleMap(e.Metadata.Labels)
+ e.Meta = recycleMap(e.Meta)
+ e.Metadata.Category = ""
+ e.Metadata.Name = ""
+ e.Metadata.Version = ""
+ e.Metadata.Datatype = ""
+ e.Agent = nil
+ e.Private = nil
+ e.QueueName = ""
+ e.TimeSeries = false
+ e.Timestamp = time.Time{}
+}
+
+// recycleMap clears a map in place when it is small enough to be worth
+// keeping, else drops it.
+func recycleMap(m map[string]interface{}) map[string]interface{} {
+ if m == nil {
+ return nil
+ }
+ if len(m) > maxRecycledEntries {
+ return nil
+ }
+ for k := range m {
+ delete(m, k)
+ }
+ return m
+}
diff --git a/core/event/pool_test.go b/core/event/pool_test.go
new file mode 100644
index 000000000..b1d539483
--- /dev/null
+++ b/core/event/pool_test.go
@@ -0,0 +1,120 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package event
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/util"
+)
+
+func TestPoolAcquireReleaseBalance(t *testing.T) {
+ a0, r0 := PoolStats()
+ e := AcquireEvent()
+ e.Fields = util.MapStr{"k": "v"}
+ if !ReleaseEvent(e) {
+ t.Fatal("releasing a pooled event must report true")
+ }
+ a1, r1 := PoolStats()
+ if a1-a0 != 1 || r1-r0 != 1 {
+ t.Fatalf("stats must balance: acquired +%d released +%d", a1-a0, r1-r0)
+ }
+}
+
+func TestPoolReleaseNonPooledIsNoop(t *testing.T) {
+ if ReleaseEvent(&Event{}) {
+ t.Fatal("a plain event must not enter the pool")
+ }
+ if ReleaseEvent(nil) {
+ t.Fatal("nil must not enter the pool")
+ }
+}
+
+func TestPoolDoubleReleaseRejected(t *testing.T) {
+ _, r0 := PoolStats()
+ e := AcquireEvent()
+ if !ReleaseEvent(e) {
+ t.Fatal("first release must succeed")
+ }
+ if ReleaseEvent(e) {
+ t.Fatal("second release of the same event must be rejected")
+ }
+ _, r1 := PoolStats()
+ if r1-r0 != 1 {
+ t.Fatalf("double release must not inflate the release count: +%d", r1-r0)
+ }
+}
+
+func TestPoolResetClearsEverything(t *testing.T) {
+ e := AcquireEvent()
+ e.Fields = util.MapStr{"document": map[string]interface{}{"id": "x"}}
+ e.Metadata = EventMetadata{
+ Labels: util.MapStr{"datasource": "d1"},
+ Category: "cat", Name: "n", Version: "v", Datatype: "dt",
+ }
+ e.Meta = util.MapStr{"m": "1"}
+ e.Agent = &AgentMeta{AgentID: "a"}
+ e.Private = struct{}{}
+ e.QueueName = "q"
+ e.TimeSeries = true
+ e.Timestamp = time.Now()
+
+ e.resetForPool()
+
+ if len(e.Fields) != 0 || len(e.Metadata.Labels) != 0 || len(e.Meta) != 0 {
+ t.Fatal("maps must be emptied")
+ }
+ if e.Agent != nil || e.Private != nil || e.QueueName != "" || e.TimeSeries {
+ t.Fatal("scalar/pointer state must be zeroed")
+ }
+ if !e.Timestamp.IsZero() || e.Metadata.Category != "" || e.Metadata.Name != "" ||
+ e.Metadata.Version != "" || e.Metadata.Datatype != "" {
+ t.Fatal("metadata and timestamp must be zeroed")
+ }
+ ReleaseEvent(e)
+}
+
+func TestPoolMapRecyclePolicy(t *testing.T) {
+ small := util.MapStr{}
+ for i := 0; i < maxRecycledEntries; i++ {
+ small["k"+string(rune('a'+i%26))+string(rune('0'+i/26))] = i
+ }
+ if got := recycleMap(small); got == nil || len(got) != 0 {
+ t.Fatal("small maps must be cleared and kept")
+ }
+ big := util.MapStr{}
+ for i := 0; i <= maxRecycledEntries; i++ {
+ big[string(rune('a'+i%26))+string(rune('0'+i/26))] = i
+ }
+ if got := recycleMap(big); got != nil {
+ t.Fatal("oversized maps must be dropped, not recycled")
+ }
+}
+
+// TestPoolConcurrentAcquireRelease is the -race gate: concurrent owners
+// recycle disjoint events and the counters balance at the end.
+func TestPoolConcurrentAcquireRelease(t *testing.T) {
+ const workers = 8
+ const rounds = 200
+ _, r0 := PoolStats()
+ var wg sync.WaitGroup
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for i := 0; i < rounds; i++ {
+ e := AcquireEvent()
+ e.Fields = util.MapStr{"message": "x"}
+ ReleaseEvent(e)
+ }
+ }()
+ }
+ wg.Wait()
+ _, r1 := PoolStats()
+ if got := r1 - r0; got != workers*rounds {
+ t.Fatalf("every acquire must be released exactly once: released %d of %d", got, workers*rounds)
+ }
+}
diff --git a/core/lane/lane.go b/core/lane/lane.go
new file mode 100644
index 000000000..d681efbdf
--- /dev/null
+++ b/core/lane/lane.go
@@ -0,0 +1,514 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+// Package lane is the in-process record channel: the real-time
+// counterpart of the disk queue. Payloads are *event.Event POINTERS
+// (the record IS the message) — zero serialization in transit; delivery
+// is at-most-once (Emit returning marks the producer's responsibility
+// end), bounded memory with explicit backpressure (block by default,
+// drop for best-effort taps, spill only when fast and slow paths run
+// equivalent chains).
+package lane
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "time"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// LaneConfig declares one lane.
+type LaneConfig struct {
+ Name string `config:"name" json:"name"`
+ Namespace string `config:"namespace" json:"namespace"` // record family contract
+ Capacity int `config:"capacity" json:"capacity"` // bounded; in-flight records
+ // Overflow policy when the lane is full:
+ // block default — Emit waits, backpressure reaches the producer
+ // drop best-effort — discard and count
+ // spill: degrade to the disk queue (fast/slow paths must run
+ // EQUIVALENT chains — a required step on the fast path
+ // would be bypassed for spilled records)
+ Overflow string `config:"overflow" json:"overflow"`
+ // Owner marks dynamic lanes for cascade cleanup ("" = config-owned).
+ Owner string `config:"owner,omitempty" json:"owner,omitempty"`
+}
+
+const (
+ defaultCapacity = 4096
+ dynamicCapacity = 512
+)
+
+// Handler consumes one delivered record.
+type Handler func(rec *event.Event)
+
+// Stats snapshots a lane's counters for observability.
+type Stats struct {
+ Name string `json:"name"`
+ Namespace string `json:"namespace"`
+ Owner string `json:"owner,omitempty"`
+ Capacity int `json:"capacity"`
+ Depth int `json:"depth"`
+ Emitted int64 `json:"emitted"`
+ Delivered int64 `json:"delivered"`
+ Drops int64 `json:"drops"`
+ Spills int64 `json:"spills"`
+ BlockedWaits int64 `json:"blocked_waits"`
+ Closed bool `json:"closed"`
+}
+
+type laneImpl struct {
+ cfg LaneConfig
+ ch chan *event.Event
+ mu sync.Mutex // guards counters + closed + pending
+ stats Stats
+ spillQ *queue.QueueConfig
+ pending map[*event.Event]*laneReceipt
+ quit chan struct{}
+ once sync.Once
+}
+
+// Lane is one named bounded record channel.
+type Lane interface {
+ Config() LaneConfig
+ Emit(rec *event.Event) error
+ TryEmit(rec *event.Event) bool
+ // EmitAck emits with an acknowledgment receipt (commit-gate support).
+ EmitAck(rec *event.Event) pipeline.Receipt
+ // SubscribeBatch consumes in micro-batches: receipts of a batch's
+ // records settle only after fn(batch) returns (batch hosts flush on
+ // size or idle timeout).
+ SubscribeBatch(max int, flushInterval time.Duration, fn func(recs []*event.Event)) (cancel func())
+ Subscribe(h Handler) (cancel func())
+ Depth() int
+ Snapshot() Stats
+ Close() error // drain-or-drop; at-most-once semantics end here
+}
+
+func (l *laneImpl) Config() LaneConfig { return l.cfg }
+
+func (l *laneImpl) Emit(rec *event.Event) error {
+ if rec == nil {
+ return fmt.Errorf("lane %s: nil record", l.cfg.Name)
+ }
+ l.mu.Lock()
+ if l.stats.Closed {
+ l.mu.Unlock()
+ return fmt.Errorf("lane %s: closed", l.cfg.Name)
+ }
+ l.mu.Unlock()
+
+ for {
+ select {
+ case l.ch <- rec:
+ l.mu.Lock()
+ l.stats.Emitted++
+ l.mu.Unlock()
+ return nil
+ default:
+ }
+ switch l.overflowPolicy() {
+ case "drop":
+ l.mu.Lock()
+ l.stats.Drops++
+ r := l.pending[rec]
+ delete(l.pending, rec)
+ l.mu.Unlock()
+ if r != nil {
+ r.settle(nil) // explicit drop is a settlement
+ }
+ return nil
+ case "spill":
+ if err := l.spill(rec); err == nil {
+ l.mu.Lock()
+ l.stats.Spills++
+ l.mu.Unlock()
+ return nil
+ } // spill failure falls through to block
+ }
+ // block: wait for room or shutdown
+ l.mu.Lock()
+ l.stats.BlockedWaits++
+ l.mu.Unlock()
+ select {
+ case l.ch <- rec:
+ l.mu.Lock()
+ l.stats.Emitted++
+ l.mu.Unlock()
+ return nil
+ case <-time.After(50 * time.Millisecond):
+ l.mu.Lock()
+ closed := l.stats.Closed
+ l.mu.Unlock()
+ if closed {
+ return fmt.Errorf("lane %s: closed while blocking", l.cfg.Name)
+ }
+ }
+ }
+}
+
+// laneReceipt satisfies pipeline.Receipt.
+type laneReceipt struct{ done chan error }
+
+func (r *laneReceipt) Wait(ctx context.Context) error {
+ select {
+ case err := <-r.done:
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// Done exposes the settle signal for commit-gate polling.
+func (r *laneReceipt) Done() <-chan error { return r.done }
+
+func (r *laneReceipt) settle(err error) {
+ select {
+ case r.done <- err:
+ default:
+ }
+}
+
+// EmitAck emits and returns a receipt settled when the record's
+// processing completes in the subscriber (handler return = settlement:
+// output_queue/dead_letter pushes happen inside the handler's chain).
+// Failed emissions settle with the error immediately.
+func (l *laneImpl) EmitAck(rec *event.Event) pipeline.Receipt {
+ r := &laneReceipt{done: make(chan error, 1)}
+ if rec == nil {
+ r.settle(fmt.Errorf("lane %s: nil record", l.cfg.Name))
+ return r
+ }
+ l.mu.Lock()
+ if l.stats.Closed {
+ l.mu.Unlock()
+ r.settle(fmt.Errorf("lane %s: closed", l.cfg.Name))
+ return r
+ }
+ if l.pending == nil {
+ l.pending = map[*event.Event]*laneReceipt{}
+ }
+ l.pending[rec] = r
+ l.mu.Unlock()
+ if err := l.Emit(rec); err != nil {
+ l.settlePending(rec, err)
+ }
+ return r
+}
+
+func (l *laneImpl) settlePending(rec *event.Event, err error) {
+ l.mu.Lock()
+ r := l.pending[rec]
+ delete(l.pending, rec)
+ l.mu.Unlock()
+ if r != nil {
+ r.settle(err)
+ }
+}
+
+func (l *laneImpl) TryEmit(rec *event.Event) bool {
+ select {
+ case l.ch <- rec:
+ l.mu.Lock()
+ l.stats.Emitted++
+ l.mu.Unlock()
+ return true
+ default:
+ return false
+ }
+}
+
+func (l *laneImpl) overflowPolicy() string {
+ if l.cfg.Overflow != "" {
+ return l.cfg.Overflow
+ }
+ return "block"
+}
+
+// spill encodes the record (three-level codec resolution, mirroring the
+// lane_consumer output bridge) and degrades it to the disk queue.
+func (l *laneImpl) spill(rec *event.Event) error {
+ if l.spillQ == nil {
+ qname := util.TrimSpaces(l.cfg.Overflow[len("spill:"):])
+ if qname == "" {
+ return fmt.Errorf("lane %s: spill target missing", l.cfg.Name)
+ }
+ cfg := &queue.QueueConfig{Name: qname}
+ l.spillQ = queue.SmartGetOrInitConfig(cfg)
+ }
+ codec := l.resolveCodec(rec)
+ b, err := codec.Encode(rec)
+ if err != nil {
+ return err
+ }
+ return queue.Push(l.spillQ, b)
+}
+
+// resolveCodec picks the output codec: the record's _codec hint, then
+// this lane's namespace primary codec, then the default.
+func (l *laneImpl) resolveCodec(rec *event.Event) pipeline.RecordCodec {
+ if id := pipeline.CodecHint(rec); id != "" {
+ if c := pipeline.GetRecordCodec(id); c.Name() == id {
+ return c
+ }
+ }
+ if l.cfg.Namespace != "" {
+ for _, ci := range pipeline.ListRecordCodecs() {
+ if ci.Namespace == l.cfg.Namespace {
+ return pipeline.GetRecordCodec(ci.Name)
+ }
+ }
+ }
+ return pipeline.GetRecordCodec("")
+}
+
+func (l *laneImpl) Subscribe(h Handler) (cancel func()) {
+ return l.SubscribeBatch(1, 0, func(recs []*event.Event) {
+ for _, rec := range recs {
+ h(rec)
+ }
+ })
+}
+
+// SubscribeBatch delivers micro-batches: a batch flushes when it reaches
+// max records or after flushInterval of idleness. Receipts of every record
+// in the batch settle AFTER fn returns — batch processing completion is
+// the settlement point, keeping EmitAck's commit-gate semantics exact for
+// batching consumers.
+func (l *laneImpl) SubscribeBatch(max int, flushInterval time.Duration, fn func(recs []*event.Event)) (cancel func()) {
+ if max <= 0 {
+ max = 1
+ }
+ go func() {
+ var batch []*event.Event
+ var receipts []*laneReceipt
+ timer := time.NewTimer(0)
+ if !timer.Stop() {
+ <-timer.C
+ }
+ flush := func() {
+ if len(batch) == 0 {
+ return
+ }
+ l.mu.Lock()
+ l.stats.Delivered += int64(len(batch))
+ l.mu.Unlock()
+ fn(batch)
+ for _, r := range receipts {
+ r.settle(nil)
+ }
+ batch = nil
+ receipts = nil
+ }
+ for {
+ var flushC <-chan time.Time
+ if len(batch) > 0 && flushInterval > 0 {
+ flushC = timer.C
+ }
+ select {
+ case rec, ok := <-l.ch:
+ if !ok {
+ flush()
+ return
+ }
+ l.mu.Lock()
+ r := l.pending[rec]
+ delete(l.pending, rec)
+ l.mu.Unlock()
+ batch = append(batch, rec)
+ if r != nil {
+ receipts = append(receipts, r)
+ }
+ if len(batch) >= max {
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
+ flush()
+ continue
+ }
+ if flushInterval > 0 && len(batch) == 1 {
+ timer.Reset(flushInterval)
+ }
+ case <-flushC:
+ flush()
+ case <-l.quit:
+ // drain what is already buffered, then stop
+ for {
+ select {
+ case rec, ok := <-l.ch:
+ if !ok {
+ flush()
+ return
+ }
+ l.mu.Lock()
+ r := l.pending[rec]
+ delete(l.pending, rec)
+ l.mu.Unlock()
+ batch = append(batch, rec)
+ if r != nil {
+ receipts = append(receipts, r)
+ }
+ default:
+ flush()
+ return
+ }
+ }
+ }
+ }
+ }()
+ return func() { /* subscriber cancel: lanes are single-consumer v1 */ }
+}
+
+func (l *laneImpl) Depth() int { return len(l.ch) }
+
+func (l *laneImpl) Snapshot() Stats {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ s := l.stats
+ s.Name, s.Namespace, s.Owner, s.Capacity = l.cfg.Name, l.cfg.Namespace, l.cfg.Owner, l.cfg.Capacity
+ s.Depth = len(l.ch)
+ return s
+}
+
+func (l *laneImpl) Close() error {
+ l.once.Do(func() {
+ l.mu.Lock()
+ l.stats.Closed = true
+ for rec, r := range l.pending {
+ r.settle(fmt.Errorf("lane %s: closed with record in flight", l.cfg.Name))
+ delete(l.pending, rec)
+ }
+ l.mu.Unlock()
+ close(l.quit)
+ })
+ return nil
+}
+
+// ─── registry ────────────────────────────────────────────────────────────────
+
+var (
+ regMu sync.RWMutex
+ registry = map[string]Lane{}
+)
+
+// Create registers a lane (duplicate name errors; runtime callers react,
+// config-plane init uses the framework's fail-fast conventions upstream).
+func Create(cfg LaneConfig) (Lane, error) {
+ if cfg.Name == "" {
+ return nil, fmt.Errorf("lane name is empty")
+ }
+ if cfg.Capacity <= 0 {
+ cfg.Capacity = defaultCapacity
+ if cfg.Owner != "" {
+ cfg.Capacity = dynamicCapacity // dynamic lanes default small
+ }
+ }
+ regMu.Lock()
+ defer regMu.Unlock()
+ if _, exists := registry[cfg.Name]; exists {
+ return nil, fmt.Errorf("lane %s already exists", cfg.Name)
+ }
+ l := &laneImpl{cfg: cfg, ch: make(chan *event.Event, cfg.Capacity), quit: make(chan struct{})}
+ registry[cfg.Name] = l
+ return l, nil
+}
+
+// GetOrCreate returns the named OPEN lane, creating it lazily on first Emit
+// (dynamic pipelines; owner tags it for cascade cleanup). A registered but
+// CLOSED instance is replaced with a fresh one built from cfg — lane
+// consumers close their lane on task exit, and without replacement the
+// registry would hold a permanently-unusable lane that wedges every
+// producer (emit fails "closed" forever, consumer restarts cannot recover).
+func GetOrCreate(cfg LaneConfig) Lane {
+ regMu.RLock()
+ l, ok := registry[cfg.Name]
+ closed := ok && l.Snapshot().Closed
+ regMu.RUnlock()
+ if ok && !closed {
+ return l
+ }
+ if closed {
+ regMu.Lock()
+ // re-check under the write lock: a concurrent replacer may have won
+ if cur, still := registry[cfg.Name]; still && !cur.Snapshot().Closed {
+ regMu.Unlock()
+ return cur
+ }
+ delete(registry, cfg.Name)
+ regMu.Unlock()
+ }
+ l, err := Create(cfg)
+ if err != nil {
+ regMu.RLock()
+ existing := registry[cfg.Name]
+ regMu.RUnlock()
+ return existing // lost the race: the winner's instance wins
+ }
+ return l
+}
+
+// Get returns the named OPEN lane; nil when absent or closed. A closed lane
+// accepts no emits and no subscribers — callers want a usable channel, and
+// GetOrCreate is the re-creation path.
+func Get(name string) Lane {
+ regMu.RLock()
+ defer regMu.RUnlock()
+ l := registry[name]
+ if l == nil || l.Snapshot().Closed {
+ return nil
+ }
+ return l
+}
+
+// Delete removes a lane; refuses while records are in flight unless force.
+func Delete(name string, force bool) error {
+ regMu.Lock()
+ l, ok := registry[name]
+ if !ok {
+ return nil
+ }
+ regMu.Unlock()
+ if !force && l.Depth() > 0 {
+ return fmt.Errorf("lane %s has %d in-flight records (force to drop)", name, l.Depth())
+ }
+ l.Close()
+ regMu.Lock()
+ delete(registry, name)
+ regMu.Unlock()
+ return nil
+}
+
+// DeleteOwner cascades dynamic-lane cleanup (stream delete → lanes die).
+func DeleteOwner(owner string) int {
+ regMu.RLock()
+ names := make([]string, 0)
+ for n, l := range registry {
+ if l.Config().Owner == owner && owner != "" {
+ names = append(names, n)
+ }
+ }
+ regMu.RUnlock()
+ for _, n := range names {
+ _ = Delete(n, true)
+ }
+ return len(names)
+}
+
+// List snapshots all lanes for observability.
+func List() []Stats {
+ regMu.RLock()
+ defer regMu.RUnlock()
+ out := make([]Stats, 0, len(registry))
+ for _, l := range registry {
+ out = append(out, l.Snapshot())
+ }
+ return out
+}
diff --git a/core/lane/lane_test.go b/core/lane/lane_test.go
new file mode 100644
index 000000000..d1d009189
--- /dev/null
+++ b/core/lane/lane_test.go
@@ -0,0 +1,289 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package lane
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+)
+
+func rec(msg string) *event.Event {
+ return &event.Event{Fields: util.MapStr{"message": msg}}
+}
+
+func mustCreate(t *testing.T, cfg LaneConfig) Lane {
+ t.Helper()
+ l, err := Create(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return l
+}
+
+// TestRegistry_ClosedLaneSelfHeals pins the wedge fix: a lane closed by its
+// consumer's task exit must not linger as a permanently-unusable registry
+// entry — Get skips it and GetOrCreate replaces it with a fresh instance.
+func TestRegistry_ClosedLaneSelfHeals(t *testing.T) {
+ l1 := mustCreate(t, LaneConfig{Name: "heal", Capacity: 4})
+ if err := l1.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if Get("heal") != nil {
+ t.Fatal("Get must skip a closed lane")
+ }
+ l2 := GetOrCreate(LaneConfig{Name: "heal", Capacity: 4})
+ if l2 == nil {
+ t.Fatal("GetOrCreate must replace the closed lane")
+ }
+ if l2 == l1 {
+ t.Fatal("GetOrCreate returned the closed instance")
+ }
+ if err := l2.Emit(rec("fresh")); err != nil {
+ t.Fatalf("emit on replaced lane: %v", err)
+ }
+ if err := l1.Emit(rec("stale")); err == nil {
+ t.Fatal("stale handle must keep erroring (drained instance)")
+ }
+ got := make(chan *event.Event, 1)
+ l2.Subscribe(func(r *event.Event) { got <- r })
+ select {
+ case r := <-got:
+ if r.Fields["message"] != "fresh" {
+ t.Fatalf("got %v", r)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("replaced lane did not deliver")
+ }
+ if err := Delete("heal", true); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestLane_BlockBackpressure(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "bp", Capacity: 2})
+ // no subscriber yet: fills up deterministically
+ for i := 0; i < 2; i++ {
+ if err := l.Emit(rec("m")); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if l.TryEmit(rec("overflow")) {
+ t.Fatal("TryEmit should fail when full")
+ }
+ delivered := make(chan struct{}, 4)
+ cancel := l.Subscribe(func(r *event.Event) { delivered <- struct{}{} })
+ defer cancel()
+ // drain one → room appears
+ <-delivered
+ deadline := time.Now().Add(2 * time.Second)
+ for !l.TryEmit(rec("next")) && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ <-delivered
+ s := l.Snapshot()
+ if s.Emitted != 3 || s.Drops != 0 {
+ t.Fatalf("stats: %+v", s)
+ }
+}
+
+func TestLane_DropPolicy(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "dp", Capacity: 1, Overflow: "drop"})
+ if err := l.Emit(rec("a")); err != nil {
+ t.Fatal(err)
+ }
+ if err := l.Emit(rec("b")); err != nil {
+ t.Fatal(err) // drop policy: not an error
+ }
+ s := l.Snapshot()
+ if s.Drops != 1 || s.Depth != 1 {
+ t.Fatalf("drop stats: %+v", s)
+ }
+}
+
+func TestLane_BlockingEmitUnblocksOnDrain(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "blk", Capacity: 1})
+ if err := l.Emit(rec("first")); err != nil {
+ t.Fatal(err)
+ }
+ done := make(chan error, 1)
+ go func() { done <- l.Emit(rec("second")) }()
+ select {
+ case err := <-done:
+ t.Fatalf("second Emit should block: %v", err)
+ case <-time.After(80 * time.Millisecond):
+ }
+ cancel := l.Subscribe(func(r *event.Event) {})
+ defer cancel()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Emit never unblocked after drain")
+ }
+ if s := l.Snapshot(); s.BlockedWaits == 0 {
+ t.Fatal("blocked_waits counter not incremented")
+ }
+}
+
+func TestRegistry_Lifecycle(t *testing.T) {
+ if _, err := Create(LaneConfig{Name: "dup"}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Create(LaneConfig{Name: "dup"}); err == nil {
+ t.Fatal("duplicate Create must error")
+ }
+ // GetOrCreate: same instance
+ if GetOrCreate(LaneConfig{Name: "dup"}) != Get("dup") {
+ t.Fatal("GetOrCreate returned a different instance")
+ }
+ // dynamic capacity default
+ dyn := GetOrCreate(LaneConfig{Name: "dyn", Owner: "stream:x"})
+ if dyn.Config().Capacity != dynamicCapacity {
+ t.Fatalf("dynamic capacity = %d", dyn.Config().Capacity)
+ }
+ // delete refuses with depth
+ if err := dyn.Emit(rec("in-flight")); err != nil {
+ t.Fatal(err)
+ }
+ if err := Delete("dyn", false); err == nil {
+ t.Fatal("Delete must refuse with in-flight records")
+ }
+ if err := Delete("dyn", true); err != nil {
+ t.Fatal(err)
+ }
+ if Get("dyn") != nil {
+ t.Fatal("deleted lane still registered")
+ }
+ // owner cascade
+ GetOrCreate(LaneConfig{Name: "s1", Owner: "stream:y"})
+ GetOrCreate(LaneConfig{Name: "s2", Owner: "stream:y"})
+ if n := DeleteOwner("stream:y"); n != 2 {
+ t.Fatalf("cascade deleted %d, want 2", n)
+ }
+ _ = Delete("dup", true)
+}
+
+func TestLane_CloseSemantics(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "cl"})
+ if err := l.Emit(rec("x")); err != nil {
+ t.Fatal(err)
+ }
+ got := make(chan *event.Event, 1)
+ cancel := l.Subscribe(func(r *event.Event) { got <- r })
+ defer cancel()
+ if err := l.Close(); err != nil {
+ t.Fatal(err)
+ }
+ // drain delivers the in-flight record
+ select {
+ case r := <-got:
+ if r == nil {
+ t.Fatal("nil record")
+ }
+ case <-time.After(time.Second):
+ t.Fatal("in-flight record lost on close")
+ }
+ if err := l.Emit(rec("after")); err == nil {
+ t.Fatal("Emit after close must error")
+ }
+}
+
+// EmitAck receipts settle when the subscriber finishes processing the
+// record (handler return = settlement).
+func TestLane_EmitAckSettlesOnProcessing(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "ack", Capacity: 4})
+ processed := make(chan struct{})
+ release := make(chan struct{})
+ cancel := l.Subscribe(func(r *event.Event) {
+ <-release // hold processing
+ close(processed)
+ })
+ defer cancel()
+
+ r := l.EmitAck(rec("payload"))
+ // receipt must stay unsettled while processing is held
+ ctx, cancelW := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancelW()
+ if err := r.Wait(ctx); err == nil {
+ t.Fatal("receipt settled before processing finished")
+ }
+ close(release)
+ <-processed
+ ctx2, cancelW2 := context.WithTimeout(context.Background(), time.Second)
+ defer cancelW2()
+ if err := r.Wait(ctx2); err != nil {
+ t.Fatalf("receipt not settled after processing: %v", err)
+ }
+}
+
+// close with an in-flight acked record settles the receipt with an error
+func TestLane_CloseSettlesPendingWithError(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "ackcl", Capacity: 1})
+ // no subscriber: record stays in flight
+ r := l.EmitAck(rec("stuck"))
+ if err := l.Close(); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancelW := context.WithTimeout(context.Background(), time.Second)
+ defer cancelW()
+ if err := r.Wait(ctx); err == nil {
+ t.Fatal("pending receipt must settle with error on close")
+ }
+}
+
+// batch mode: receipts settle only after the BATCH completes (not per
+// record), and batches flush on size or idle timeout.
+func TestLane_SubscribeBatchReceiptSettlesAfterBatch(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "batch", Capacity: 8})
+ processing := make(chan struct{}, 4)
+ release := make(chan struct{})
+ cancel := l.SubscribeBatch(2, 50*time.Millisecond, func(recs []*event.Event) {
+ processing <- struct{}{}
+ <-release
+ })
+ defer cancel()
+
+ r1 := l.EmitAck(rec("a"))
+ r2 := l.EmitAck(rec("b"))
+ // batch of 2 formed; processing held — both receipts must be unsettled
+ <-processing
+ ctx, cancelW := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancelW()
+ if r1.Wait(ctx) == nil || r2.Wait(ctx) == nil {
+ t.Fatal("receipts settled before batch completed")
+ }
+ close(release)
+ ctx2, cancelW2 := context.WithTimeout(context.Background(), time.Second)
+ defer cancelW2()
+ if r1.Wait(ctx2) != nil || r2.Wait(ctx2) != nil {
+ t.Fatal("receipts not settled after batch completed")
+ }
+}
+
+// idle timeout flush: a single record flushes after flushInterval without
+// reaching max.
+func TestLane_SubscribeBatchIdleFlush(t *testing.T) {
+ l := mustCreate(t, LaneConfig{Name: "idle", Capacity: 4})
+ got := make(chan int, 4)
+ cancel := l.SubscribeBatch(10, 30*time.Millisecond, func(recs []*event.Event) {
+ got <- len(recs)
+ })
+ defer cancel()
+ _ = l.Emit(rec("solo"))
+ select {
+ case n := <-got:
+ if n != 1 {
+ t.Fatalf("flushed %d records, want 1", n)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("idle flush never fired")
+ }
+}
diff --git a/core/model/instance.go b/core/model/instance.go
index 0264ddcd6..56b3da17b 100644
--- a/core/model/instance.go
+++ b/core/model/instance.go
@@ -65,6 +65,11 @@ type Instance struct {
// registration/heartbeat upserts preserve the stored value.
Groups []string `json:"groups,omitempty" elastic_mapping:"groups:{type:keyword}"`
+ // Alias is a SERVER-OWNED display name (UI-managed, e.g. "edge-gateway-prod"):
+ // shown in the management UI in place of the auto-generated instance
+ // name. Like Groups, heartbeat upserts preserve the stored value.
+ Alias string `json:"alias,omitempty" elastic_mapping:"alias:{type:keyword,copy_to:search_text}"`
+
//user can pass
Description string `json:"description,omitempty" config:"description" elastic_mapping:"description:{type:keyword}"`
@@ -81,6 +86,17 @@ type Instance struct {
Services []ServiceInfo `json:"services,omitempty" elastic_mapping:"services: { type: object }"`
Status string `json:"status,omitempty" elastic_mapping:"status: { type: keyword, copy_to:search_text }"`
+ // RuntimePipelines is the instance's live pipeline state, reported via
+ // configs _sync and consumed by the deployment reconciler. Stored as
+ // a JSON string (map[string]PipelineRuntimeState).
+ RuntimePipelines string `json:"runtime_pipelines,omitempty" elastic_mapping:"runtime_pipelines:{type:text}"`
+
+ // Cordoned mirrors the instance-local maintenance flag (#121): running
+ // tasks drain but nothing starts. Reported with every _sync (#128) so
+ // the control plane can surface it; placement deliberately keeps
+ // flowing configs to cordoned instances (recall would strand tasks).
+ Cordoned bool `json:"cordoned,omitempty" elastic_mapping:"cordoned:{type:boolean}"`
+
//SearchText string `json:"search_text,omitempty" elastic_mapping:"search_text:{type:text,index_prefixes:{},index_phrases:true, analyzer:suggest_text_search }"`
}
diff --git a/core/orm/query.go b/core/orm/query.go
index 28001cd80..5ce499a8c 100644
--- a/core/orm/query.go
+++ b/core/orm/query.go
@@ -120,6 +120,11 @@ type QueryBuilder struct {
allowRequestBodyBytes bool
requestBodyBytes []byte
Aggs map[string]Aggregation
+
+ // body-level search parameters (min_score, track_total_hits, ...) —
+ // rendered at the DSL body root by the Elasticsearch builder, see
+ // BodyParameter
+ bodyParams map[string]interface{}
}
func NewQuery() *QueryBuilder {
@@ -209,6 +214,24 @@ func (q *QueryBuilder) Parameter(key param.ParaKey, val interface{}) *QueryBuild
return q
}
+// BodyParameter records a top-level search-body parameter (min_score,
+// track_total_hits, ...). Unlike Parameter, whose keys feed the bool clause,
+// these are rendered by the Elasticsearch DSL builder at the body root next
+// to query/from/size, overriding raw request-body keys of the same name.
+func (q *QueryBuilder) BodyParameter(key string, val interface{}) *QueryBuilder {
+ if q.bodyParams == nil {
+ q.bodyParams = map[string]interface{}{}
+ }
+ q.bodyParams[key] = val
+ return q
+}
+
+// BodyParametersVal returns the recorded body-level parameters (nil when
+// none were set).
+func (q *QueryBuilder) BodyParametersVal() map[string]interface{} {
+ return q.bodyParams
+}
+
func (q *QueryBuilder) DefaultQueryField(field ...string) *QueryBuilder {
q.defaultQueryFields = append(q.defaultQueryFields, field...)
return q
diff --git a/core/orm/registry.go b/core/orm/registry.go
index 292b42e36..4d13cdf5b 100644
--- a/core/orm/registry.go
+++ b/core/orm/registry.go
@@ -35,6 +35,12 @@ func InitSchema() error {
var handler ORM
+// IsHandlerRegistered reports whether an ORM backend is available —
+// lightweight instances (agents without sqlite/ES) have none.
+func IsHandlerRegistered() bool {
+ return handler != nil
+}
+
func getHandler() ORM {
if handler == nil {
panic(errors.New("ORM handler is not registered"))
diff --git a/core/otel/bench_test.go b/core/otel/bench_test.go
index 728e67fb7..cd04e0d6e 100644
--- a/core/otel/bench_test.go
+++ b/core/otel/bench_test.go
@@ -1,16 +1,16 @@
package otel
import (
- "testing"
- "time"
"infini.sh/framework/core/event"
"infini.sh/framework/core/util"
+ "testing"
+ "time"
)
func benchEvent() *event.Event {
return &event.Event{
Timestamp: time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC),
- Fields: util.MapStr{"message": "ERROR payment-svc: connection refused to db-1:5432, retry 3", "log_level": "ERROR", "service_name": "payment-svc"},
+ Fields: util.MapStr{"message": "ERROR payment-svc: connection refused to db-1:5432, retry 3", "log_level": "ERROR", "service_name": "payment-svc"},
}
}
diff --git a/core/otel/convert.go b/core/otel/convert.go
index c3e8c96ec..2620de85c 100644
--- a/core/otel/convert.go
+++ b/core/otel/convert.go
@@ -61,6 +61,7 @@ func FromEvent(e *event.Event) *LogRecord {
if v, ok := resourceMap(e.Meta); ok {
r.Resource = v
}
+ r.Resource = agentToResource(e.Agent, r.Resource)
r.SeverityText, _ = r.Attributes[FieldLogLevel].(string)
r.Body = r.Attributes[FieldMessage]
@@ -120,9 +121,75 @@ func (r *LogRecord) ToEvent() *event.Event {
}
e.Meta[MetaResourceKey] = r.Resource.Clone()
}
+ if e.Agent == nil {
+ e.Agent = agentFromResource(r.Resource)
+ }
return e
}
+// agentToResource merges the collecting agent's identity into the record's
+// Resource attributes so it survives the OTLP wire — the LogRecord model has
+// no other carrier for it. Existing keys win: a resource that already
+// describes its host (e.g. written by an upstream processor) is not
+// overwritten, which also keeps cascaded exports idempotent. The merge is
+// deterministic, so mutating a referenced resource map in place is safe.
+func agentToResource(a *event.AgentMeta, res util.MapStr) util.MapStr {
+ if a == nil {
+ return res
+ }
+ if res == nil {
+ res = util.MapStr{}
+ }
+ setIfAbsent(res, ResourceHostName, a.Hostname)
+ setIfAbsent(res, ResourceHostIP, a.MajorIP)
+ setIfAbsent(res, ResourceAgentID, a.AgentID)
+ if len(a.Tags) > 0 {
+ setIfAbsent(res, ResourceAgentTags, a.Tags)
+ }
+ return res
+}
+
+// agentFromResource rebuilds the collecting agent identity from Resource
+// attributes written by agentToResource — the OTLP intake path has no other
+// source for it. A resource without any agent-describing key yields nil.
+func agentFromResource(res util.MapStr) *event.AgentMeta {
+ if res == nil {
+ return nil
+ }
+ a := &event.AgentMeta{}
+ a.AgentID, _ = res[ResourceAgentID].(string)
+ a.Hostname, _ = res[ResourceHostName].(string)
+ a.MajorIP, _ = res[ResourceHostIP].(string)
+ switch v := res[ResourceAgentTags].(type) {
+ case []string:
+ a.Tags = v
+ case []interface{}: // OTLP wire round-trip decodes arrays as []interface{}
+ tags := make([]string, 0, len(v))
+ for _, item := range v {
+ if s, ok := item.(string); ok {
+ tags = append(tags, s)
+ }
+ }
+ if len(tags) > 0 {
+ a.Tags = tags
+ }
+ }
+ if a.AgentID == "" && a.Hostname == "" && a.MajorIP == "" && len(a.Tags) == 0 {
+ return nil
+ }
+ return a
+}
+
+func setIfAbsent(m util.MapStr, k string, v interface{}) {
+ if _, ok := m[k]; ok {
+ return
+ }
+ if s, ok := v.(string); ok && s == "" {
+ return
+ }
+ m[k] = v
+}
+
func resourceMap(meta util.MapStr) (util.MapStr, bool) {
if meta == nil {
return nil, false
diff --git a/core/otel/log_record.go b/core/otel/log_record.go
index f1928c35c..477d10e20 100644
--- a/core/otel/log_record.go
+++ b/core/otel/log_record.go
@@ -30,7 +30,9 @@
// https://opentelemetry.io/docs/specs/otel/logs/data-model/
//
// Structural standard : OTel LogRecord (top-level typed fields + Resource
-// /Attributes attribute collections).
+//
+// /Attributes attribute collections).
+//
// Field naming standard: lowercase snake_case keys ("log_level",
// "host_name", ...), matching the house convention. The
// field_standardize processor (enterprise) can switch attribute naming to
@@ -54,7 +56,7 @@ import (
const (
FieldTimestamp = "timestamp"
FieldObservedTimestamp = "observed_timestamp"
- FieldLogLevel = "log_level" // OTel SeverityText, original string
+ FieldLogLevel = "log_level" // OTel SeverityText, original string
FieldSeverityNumber = "severity_number" // OTel SeverityNumber, 1-24
FieldMessage = "message" // OTel Body
FieldEventName = "event_name"
@@ -70,15 +72,17 @@ const (
// Common Resource attribute keys (stable per source, snake_case).
const (
- ResourceHostName = "host_name"
- ResourceHostIP = "host_ip"
- ResourceServiceName = "service_name"
- ResourceServiceVersion = "service_version"
- ResourceProcessPID = "process_pid"
- ResourceProcessName = "process_name"
- ResourceCloudProvider = "cloud_provider"
- ResourceCloudRegion = "cloud_region"
- ResourceCloudAccountID = "cloud_account_id"
+ ResourceHostName = "host_name"
+ ResourceHostIP = "host_ip"
+ ResourceAgentID = "agent_id"
+ ResourceAgentTags = "agent_tags"
+ ResourceServiceName = "service_name"
+ ResourceServiceVersion = "service_version"
+ ResourceProcessPID = "process_pid"
+ ResourceProcessName = "process_name"
+ ResourceCloudProvider = "cloud_provider"
+ ResourceCloudRegion = "cloud_region"
+ ResourceCloudAccountID = "cloud_account_id"
)
// OTel SeverityNumber scale (see the Logs Data Model): 1-24, the 9 "well
diff --git a/core/otel/otel_test.go b/core/otel/otel_test.go
index 352455466..83d1b3fef 100644
--- a/core/otel/otel_test.go
+++ b/core/otel/otel_test.go
@@ -69,6 +69,55 @@ func TestEventRoundTrip(t *testing.T) {
}
}
+// The agent identity must survive the OTLP LogRecord projection: FromEvent
+// merges it into Resource attributes, ToEvent restores it when the inbound
+// event has none. A resource key set upstream must not be overwritten, and
+// an event that already carries its Agent must keep it as-is.
+func TestAgentIdentitySurvivesLogRecord(t *testing.T) {
+ e := &event.Event{
+ Timestamp: time.Now().UTC(),
+ Agent: &event.AgentMeta{
+ AgentID: "agent-7",
+ Hostname: "web-1",
+ MajorIP: "10.0.0.5",
+ Tags: []string{"prod", "edge"},
+ },
+ Fields: util.MapStr{FieldMessage: "m"},
+ }
+
+ rec := FromEvent(e)
+ if rec.Resource[ResourceAgentID] != "agent-7" || rec.Resource[ResourceHostName] != "web-1" {
+ t.Fatalf("agent identity not merged into resource: %v", rec.Resource)
+ }
+ if rec.Resource[ResourceHostIP] != "10.0.0.5" {
+ t.Fatalf("host_ip = %v", rec.Resource)
+ }
+
+ back := rec.ToEvent()
+ if back.Agent == nil || back.Agent.AgentID != "agent-7" || back.Agent.Hostname != "web-1" ||
+ back.Agent.MajorIP != "10.0.0.5" {
+ t.Fatalf("agent not restored: %v", back.Agent)
+ }
+
+ // upstream resource keys win over the agent merge
+ e2 := &event.Event{
+ Agent: &event.AgentMeta{AgentID: "agent-7", Hostname: "web-1"},
+ Meta: util.MapStr{MetaResourceKey: util.MapStr{ResourceHostName: "already-set"}},
+ Fields: util.MapStr{FieldMessage: "m"},
+ }
+ rec2 := FromEvent(e2)
+ if rec2.Resource[ResourceHostName] != "already-set" {
+ t.Fatalf("existing resource keys must not be overwritten: %v", rec2.Resource)
+ }
+
+ // no agent anywhere: nothing restored, no empty AgentMeta injected
+ e3 := &event.Event{Fields: util.MapStr{FieldMessage: "m"}}
+ back3 := FromEvent(e3).ToEvent()
+ if back3.Agent != nil {
+ t.Fatalf("agent must stay nil without identity, got %v", back3.Agent)
+ }
+}
+
func TestEnvelopeRoundTrip(t *testing.T) {
ts := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC)
e := &event.Event{
diff --git a/core/pipeline/ackset.go b/core/pipeline/ackset.go
new file mode 100644
index 000000000..c8171a62f
--- /dev/null
+++ b/core/pipeline/ackset.go
@@ -0,0 +1,270 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "time"
+
+ "infini.sh/framework/core/param"
+)
+
+// ErrAckTimeout marks a commit-gate wait that expired with unsettled
+// receipts — the caller must NOT commit the offset (redelivery follows).
+var ErrAckTimeout = errors.New("ack set: unsettled receipts at timeout")
+
+// Receipt is one pending delivery confirmation: resolved by whoever
+// finishes processing the emitted record (settlement = output/dead-letter
+// push or explicit end-of-chain).
+type Receipt interface {
+ Wait(ctx context.Context) error
+}
+
+type receiptImpl struct{ done chan error }
+
+func newReceipt() *receiptImpl { return &receiptImpl{done: make(chan error, 1)} }
+func (r *receiptImpl) settle(err error) {
+ select {
+ case r.done <- err:
+ default:
+ }
+}
+func (r *receiptImpl) Wait(ctx context.Context) error {
+ select {
+ case err := <-r.done:
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// ackSetKey carries the AckSet on the emitting chain's context.
+const ackSetKey param.ParaKey = "__ack_set"
+
+// AckSet is the commit gate: lane outputs attach receipts; the queue
+// consumer waits for all of them before committing its offset. Unsettled
+// receipts (crash downstream, stuck chain) surface as ErrAckTimeout —
+// the offset stays put and the batch replays (at-least-once across the
+// lane hop; duplicates are absorbed by idempotent sinks).
+type AckSet struct {
+ mu sync.Mutex
+ receipts []*receiptImpl
+ external []Receipt
+}
+
+// AttachAckSet installs a fresh set on the context (idempotent).
+func AttachAckSet(ctx *Context) *AckSet {
+ if ctx == nil {
+ return nil
+ }
+ if existing, ok := ctx.Get(ackSetKey).(*AckSet); ok {
+ return existing
+ }
+ s := &AckSet{}
+ ctx.Set(ackSetKey, s)
+ return s
+}
+
+// AckSetOf returns the context's set (nil when none attached).
+func AckSetOf(ctx *Context) *AckSet {
+ if ctx == nil {
+ return nil
+ }
+ s, _ := ctx.Get(ackSetKey).(*AckSet)
+ return s
+}
+
+// Add registers a receipt with this set.
+func (s *AckSet) Add(r *receiptImpl) {
+ if s == nil || r == nil {
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.receipts = append(s.receipts, r)
+}
+
+// AddReceipt registers any Receipt implementation (lane receipts et al).
+func (s *AckSet) AddReceipt(r Receipt) {
+ if s == nil || r == nil {
+ return
+ }
+ if own, ok := r.(*receiptImpl); ok {
+ s.Add(own)
+ return
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.external = append(s.external, r)
+}
+
+// Pending reports the unsettled receipt count.
+func (s *AckSet) Pending() int {
+ if s == nil {
+ return 0
+ }
+ n := 0
+ for _, r := range s.snapshot() {
+ select {
+ case <-r.done:
+ default:
+ n++
+ }
+ }
+ for _, r := range s.externalList() {
+ if !receiptSettled(r) {
+ n++
+ }
+ }
+ return n
+}
+
+// Wait blocks until every receipt settles or the timeout expires.
+func (s *AckSet) Wait(ctx context.Context, timeout time.Duration) error {
+ if s == nil {
+ return nil
+ }
+ s.mu.Lock()
+ ext := append([]Receipt(nil), s.external...)
+ s.mu.Unlock()
+ for _, r := range ext {
+ if err := r.Wait(ctx); err != nil {
+ return err
+ }
+ s.mu.Lock()
+ for i, e := range s.external {
+ if e == r {
+ s.external = append(s.external[:i], s.external[i+1:]...)
+ break
+ }
+ }
+ s.mu.Unlock()
+ }
+ deadline := time.After(timeout)
+ for {
+ pending := s.Pending()
+ if pending == 0 {
+ return nil
+ }
+ select {
+ case <-deadline:
+ return ErrAckTimeout
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(time.Millisecond):
+ }
+ }
+}
+
+func (s *AckSet) externalList() []Receipt {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]Receipt(nil), s.external...)
+}
+
+func (s *AckSet) snapshot() []*receiptImpl {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ out := make([]*receiptImpl, len(s.receipts))
+ copy(out, s.receipts)
+ return out
+}
+
+// receiptSettled reports whether an external Receipt has settled when
+// the implementation exposes its signal (lane receipts do); unresolved
+// implementations count as pending.
+func receiptSettled(r Receipt) bool {
+ type doner interface{ Done() <-chan error }
+ if d, ok := r.(doner); ok {
+ select {
+ case <-d.Done():
+ return true
+ default:
+ return false
+ }
+ }
+ return false
+}
+
+// ─── dry-run interception ────────────────────────────────────────────────────
+
+// DryRunWrite is one captured side effect (queue/lane/attachment write
+// intercepted during a dry run — never delivered).
+type DryRunWrite struct {
+ Kind string // queue | lane | attachment (host-specific labels welcome)
+ Target string
+ Records int
+ Preview string
+}
+
+// dryRunCtxKey rides the Go context chain: processor helpers derive plain
+// contexts from ctx.Context and never see the pipeline wrapper's parameter
+// store, so the state must travel with the embedded chain to stay visible
+// under orm/attachment layers.
+type dryRunCtxKey struct{}
+
+// EnableDryRun marks the context as a dry run; when capture is true,
+// output processors append intercepted writes to sink instead of
+// delivering them.
+func EnableDryRun(ctx *Context, sink *[]DryRunWrite, capture bool) {
+ if ctx == nil {
+ return
+ }
+ st := &dryRunState{sink: sink, capture: capture}
+ ctx.Context = context.WithValue(ctx.Context, dryRunCtxKey{}, st)
+}
+
+type dryRunState struct {
+ sink *[]DryRunWrite
+ capture bool
+}
+
+func dryRunStateFrom(ctx context.Context) *dryRunState {
+ if ctx == nil {
+ return nil
+ }
+ st, _ := ctx.Value(dryRunCtxKey{}).(*dryRunState)
+ return st
+}
+
+// InDryRun reports whether this context runs under a dry-run replay
+// (EnableDryRun was called), regardless of capture mode.
+func InDryRun(ctx *Context) bool {
+ if ctx == nil {
+ return false
+ }
+ return dryRunStateFrom(ctx.Context) != nil
+}
+
+// InDryRunCtx is the context-chain variant for code holding a derived
+// plain context (below processors that re-wrap the parent).
+func InDryRunCtx(ctx context.Context) bool {
+ return dryRunStateFrom(ctx) != nil
+}
+
+// DryRunCapture returns the sink when this context runs in capturing
+// dry-run mode (nil = deliver for real).
+func DryRunCapture(ctx *Context) *[]DryRunWrite {
+ if ctx == nil {
+ return nil
+ }
+ return dryRunSinkFrom(ctx.Context)
+}
+
+// DryRunCaptureCtx is the context-chain variant for code deep under a
+// pipeline (orm/attachment layers holding derived contexts).
+func DryRunCaptureCtx(ctx context.Context) *[]DryRunWrite {
+ return dryRunSinkFrom(ctx)
+}
+
+func dryRunSinkFrom(ctx context.Context) *[]DryRunWrite {
+ st := dryRunStateFrom(ctx)
+ if st == nil || !st.capture {
+ return nil
+ }
+ return st.sink
+}
diff --git a/core/pipeline/case_processor.go b/core/pipeline/case_processor.go
new file mode 100644
index 000000000..97fa7bc0c
--- /dev/null
+++ b/core/pipeline/case_processor.go
@@ -0,0 +1,239 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello@infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package pipeline
+
+import (
+ "errors"
+ "fmt"
+ "reflect"
+ "strings"
+
+ log "github.com/cihub/seelog"
+ "infini.sh/framework/core/conditions"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/util"
+)
+
+// caseBranchConfig is one arm of a case block as written in YAML: an
+// optional enumerated value set (matched against the case block's field),
+// an optional full condition (`when`, any operator the if/then/else block
+// accepts), and the chain to run on match. At least one of values/when is
+// required; specifying both means AND.
+type caseBranchConfig struct {
+ Values []interface{} `config:"values"`
+ When conditions.Config `config:"when"`
+ Then *config.Config `config:"then"`
+}
+
+type caseConfig struct {
+ Field string `config:"field"`
+ Branches []caseBranchConfig `config:"branches"`
+ Default *config.Config `config:"default"`
+}
+
+// caseBranch is the compiled form of one arm.
+type caseBranch struct {
+ values []interface{}
+ hasValues bool
+ cond conditions.Condition // nil = values-only arm
+ then *Processors
+}
+
+// CaseProcessor is the multi-way dispatch the if/then/else ladder becomes
+// unwieldy for: ordered branches, FIRST match wins (exactly one arm runs),
+// an optional default chain when nothing matches.
+//
+// - case:
+// field: level
+// branches:
+// - values: [error, fatal]
+// then: [ ... ]
+// - when:
+// prefix: {level: wa}
+// then: [ ... ]
+// default: [ ... ]
+type CaseProcessor struct {
+ field string // "" = condition-only arms
+ branches []caseBranch
+ def *Processors
+}
+
+// NewCaseProcessor compiles a case step config at the default batch scope.
+func NewCaseProcessor(cfg *config.Config) (*CaseProcessor, error) {
+ return NewCaseProcessorWithScope(cfg, ScopeInfo{Kind: ScopeBatch})
+}
+
+// NewCaseProcessorWithScope compiles the arm chains under the ENCLOSING
+// scope — the same inheritance if/then/else applies: inside a per-record
+// sub-chain (for_each, lane_consumer) record-scope contracts keep
+// applying and batch_only processors stay rejected in nested chains.
+func NewCaseProcessorWithScope(cfg *config.Config, scope ScopeInfo) (*CaseProcessor, error) {
+ sub, err := cfg.Child("case", -1)
+ if err != nil {
+ return nil, err
+ }
+
+ var c caseConfig
+ if err := sub.Unpack(&c); err != nil {
+ return nil, err
+ }
+ if len(c.Branches) == 0 {
+ return nil, errors.New("case requires at least one branch")
+ }
+
+ newChain := func(cc *config.Config) (*Processors, error) {
+ if cc == nil {
+ return nil, nil
+ }
+ if !cc.IsArray() {
+ return newPipelineWithScope([]*config.Config{cc}, scope, false)
+ }
+ var pc []*config.Config
+ if err := cc.Unpack(&pc); err != nil {
+ return nil, err
+ }
+ return newPipelineWithScope(pc, scope, false)
+ }
+
+ p := &CaseProcessor{field: c.Field}
+ for i, b := range c.Branches {
+ hasValues := len(b.Values) > 0
+ hasWhen := !reflect.DeepEqual(b.When, conditions.Config{})
+ if !hasValues && !hasWhen {
+ return nil, fmt.Errorf("case branch %d requires `values` or `when`", i+1)
+ }
+ if hasValues && c.Field == "" {
+ return nil, fmt.Errorf("case branch %d uses `values` but the case block declares no `field`", i+1)
+ }
+ arm := caseBranch{values: b.Values, hasValues: hasValues}
+ if hasWhen {
+ cond, err := conditions.NewCondition(&b.When)
+ if err != nil {
+ return nil, fmt.Errorf("case branch %d: %v", i+1, err)
+ }
+ arm.cond = cond
+ }
+ then, err := newChain(b.Then)
+ if err != nil {
+ return nil, fmt.Errorf("case branch %d: %v", i+1, err)
+ }
+ if then == nil {
+ return nil, fmt.Errorf("case branch %d requires a `then` chain", i+1)
+ }
+ arm.then = then
+ p.branches = append(p.branches, arm)
+ }
+
+ def, err := newChain(c.Default)
+ if err != nil {
+ return nil, fmt.Errorf("case default: %v", err)
+ }
+ p.def = def
+ return p, nil
+}
+
+// Process routes to the first matching arm. Value resolution matches
+// if/then/else: inside a per-record sub-chain arms see the current
+// record's fields; pipeline-level usage reads the context.
+func (p *CaseProcessor) Process(ctx *Context) error {
+ if !ctx.ShouldContinue() {
+ if global.Env().IsDebug {
+ log.Debugf("filter [%v] not continued", p.Name())
+ }
+ ctx.AddFlowProcess(p.Name() + "-skipped")
+ return nil
+ }
+
+ values := conditions.ValuesMap(ctx)
+ if rec, ok := CurrentRecord(ctx); ok {
+ values = rec
+ }
+
+ for i := range p.branches {
+ arm := &p.branches[i]
+ if arm.hasValues && !p.matchValues(values, arm.values) {
+ continue
+ }
+ if arm.cond != nil && !arm.cond.Check(values) {
+ continue
+ }
+ if global.Env().IsDebug {
+ log.Tracef("case -> branch %d", i+1)
+ }
+ ctx.AddFlowProcess(fmt.Sprintf("%s:%d", p.Name(), i+1))
+ return arm.then.Process(ctx)
+ }
+
+ if p.def != nil {
+ if global.Env().IsDebug {
+ log.Trace("case -> default")
+ }
+ ctx.AddFlowProcess(p.Name() + ":default")
+ return p.def.Process(ctx)
+ }
+ ctx.AddFlowProcess(p.Name() + ":no-match")
+ return nil
+}
+
+// matchValues mirrors the `in` condition's membership semantics (typed
+// comparison via util.ContainsAnyInAnyIntArray), so `field`+`values` arms
+// and `when: {in: ...}` behave identically.
+func (p *CaseProcessor) matchValues(values conditions.ValuesMap, want []interface{}) bool {
+ v, err := values.GetValue(p.field)
+ if err != nil {
+ return false
+ }
+ return util.ContainsAnyInAnyIntArray(v, want)
+}
+
+func (p *CaseProcessor) Name() string {
+ return "case"
+}
+
+func (p *CaseProcessor) String() string {
+ var sb strings.Builder
+ sb.WriteString("case ")
+ for i, arm := range p.branches {
+ if i > 0 {
+ sb.WriteString("; ")
+ }
+ if arm.hasValues {
+ sb.WriteString(fmt.Sprintf("%v in %v", p.field, arm.values))
+ }
+ if arm.cond != nil {
+ if arm.hasValues {
+ sb.WriteString(" and ")
+ }
+ sb.WriteString(arm.cond.String())
+ }
+ sb.WriteString(" then ")
+ sb.WriteString(arm.then.Name())
+ }
+ if p.def != nil {
+ sb.WriteString("; default ")
+ sb.WriteString(p.def.Name())
+ }
+ return sb.String()
+}
diff --git a/core/pipeline/case_processor_test.go b/core/pipeline/case_processor_test.go
new file mode 100644
index 000000000..fbdb05b71
--- /dev/null
+++ b/core/pipeline/case_processor_test.go
@@ -0,0 +1,302 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+)
+
+// buildCase compiles `body` as the content of a case step.
+func buildCase(t *testing.T, body util.MapStr) *CaseProcessor {
+ t.Helper()
+ c, err := config.NewConfigFrom(util.MapStr{"case": body})
+ if err != nil {
+ t.Fatal(err)
+ }
+ p, err := NewCaseProcessor(c)
+ if err != nil {
+ t.Fatalf("NewCaseProcessor: %v", err)
+ }
+ return p
+}
+
+func mark(mark string) util.MapStr {
+ return util.MapStr{"test_branch_marker": util.MapStr{"mark": mark}}
+}
+
+func levelSwitch(withDefault bool) util.MapStr {
+ body := util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"error", "fatal"}, "then": mark("sev1")},
+ util.MapStr{"values": []interface{}{"warn"}, "then": mark("sev2")},
+ },
+ }
+ if withDefault {
+ body["default"] = mark("sev3")
+ }
+ return body
+}
+
+// Enumerated switch on a record field: each value set routes to its arm.
+func TestCaseSwitchEnumRouting(t *testing.T) {
+ p := buildCase(t, levelSwitch(true))
+
+ for _, tc := range []struct {
+ level string
+ want string
+ }{
+ {"error", "sev1"},
+ {"fatal", "sev1"},
+ {"warn", "sev2"},
+ {"debug", "sev3"}, // no arm matches -> default
+ } {
+ ctx := newRecordContext(t, util.MapStr{"level": tc.level})
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, tc.want, branchOf(t, ctx), "level=%s", tc.level)
+ }
+}
+
+// No match and no default: the case block is a no-op, nothing crashed.
+func TestCaseNoMatchWithoutDefault(t *testing.T) {
+ p := buildCase(t, levelSwitch(false))
+ ctx := newRecordContext(t, util.MapStr{"level": "debug"})
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ rec, ok := CurrentRecord(ctx)
+ assert.True(t, ok)
+ assert.NotContains(t, rec.Fields, "branch")
+ assert.Contains(t, ctx.GetFlowProcess(), "case:no-match")
+}
+
+// A record missing the switch field never matches a values arm.
+func TestCaseFieldMissingFallsThrough(t *testing.T) {
+ p := buildCase(t, levelSwitch(true))
+ ctx := newRecordContext(t, util.MapStr{"other": 1})
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "sev3", branchOf(t, ctx))
+}
+
+// Arms accept full conditions (`when`) with any operator the if/then/else
+// block supports; evaluated against the current record inside sub-chains.
+func TestCaseWhenArmsOnRecord(t *testing.T) {
+ p := buildCase(t, util.MapStr{
+ "branches": []interface{}{
+ util.MapStr{"when": util.MapStr{"equals": util.MapStr{"svc": "api"}}, "then": mark("api")},
+ util.MapStr{"when": util.MapStr{"prefix": util.MapStr{"svc": "db"}}, "then": mark("db")},
+ },
+ "default": mark("other"),
+ })
+
+ api := newRecordContext(t, util.MapStr{"svc": "api"})
+ if err := p.Process(api); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "api", branchOf(t, api))
+
+ db := newRecordContext(t, util.MapStr{"svc": "db-replica"})
+ if err := p.Process(db); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "db", branchOf(t, db))
+
+ other := newRecordContext(t, util.MapStr{"svc": "cache"})
+ if err := p.Process(other); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "other", branchOf(t, other))
+}
+
+// First match wins even when several arms match.
+func TestCaseFirstMatchWins(t *testing.T) {
+ p := buildCase(t, util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"warn"}, "then": mark("first")},
+ util.MapStr{"values": []interface{}{"warn", "info"}, "then": mark("second")},
+ },
+ })
+ ctx := newRecordContext(t, util.MapStr{"level": "warn"})
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "first", branchOf(t, ctx))
+}
+
+// values + when on the same arm AND together; values alone on another arm
+// still matches the same record — the mixed-style ladder stays coherent.
+func TestCaseValuesAndWhenCombineAsAnd(t *testing.T) {
+ p := buildCase(t, util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{
+ "values": []interface{}{"error"},
+ "when": util.MapStr{"exists": []interface{}{"svc"}},
+ "then": mark("both"),
+ },
+ util.MapStr{"values": []interface{}{"error"}, "then": mark("values-only")},
+ },
+ })
+
+ withSvc := newRecordContext(t, util.MapStr{"level": "error", "svc": "api"})
+ if err := p.Process(withSvc); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "both", branchOf(t, withSvc))
+
+ withoutSvc := newRecordContext(t, util.MapStr{"level": "error"})
+ if err := p.Process(withoutSvc); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "values-only", branchOf(t, withoutSvc))
+}
+
+// Pipeline-level usage (no record bound) evaluates arms against the
+// context parameters, mirroring if/then/else.
+func TestCaseContextFallbackWithoutRecord(t *testing.T) {
+ p := buildCase(t, util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"error"}, "then": mark("err")},
+ },
+ "default": mark("rest"),
+ })
+
+ ctx := &Context{Context: t.Context()}
+ ctx.Set("level", "error")
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "err", util.ToString(ctx.Get("branch")))
+
+ ctx2 := &Context{Context: t.Context()}
+ ctx2.Set("level", "info")
+ if err := p.Process(ctx2); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "rest", util.ToString(ctx2.Get("branch")))
+}
+
+// A bound record takes precedence over context parameters carrying the
+// same key (same rule the if/then/else block applies).
+func TestCaseRecordTakesPrecedenceOverContext(t *testing.T) {
+ p := buildCase(t, util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"error"}, "then": mark("err")},
+ },
+ })
+ ctx := &Context{Context: t.Context()}
+ ctx.Set("level", "error")
+ ctx.Set(RecordContextKey, &event.Event{Fields: util.MapStr{"level": "info"}})
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Contains(t, ctx.GetFlowProcess(), "case:no-match")
+}
+
+// The step compiles through the generic chain compiler (the same path
+// ValidatePipelineConfig takes) and records its routing in the flow trace.
+func TestCaseStepCompilesViaNewPipeline(t *testing.T) {
+ step := util.MapStr{"case": util.MapStr{
+ "field": "level",
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"warn"}, "then": []interface{}{mark("sev2")}},
+ },
+ }}
+ c, err := config.NewConfigFrom(step)
+ if err != nil {
+ t.Fatal(err)
+ }
+ procs, err := NewPipeline([]*config.Config{c})
+ if err != nil {
+ t.Fatalf("NewPipeline: %v", err)
+ }
+ assert.Equal(t, 1, len(procs.List))
+ assert.Equal(t, "case", procs.List[0].Name())
+
+ ctx := newRecordContext(t, util.MapStr{"level": "warn"})
+ if err := procs.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, "sev2", branchOf(t, ctx))
+ assert.Contains(t, ctx.GetFlowProcess(), "case:1")
+ assert.Contains(t, ctx.GetFlowProcess(), "test_branch_marker")
+}
+
+func TestCaseCompileErrors(t *testing.T) {
+ mk := func(body util.MapStr) *config.Config {
+ c, err := config.NewConfigFrom(util.MapStr{"case": body})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c
+ }
+
+ for name, body := range map[string]util.MapStr{
+ "no branches": {
+ "field": "level",
+ },
+ "arm without values or when": {
+ "field": "level",
+ "branches": []interface{}{util.MapStr{"then": mark("x")}},
+ },
+ "values without field": {
+ "branches": []interface{}{
+ util.MapStr{"values": []interface{}{"error"}, "then": mark("x")},
+ },
+ },
+ "arm without then": {
+ "field": "level",
+ "branches": []interface{}{util.MapStr{"values": []interface{}{"error"}}},
+ },
+ "unknown condition operator": {
+ "branches": []interface{}{
+ util.MapStr{"when": util.MapStr{"nope": util.MapStr{"x": 1}}, "then": mark("x")},
+ },
+ },
+ } {
+ _, err := NewCaseProcessor(mk(body))
+ assert.Error(t, err, name)
+ }
+}
+
+// Arm chains inherit the enclosing scope: inside a record host (strict
+// scope) a batch_only processor nested in an arm is a compile error, the
+// same placement rule if/then/else branches follow.
+func TestCaseArmInheritsRecordScope(t *testing.T) {
+ step := util.MapStr{"case": util.MapStr{
+ "branches": []interface{}{
+ util.MapStr{"when": util.MapStr{"equals": util.MapStr{"x": 1}}, "then": []interface{}{
+ util.MapStr{"test_batch_only": util.MapStr{}},
+ }},
+ },
+ }}
+ c, err := config.NewConfigFrom([]util.MapStr{step})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var list []*config.Config
+ if err := c.Unpack(&list); err != nil {
+ t.Fatal(err)
+ }
+ _, err = NewPipelineWithScope(list,
+ ScopeInfo{Kind: ScopeRecord, Strict: true}, true)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "batch_only")
+}
diff --git a/core/pipeline/catalog_contract_test.go b/core/pipeline/catalog_contract_test.go
new file mode 100644
index 000000000..25dc47ae3
--- /dev/null
+++ b/core/pipeline/catalog_contract_test.go
@@ -0,0 +1,62 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/util"
+)
+
+func TestCatalog_CarriesContractFields(t *testing.T) {
+ // a registered-with-contract processor surfaces scope + record_contract
+ RegisterProcessorPlugin("test_catalog_contract", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_catalog_contract", RecordContract{Scope: ScopeRecord, Namespace: "document", Granularity: GranularityBatchCapable}}, nil
+ })
+ RegisterProcessorContract("test_catalog_contract", RecordContract{
+ Scope: ScopeRecord, Namespace: "document", Granularity: GranularityBatchCapable,
+ RequiredFields: []string{"document.content"},
+ })
+
+ cat := GetProcessorCatalog()
+ // category defaults to misc for un-categorized names; search everywhere
+ var entry util.MapStr
+ for _, group := range cat {
+ if m, ok := group.(util.MapStr); ok {
+ if e, ok := m["test_catalog_contract"].(util.MapStr); ok {
+ entry = e
+ }
+ }
+ }
+ if entry == nil {
+ t.Fatal("contract processor missing from catalog")
+ }
+ if entry["scope"] != "record" {
+ t.Fatalf("scope = %v", entry["scope"])
+ }
+ rc, ok := entry["record_contract"].(util.MapStr)
+ if !ok || rc["namespace"] != "document" || rc["granularity"] != "batch_capable" {
+ t.Fatalf("record_contract = %v", entry["record_contract"])
+ }
+
+ // unconstrained processors stay unconstrained (no contract registered,
+ // never constructed)
+ RegisterProcessorPlugin("test_catalog_plain", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_catalog_plain", RecordContract{}}, nil
+ })
+ cat2 := GetProcessorCatalog()
+ var plain util.MapStr
+ for _, group := range cat2 {
+ if m, ok := group.(util.MapStr); ok {
+ if e, ok := m["test_catalog_plain"].(util.MapStr); ok {
+ plain = e
+ }
+ }
+ }
+ if plain == nil || plain["scope"] != "unconstrained" {
+ t.Fatalf("plain entry = %v", plain)
+ }
+}
diff --git a/core/pipeline/codec.go b/core/pipeline/codec.go
new file mode 100644
index 000000000..7023100c3
--- /dev/null
+++ b/core/pipeline/codec.go
@@ -0,0 +1,148 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+ "sync"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/otel"
+)
+
+// RecordCodec converts between a queue message payload and the *event.Event
+// the per-record processor convention operates on. for_each is generic over
+// payloads: the default "otel" codec decodes the otel envelope (byte-
+// compatible with the agent LogEvent JSON); other payload formats register
+// their own codec and select it via the codec config field.
+//
+// Namespace reports the record family the codec produces ("document",
+// "attachment", "" for generic). Record contracts (D2) match on the
+// namespace, not the codec name, so v2 codecs producing the same family
+// keep dependent processors source-compatible.
+type RecordCodec interface {
+ Name() string
+ Namespace() string
+ Decode(data []byte) (*event.Event, error)
+ Encode(rec *event.Event) ([]byte, error)
+}
+
+// CodecLabel is the metadata label stamping the codec that decoded a
+// record, so output encoding round-trips symmetrically (hosts and codecs
+// stamp it at decode time).
+const CodecLabel = "_codec"
+
+// CodecHint returns the record's originating codec name ("" when unset).
+func CodecHint(rec *event.Event) string {
+ if rec == nil || rec.Metadata.Labels == nil {
+ return ""
+ }
+ v, _ := rec.Metadata.Labels[CodecLabel].(string)
+ return v
+}
+
+// CodecInfo describes a registered codec for discovery APIs
+// (GET /pipeline/codecs).
+type CodecInfo struct {
+ Name string `json:"name"`
+ Namespace string `json:"namespace"`
+ Version int64 `json:"version"`
+ Description string `json:"description,omitempty"`
+}
+
+type codecEntry struct {
+ codec RecordCodec
+ version int64
+}
+
+var (
+ codecsMu sync.RWMutex
+ codecs = map[string]*codecEntry{}
+)
+
+// RegisterRecordCodec adds a payload codec under its name. Intended for
+// package init() and runtime registration alike: unlike the historical
+// panic-on-duplicate behavior it returns an error so runtime callers can
+// react; init-time callers wanting the old hard-fail semantics use
+// MustRegisterRecordCodec.
+func RegisterRecordCodec(c RecordCodec) error {
+ codecsMu.Lock()
+ defer codecsMu.Unlock()
+ if _, exists := codecs[c.Name()]; exists {
+ return fmt.Errorf("record codec with same name already exists: %v", c.Name())
+ }
+ codecs[c.Name()] = &codecEntry{codec: c, version: 1}
+ return nil
+}
+
+// MustRegisterRecordCodec is the init-time variant: a duplicate name is a
+// programming error and panics, keeping the historical defense against two
+// packages registering the same codec.
+func MustRegisterRecordCodec(c RecordCodec) {
+ if err := RegisterRecordCodec(c); err != nil {
+ panic(err)
+ }
+}
+
+// RegisterRecordCodecOrReplace upgrades an existing registration under the
+// same name (version increments); registers fresh otherwise. For dev/test
+// iteration and controlled upgrades — running pipelines keep their compiled
+// reference (registration is dynamic, binding is compile-time), a pipeline
+// reload/rebuild picks up the replacement.
+func RegisterRecordCodecOrReplace(c RecordCodec) error {
+ if c.Name() == "" {
+ return fmt.Errorf("record codec name is empty")
+ }
+ codecsMu.Lock()
+ defer codecsMu.Unlock()
+ version := int64(1)
+ if prev, exists := codecs[c.Name()]; exists {
+ version = prev.version + 1
+ }
+ codecs[c.Name()] = &codecEntry{codec: c, version: version}
+ return nil
+}
+
+// GetRecordCodec returns the named codec, falling back to the default.
+func GetRecordCodec(name string) RecordCodec {
+ codecsMu.RLock()
+ defer codecsMu.RUnlock()
+ if e, ok := codecs[name]; ok {
+ return e.codec
+ }
+ return defaultCodec
+}
+
+// ListRecordCodecs snapshots the registry for discovery.
+func ListRecordCodecs() []CodecInfo {
+ codecsMu.RLock()
+ defer codecsMu.RUnlock()
+ out := make([]CodecInfo, 0, len(codecs))
+ for _, e := range codecs {
+ out = append(out, CodecInfo{
+ Name: e.codec.Name(),
+ Namespace: e.codec.Namespace(),
+ Version: e.version,
+ })
+ }
+ return out
+}
+
+// otelCodec is the default: the otel envelope JSON shared with the agent
+// LogEvent format and the OTLP transport boundary.
+type otelCodec struct{}
+
+func (otelCodec) Name() string { return "otel" }
+func (otelCodec) Namespace() string { return "" }
+func (otelCodec) Decode(data []byte) (*event.Event, error) {
+ return otel.DecodeEnvelope(data)
+}
+func (otelCodec) Encode(rec *event.Event) ([]byte, error) {
+ return otel.EncodeEnvelope(rec)
+}
+
+var defaultCodec RecordCodec = otelCodec{}
+
+func init() { MustRegisterRecordCodec(otelCodec{}) }
diff --git a/core/pipeline/conditionals.go b/core/pipeline/conditionals.go
index 357e8f33f..4aac4f480 100644
--- a/core/pipeline/conditionals.go
+++ b/core/pipeline/conditionals.go
@@ -92,7 +92,16 @@ func (r WhenProcessor) Process(ctx *Context) error {
return nil
}
- if !(r.condition).Check(ctx) {
+ // Inside a per-record sub-chain (for_each, direct-ship chains) the
+ // condition must see the current record's fields — the same fallback
+ // IfThenElseProcessor applies; pipeline-level usage keeps reading the
+ // parameters.
+ values := conditions.ValuesMap(ctx)
+ if rec, ok := CurrentRecord(ctx); ok {
+ values = rec
+ }
+
+ if !(r.condition).Check(values) {
ctx.AddFlowProcess(r.p.Name() + "-skipped")
return nil
}
@@ -145,6 +154,13 @@ type IfThenElseProcessor struct {
// NewIfElseThenProcessor construct a new IfThenElseProcessor.
func NewIfElseThenProcessor(cfg *config.Config) (*IfThenElseProcessor, error) {
+ return NewIfElseThenProcessorWithScope(cfg, ScopeInfo{Kind: ScopeBatch})
+}
+
+// NewIfElseThenProcessorWithScope compiles the branches under the
+// ENCLOSING scope (kind+namespace inherited) but non-direct: batch_only
+// processors are rejected inside branches regardless of the outer host.
+func NewIfElseThenProcessorWithScope(cfg *config.Config, scope ScopeInfo) (*IfThenElseProcessor, error) {
var tempConfig ifThenElseConfig
if err := cfg.Unpack(&tempConfig); err != nil {
return nil, err
@@ -160,14 +176,14 @@ func NewIfElseThenProcessor(cfg *config.Config) (*IfThenElseProcessor, error) {
return nil, nil
}
if !c.IsArray() {
- return NewPipeline([]*config.Config{c})
+ return newPipelineWithScope([]*config.Config{c}, scope, false)
}
var pc []*config.Config
if err := c.Unpack(&pc); err != nil {
return nil, err
}
- return NewPipeline(pc)
+ return newPipelineWithScope(pc, scope, false)
}
var ifProcessors, elseProcessors *Processors
diff --git a/core/pipeline/config.go b/core/pipeline/config.go
index efa439e8b..cfc9fd469 100644
--- a/core/pipeline/config.go
+++ b/core/pipeline/config.go
@@ -44,7 +44,8 @@ type PipelineConfigV2 struct {
// Human-readable identifier for this pipeline.
// Required when creating a pipeline via the API.
- Name string `config:"name" json:"name,omitempty"`
+ // Keyword so the pipeline list API can term-query and sort on it.
+ Name string `config:"name" json:"name,omitempty" elastic_mapping:"name: { type: keyword }"`
// A disabled pipeline is treated as if it does not exist. Nil means
// "use the module default".
@@ -98,11 +99,29 @@ type PipelineConfigV2 struct {
// Processors is the ordered chain of processor definitions that makes
// up the actual work the pipeline performs. Each entry is a single
// processor config keyed by processor type.
- Processors []map[string]interface{} `config:"processor" json:"processor"`
+ // Indexing is disabled: processor configs are arbitrary per-processor
+ // JSON and would otherwise be dynamically mapped, risking field-type
+ // conflicts between pipelines on the same index.
+ Processors []map[string]interface{} `config:"processor" json:"processor" elastic_mapping:"processor: { enabled: false }"`
+
+ // StrictContract turns record-contract mismatches into compile errors
+ // instead of warnings. Absent = legacy warn-only (P8 flips the default).
+ StrictContract *bool `config:"strict_contract" json:"strict_contract,omitempty"`
// Labels are arbitrary metadata attached to the pipeline.
Labels map[string]interface{} `config:"labels" json:"labels"`
+ // CanvasLayout persists the orchestration UI's node positions: an
+ // arbitrary, UI-owned mapping (processor-entry key → {x, y}). Pure
+ // presentation state — never read by execution; stored with the
+ // config so any client editing the pipeline restores the same
+ // arrangement. Not indexed (arbitrary JSON shapes).
+ //
+ // Keys must NOT be purely numeric: ucfg reads digit-only keys as
+ // array indices and silently drops them from map unpacking, so "0"
+ // unpacks to nothing. Clients use a prefixed form such as "n0", "n1".
+ CanvasLayout map[string]interface{} `config:"canvas_layout" json:"canvas_layout,omitempty" elastic_mapping:"canvas_layout: { enabled: false }"`
+
// Transient, if false, marks that this pipeline config shpuld be persisted.
//
// In the current implementation, only the pipeline configs created with the
diff --git a/core/pipeline/config_layout_test.go b/core/pipeline/config_layout_test.go
new file mode 100644
index 000000000..649e153ac
--- /dev/null
+++ b/core/pipeline/config_layout_test.go
@@ -0,0 +1,70 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package pipeline
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/util"
+)
+
+// TestPipelineConfigV2CanvasLayoutRoundTrip (#109): the orchestration
+// UI's node positions ride with the config through config unpacking and
+// JSON serialization — presentation state only, but it must survive both
+// directions untouched.
+//
+// Keys follow the "n" convention documented on the field: ucfg
+// reads purely numeric map keys as array indices and drops them, so "0"
+// would unpack to nothing.
+func TestPipelineConfigV2CanvasLayoutRoundTrip(t *testing.T) {
+ cfgMap := util.MapStr{
+ "name": "layout-probe",
+ "processor": []interface{}{
+ util.MapStr{"echo": util.MapStr{"message": "hi"}},
+ },
+ "canvas_layout": util.MapStr{
+ "n0": util.MapStr{"x": 120.5, "y": 40},
+ "n1": util.MapStr{"x": 340, "y": 40},
+ },
+ }
+ c, err := config.NewConfigFrom(cfgMap)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var cfg PipelineConfigV2
+ if err := c.Unpack(&cfg); err != nil {
+ t.Fatal(err)
+ }
+ if cfg.CanvasLayout == nil {
+ t.Fatal("canvas_layout must unpack onto the config")
+ }
+ n0, _ := cfg.CanvasLayout["n0"].(map[string]interface{})
+ if n0 == nil || n0["x"] != 120.5 {
+ t.Fatalf("layout node n0 lost: %v", cfg.CanvasLayout["n0"])
+ }
+
+ // JSON round-trip (API + persistence shape)
+ b, err := json.Marshal(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var back map[string]interface{}
+ if err := json.Unmarshal(b, &back); err != nil {
+ t.Fatal(err)
+ }
+ layout, _ := back["canvas_layout"].(map[string]interface{})
+ if len(layout) != 2 {
+ t.Fatalf("canvas_layout must survive JSON: %v", back["canvas_layout"])
+ }
+
+ // absent stays absent (omitempty) — legacy configs are unchanged
+ var legacy PipelineConfigV2
+ lb, _ := json.Marshal(legacy)
+ if strings.Contains(string(lb), "canvas_layout") {
+ t.Fatalf("empty layout must not serialize: %s", lb)
+ }
+}
diff --git a/core/pipeline/context.go b/core/pipeline/context.go
index 97fe18af8..a462e5a00 100755
--- a/core/pipeline/context.go
+++ b/core/pipeline/context.go
@@ -89,6 +89,7 @@ type Context struct {
exitErr error
processErrs []error
processHistory []string
+ stepStats []StepStat
// steps tracks the count of state transition, doesn't reset within the context lifecycle
id string
steps int64
@@ -104,6 +105,10 @@ type Context struct {
pause sync.WaitGroup
// Set this to true if you want to stop the pipeline, and then, pause (suspend) the goroutine.
isQuit bool
+ // drainRequested asks long-running consumers to stop FETCHING new
+ // work: finish the in-flight batch, commit its offset, then return —
+ // as opposed to CancelTask, which aborts mid-batch (redelivery).
+ drainRequested bool
stateLock sync.Mutex
// Set this to true if you want to let the goroutine exit, i.e., the kill signal.
released bool
@@ -187,6 +192,7 @@ func (ctx *Context) ResetContext() {
ctx.exitErr = nil
ctx.processErrs = []error{}
ctx.processHistory = []string{}
+ ctx.stepStats = nil
ctx.ResetParameters()
}
@@ -198,10 +204,46 @@ func (ctx *Context) GetRequestProcess() []string {
return ctx.processHistory
}
+// maxProcessHistory bounds the flow trace: long-lived contexts run
+// unbounded record batches and the history feeds log.Trace — an uncapped
+// slice makes per-step logging cost grow linearly with total steps (and
+// the whole loop quadratic).
+const maxProcessHistory = 32
+
func (ctx *Context) AddFlowProcess(str string) {
- if str != "" {
- ctx.processHistory = append(ctx.processHistory, str)
+ if str == "" {
+ return
+ }
+ if len(ctx.processHistory) >= maxProcessHistory {
+ // keep the MOST RECENT entries (drop from the head)
+ n := copy(ctx.processHistory, ctx.processHistory[len(ctx.processHistory)-maxProcessHistory+1:])
+ ctx.processHistory = ctx.processHistory[:n]
+ }
+ ctx.processHistory = append(ctx.processHistory, str)
+}
+
+// StepStat is the per-processor outcome of one executed pipeline step:
+// which processor ran, how long it took and whether it errored. Recorded by
+// the sequential runner for document-level pipeline observability.
+type StepStat struct {
+ Name string
+ TookUs int64
+ Error string
+}
+
+func (ctx *Context) GetStepStats() []StepStat {
+ return ctx.stepStats
+}
+
+func (ctx *Context) AddStepStat(name string, took time.Duration, err error) {
+ if name == "" {
+ return
}
+ s := StepStat{Name: name, TookUs: took.Microseconds()}
+ if err != nil {
+ s.Error = err.Error()
+ }
+ ctx.stepStats = append(ctx.stepStats, s)
}
func (ctx *Context) IsFailed() bool {
@@ -332,6 +374,9 @@ func (ctx *Context) Restart() {
defer ctx.stateLock.Unlock()
ctx.isQuit = false
+ // a fresh run is no longer draining — otherwise a consumer started
+ // after _drain would immediately drain again
+ ctx.drainRequested = false
}
// IsExit means pipeline has been manually stopped, will not running until Restart
@@ -350,6 +395,23 @@ func (ctx *Context) Exit() {
ctx.isQuit = true
}
+// Drain requests a graceful stop: consumers finish the current batch,
+// commit its offset, then wind down instead of fetching more work.
+func (ctx *Context) Drain() {
+ ctx.stateLock.Lock()
+ defer ctx.stateLock.Unlock()
+
+ ctx.drainRequested = true
+}
+
+// IsDraining reports whether a graceful drain was requested.
+func (ctx *Context) IsDraining() bool {
+ ctx.stateLock.Lock()
+ defer ctx.stateLock.Unlock()
+
+ return ctx.drainRequested
+}
+
func (ctx *Context) CancelTask() {
ctx.cancelFunc()
}
diff --git a/core/pipeline/contract.go b/core/pipeline/contract.go
new file mode 100644
index 000000000..3455bae58
--- /dev/null
+++ b/core/pipeline/contract.go
@@ -0,0 +1,131 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+)
+
+// Granularity declares the execution granularity a processor supports.
+// It is DECLARED, never detected at runtime (D2): the compiler routes by
+// it and the catalog surfaces it for the canvas.
+type Granularity int
+
+const (
+ // GranularityRecord is the default: only the per-record path is
+ // meaningful (grok, mutate, ...).
+ GranularityRecord Granularity = iota
+ // GranularityBatchCapable: batch + single-record paths are both valid
+ // (batch preferred when the host provides one). Implementing
+ // BatchProcessor implies the single-record fallback contract.
+ GranularityBatchCapable
+ // GranularityBatchOnly: only the batch channel is meaningful
+ // (cross-record aggregation). Must be a DIRECT member of a record
+ // host's sub-chain — nesting under if/then is a compile error there,
+ // because the batch channel cannot reach it.
+ GranularityBatchOnly
+)
+
+func (g Granularity) String() string {
+ switch g {
+ case GranularityBatchCapable:
+ return "batch_capable"
+ case GranularityBatchOnly:
+ return "batch_only"
+ default:
+ return "record"
+ }
+}
+
+// RecordContract is the optional declaration processors make about where
+// they may run and what they need. Unconstrained processors (the entire
+// historical inventory) implement nothing and pass every placement.
+type RecordContract struct {
+ // Scope is the scope the processor REQUIRES. ScopeRecord demands a
+ // record host (for_each/lane_consumer sub-chain); ScopeBatch demands
+ // the chain root / consumer sub-chain. Only set what you require.
+ Scope ScopeKind
+ // Namespace is the record family the processor depends on
+ // ("document", "attachment"); "" accepts any namespace.
+ Namespace string
+ // RequiredFields are optional hard assertions on record paths
+ // ("document.content"), enforced by the runtime guard and by dry-run
+ // sample validation — never by placement (data does not exist at
+ // compile time).
+ RequiredFields []string
+ // Granularity is the declared execution granularity (see Granularity).
+ Granularity Granularity
+}
+
+// ContractProvider is implemented by processors that declare a contract.
+type ContractProvider interface {
+ RecordContract() RecordContract
+}
+
+// ScopeAwareProcessor is implemented by hosts that compile nested chains
+// of their own (for_each, lane_consumer). The registry constructor takes
+// no scope, so those hosts compile their sub-chains warn-only; after
+// construction the enclosing compile hands them its ScopeInfo so
+// strictness reaches the nested chain too. ApplyScope must be
+// idempotent — a host may be constructed and scoped more than once.
+type ScopeAwareProcessor interface {
+ ApplyScope(scope ScopeInfo) error
+}
+
+// validatePlacement checks one constructed processor against the compile
+// scope. The contract comes from the instance (ContractProvider) or from
+// the registry (RegisterProcessorContract — the path every product
+// processor takes); without the registry fallback the placement check
+// would silently pass for all of them. Returns a descriptive error; the
+// caller decides error-vs-warning (strict mode) — this function stays pure.
+func validatePlacement(p Processor, stepIdx int, scope ScopeInfo, direct bool) error {
+ var c RecordContract
+ if cp, ok := p.(ContractProvider); ok {
+ c = cp.RecordContract()
+ } else {
+ c = RecordContractOf(p.Name())
+ }
+ if c.Scope == ScopeAny && c.Namespace == "" && c.Granularity == GranularityRecord && len(c.RequiredFields) == 0 {
+ return nil // unconstrained
+ }
+ where := fmt.Sprintf("processor [%s] at step %d", p.Name(), stepIdx+1)
+
+ if c.Scope == ScopeRecord && scope.Kind != ScopeRecord {
+ return fmt.Errorf("%s requires record scope (inside a for_each/lane_consumer sub-chain), found %s scope", where, scope)
+ }
+ if c.Scope == ScopeBatch && scope.Kind != ScopeBatch {
+ return fmt.Errorf("%s requires batch scope (chain root / consumer sub-chain), found %s scope", where, scope)
+ }
+ if c.Scope == ScopeRecord && c.Namespace != "" && scope.Kind == ScopeRecord && scope.Namespace != c.Namespace {
+ return fmt.Errorf("%s requires record namespace [%s], the host sub-chain provides [%s]", where, c.Namespace, scope.Namespace)
+ }
+ if c.Granularity == GranularityBatchOnly && scope.Kind == ScopeRecord && !direct {
+ return fmt.Errorf("%s declares batch_only granularity and must be a direct member of the record host's processor list (batch channel cannot reach nested chains)", where)
+ }
+ return nil
+}
+
+// CheckContract is the runtime guard (defense in depth for chains that
+// bypass compile-time validation — legacy stored configs, ad-hoc runs).
+// Processors declaring a contract call it at the top of Process and hand
+// the error to their host's on_failure policy.
+func CheckContract(ctx *Context, c RecordContract) error {
+ if ctx == nil {
+ return nil
+ }
+ if c.Scope == ScopeRecord {
+ if _, ok := CurrentRecord(ctx); !ok {
+ return fmt.Errorf("contract requires record scope, no current record in context")
+ }
+ for _, f := range c.RequiredFields {
+ if rec, ok := CurrentRecord(ctx); ok {
+ if _, err := rec.Fields.GetValue(f); err != nil {
+ return fmt.Errorf("contract requires field [%s], missing on current record", f)
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/core/pipeline/contract_test.go b/core/pipeline/contract_test.go
new file mode 100644
index 000000000..820c042b6
--- /dev/null
+++ b/core/pipeline/contract_test.go
@@ -0,0 +1,129 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/util"
+)
+
+// ---- contract test doubles (registered like real plugins) -------------------
+
+type contractProc struct {
+ name string
+ contract RecordContract
+}
+
+func (p *contractProc) Name() string { return p.name }
+func (p *contractProc) Process(ctx *Context) error {
+ return CheckContract(ctx, p.contract)
+}
+func (p *contractProc) RecordContract() RecordContract { return p.contract }
+
+func init() {
+ RegisterProcessorPlugin("test_record_only", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_record_only", RecordContract{Scope: ScopeRecord}}, nil
+ })
+ RegisterProcessorPlugin("test_doc_only", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_doc_only", RecordContract{Scope: ScopeRecord, Namespace: "document"}}, nil
+ })
+ RegisterProcessorPlugin("test_batch_scope", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_batch_scope", RecordContract{Scope: ScopeBatch}}, nil
+ })
+ RegisterProcessorPlugin("test_batch_only", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_batch_only", RecordContract{Scope: ScopeRecord, Granularity: GranularityBatchOnly}}, nil
+ })
+ RegisterProcessorPlugin("test_unconstrained", func(c *config.Config) (Processor, error) {
+ return &contractProc{"test_unconstrained", RecordContract{}}, nil
+ })
+}
+
+func cfgList(t *testing.T, name string) []*config.Config {
+ t.Helper()
+ c, err := config.NewConfigFrom([]util.MapStr{{name: util.MapStr{}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var list []*config.Config
+ if err := c.Unpack(&list); err != nil {
+ t.Fatal(err)
+ }
+ return list
+}
+
+func TestContract_RecordScopeInBatchChainStrict(t *testing.T) {
+ scope := ScopeInfo{Kind: ScopeBatch, Strict: true}
+ _, err := newPipelineWithScope(cfgList(t, "test_record_only"), scope, false)
+ if err == nil || !strings.Contains(err.Error(), "requires record scope") {
+ t.Fatalf("want record-scope error, got %v", err)
+ }
+}
+
+func TestContract_RecordScopeInBatchChainNonStrictWarns(t *testing.T) {
+ // legacy default: NewPipeline never hard-fails on contracts
+ procs, err := NewPipeline(cfgList(t, "test_record_only"))
+ if err != nil {
+ t.Fatalf("legacy NewPipeline must compile contract mismatches (warn only): %v", err)
+ }
+ if len(procs.List) != 1 {
+ t.Fatal("processor dropped in non-strict mode")
+ }
+}
+
+func TestContract_NamespaceMismatch(t *testing.T) {
+ scope := ScopeInfo{Kind: ScopeRecord, Namespace: "otel-generic", Strict: true}
+ _, err := newPipelineWithScope(cfgList(t, "test_doc_only"), scope, true)
+ if err == nil || !strings.Contains(err.Error(), "namespace") {
+ t.Fatalf("want namespace mismatch error, got %v", err)
+ }
+ // matching namespace compiles
+ scope.Namespace = "document"
+ if _, err := newPipelineWithScope(cfgList(t, "test_doc_only"), scope, true); err != nil {
+ t.Fatalf("matching namespace must compile: %v", err)
+ }
+}
+
+func TestContract_BatchScopeInRecordChain(t *testing.T) {
+ scope := ScopeInfo{Kind: ScopeRecord, Namespace: "", Strict: true}
+ _, err := newPipelineWithScope(cfgList(t, "test_batch_scope"), scope, true)
+ if err == nil || !strings.Contains(err.Error(), "requires batch scope") {
+ t.Fatalf("want batch-scope error, got %v", err)
+ }
+}
+
+func TestContract_BatchOnlyNestedIsRejected(t *testing.T) {
+ // direct member of a record host sub-chain: OK
+ direct := ScopeInfo{Kind: ScopeRecord, Namespace: "", Strict: true}
+ if _, err := newPipelineWithScope(cfgList(t, "test_batch_only"), direct, true); err != nil {
+ t.Fatalf("batch_only as direct member must compile: %v", err)
+ }
+ // nested under if.then (same scope, non-direct): compile error
+ c, err := config.NewConfigFrom([]util.MapStr{{"if": util.MapStr{"equals": util.MapStr{"x": "1"}},
+ "then": []util.MapStr{{"test_batch_only": util.MapStr{}}}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var list []*config.Config
+ if err := c.Unpack(&list); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := newPipelineWithScope(list, direct, true); err == nil || !strings.Contains(err.Error(), "batch_only") {
+ t.Fatalf("want batch_only nesting error, got %v", err)
+ }
+}
+
+func TestContract_UnconstrainedPassesEverywhere(t *testing.T) {
+ for _, scope := range []ScopeInfo{
+ {Kind: ScopeBatch, Strict: true},
+ {Kind: ScopeRecord, Namespace: "anything", Strict: true},
+ } {
+ if _, err := newPipelineWithScope(cfgList(t, "test_unconstrained"), scope, true); err != nil {
+ t.Fatalf("unconstrained processor rejected in %v: %v", scope, err)
+ }
+ }
+}
diff --git a/core/pipeline/filter.go b/core/pipeline/filter.go
index 3a59dfbac..bfd1a2e89 100644
--- a/core/pipeline/filter.go
+++ b/core/pipeline/filter.go
@@ -100,7 +100,7 @@ func NewFilter(cfg []*config.Config) (*Filters, error) {
}
if len(procs.List) > 0 {
- log.Debugf("generated new filters: %v", procs)
+ log.Debugf("generated new filters: %s", procs.String())
}
return procs, nil
}
@@ -150,7 +150,7 @@ func (procs *Filters) Close() error {
func (procs *Filters) Filter(ctx *fasthttp.RequestCtx) {
if procs == nil {
- log.Errorf("invalid filter: %v", procs)
+ log.Errorf("invalid filter: %s", procs.String())
return
}
for _, p := range procs.List {
diff --git a/core/pipeline/observer.go b/core/pipeline/observer.go
new file mode 100644
index 000000000..59988fd4a
--- /dev/null
+++ b/core/pipeline/observer.go
@@ -0,0 +1,148 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "sync"
+ "time"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/util"
+)
+
+// StepEvent is the canonical description of one processor invocation on one
+// batch/record. It is the single source for the three observability
+// projections (D5): dry-run captures it fully (snapshots + diff), runtime
+// metrics aggregate it (counters/latency histograms), and the flight
+// recorder keeps a sampled ring of it (errors always kept).
+type StepEvent struct {
+ TraceID string // record-level trace id ("" when absent)
+ // InstanceID identifies the executing node. Multi-instance
+ // deployments attribute events per instance (cluster views aggregate
+ // them); emitStep stamps it, so it is "" only in unit tests that
+ // build events by hand.
+ InstanceID string
+ Pipeline string // pipeline/task name
+ Step int // 0-based index within the executing chain
+ Processor string // processor Name()
+ Scope ScopeInfo // batch | record + namespace ("" = unset)
+ Before util.MapStr // optional pre-state snapshot (nil unless captured)
+ After util.MapStr // optional post-state snapshot (nil unless captured)
+ Duration time.Duration
+ Outcome Outcome
+ Err string
+}
+
+// Outcome classifies how a step ended.
+type Outcome string
+
+const (
+ OutcomeOK Outcome = "ok"
+ OutcomeError Outcome = "error"
+ OutcomeDropped Outcome = "dropped"
+ OutcomeSkippedContract Outcome = "skipped-contract"
+
+ // OutcomeTakeover marks a singleton lock takeover (a control-plane
+ // lifecycle event, not a chain step): this instance acquired a lock
+ // whose previous holder's lease expired. Not an error, but always
+ // retained by the flight recorder (any non-ok outcome is).
+ OutcomeTakeover Outcome = "lock-takeover"
+)
+
+// ExecutionObserver receives StepEvents as chains execute. Observers must
+// be cheap and non-blocking: they run on pipeline hot paths. The nil/empty
+// registry is a zero-cost fast path — production without observability
+// installs nothing and pays one len() check per step.
+type ExecutionObserver interface {
+ OnStep(e *StepEvent)
+}
+
+// ScopeInfo carries the compile/run-time scope a step executed in. Batch
+// scope is the chain root (messages convention); record scope is inside a
+// record host (for_each / lane_consumer) with the namespace of the active
+// codec/lane.
+type ScopeInfo struct {
+ Kind ScopeKind
+ Namespace string
+ // Strict turns contract mismatches into compile errors instead of
+ // warnings (threaded from PipelineConfigV2.StrictContract by hosts).
+ Strict bool
+}
+
+type ScopeKind int
+
+const (
+ // ScopeAny is the zero value: a contract without a scope requirement
+ // accepts every placement (the "undeclared = pass" rule).
+ ScopeAny ScopeKind = iota
+ ScopeBatch
+ ScopeRecord
+)
+
+func (s ScopeInfo) String() string {
+ if s.Kind == ScopeRecord {
+ return "record:" + s.Namespace
+ }
+ return "batch"
+}
+
+var (
+ observersMu sync.RWMutex
+ observers []ExecutionObserver
+)
+
+// RegisterObserver installs a global execution observer (idempotent per
+// instance). Cheap enough to call from tests and tooling; the empty
+// registry keeps the hot path free.
+func RegisterObserver(o ExecutionObserver) {
+ if o == nil {
+ return
+ }
+ observersMu.Lock()
+ defer observersMu.Unlock()
+ for _, existing := range observers {
+ if existing == o {
+ return
+ }
+ }
+ observers = append(observers, o)
+}
+
+// UnregisterObserver removes a previously installed observer.
+func UnregisterObserver(o ExecutionObserver) {
+ observersMu.Lock()
+ defer observersMu.Unlock()
+ for i, existing := range observers {
+ if existing == o {
+ observers = append(observers[:i], observers[i+1:]...)
+ return
+ }
+ }
+}
+
+func hasObservers() bool {
+ observersMu.RLock()
+ defer observersMu.RUnlock()
+ return len(observers) > 0
+}
+
+// emitStep fans a StepEvent out to the installed observers. Caller fills
+// the event; emitStep only stamps what it can derive (TraceID from the
+// current record when in record scope, InstanceID from the local node).
+func emitStep(e *StepEvent, ctx *Context) {
+ if e.InstanceID == "" {
+ e.InstanceID = global.Env().SystemConfig.NodeConfig.ID
+ }
+ if ctx != nil {
+ if rec, ok := CurrentRecord(ctx); ok {
+ e.TraceID = TraceID(rec)
+ }
+ }
+ observersMu.RLock()
+ defer observersMu.RUnlock()
+ for _, o := range observers {
+ o.OnStep(e)
+ }
+}
diff --git a/core/pipeline/observer_test.go b/core/pipeline/observer_test.go
new file mode 100644
index 000000000..434385d00
--- /dev/null
+++ b/core/pipeline/observer_test.go
@@ -0,0 +1,156 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+)
+
+type obsNoopProc struct{ name string }
+
+func (p obsNoopProc) Name() string { return p.name }
+func (p obsNoopProc) Process(ctx *Context) error { return nil }
+
+type obsFailProc struct{ name string }
+
+func (p obsFailProc) Name() string { return p.name }
+func (p obsFailProc) Process(ctx *Context) error { return fmt.Errorf("boom") }
+
+type capture struct{ events []*StepEvent }
+
+func (c *capture) OnStep(e *StepEvent) {
+ cp := *e
+ c.events = append(c.events, &cp)
+}
+
+func newTestContext(t *testing.T) *Context {
+ t.Helper()
+ cfg := PipelineConfigV2{Name: "observer-test"}
+ ctx := AcquireContext(cfg)
+ ctx.Context = context.Background()
+ ctx.Started()
+ return ctx
+}
+
+func TestObserver_CapturesBatchStepsInOrder(t *testing.T) {
+ chain := &Processors{List: []Processor{obsNoopProc{"a"}, obsNoopProc{"b"}}}
+ ctx := newTestContext(t)
+
+ cap := &capture{}
+ RegisterObserver(cap)
+ defer UnregisterObserver(cap)
+
+ if err := chain.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if len(cap.events) != 2 {
+ t.Fatalf("want 2 step events, got %d", len(cap.events))
+ }
+ for i, e := range cap.events {
+ if e.Step != i || e.Pipeline != "observer-test" || e.Processor != string(rune('a'+i)) {
+ t.Fatalf("event %d mismatch: %+v", i, e)
+ }
+ if e.Scope.Kind != ScopeBatch || e.Outcome != OutcomeOK {
+ t.Fatalf("event %d scope/outcome: %+v", i, e)
+ }
+ }
+}
+
+func TestObserver_RecordScopeAndTraceID(t *testing.T) {
+ chain := &Processors{List: []Processor{obsNoopProc{"r"}}}
+ ctx := newTestContext(t)
+ rec := &event.Event{Fields: util.MapStr{}}
+ SetTraceID(rec, "tr-123")
+ ctx.Set(RecordContextKey, rec)
+
+ cap := &capture{}
+ RegisterObserver(cap)
+ defer UnregisterObserver(cap)
+
+ if err := chain.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if len(cap.events) != 1 {
+ t.Fatalf("want 1 event, got %d", len(cap.events))
+ }
+ e := cap.events[0]
+ if e.Scope.Kind != ScopeRecord {
+ t.Fatalf("scope = %v, want record", e.Scope)
+ }
+ if e.TraceID != "tr-123" {
+ t.Fatalf("trace id = %q", e.TraceID)
+ }
+}
+
+func TestObserver_ErrorOutcome(t *testing.T) {
+ chain := &Processors{List: []Processor{obsFailProc{"bad"}}}
+ ctx := newTestContext(t)
+
+ cap := &capture{}
+ RegisterObserver(cap)
+ defer UnregisterObserver(cap)
+
+ if err := chain.Process(ctx); err == nil {
+ t.Fatal("expected chain error")
+ }
+ if len(cap.events) != 1 || cap.events[0].Outcome != OutcomeError || cap.events[0].Err != "boom" {
+ t.Fatalf("error event mismatch: %+v", cap.events)
+ }
+}
+
+func TestObserver_UnregisterStopsDelivery(t *testing.T) {
+ chain := &Processors{List: []Processor{obsNoopProc{"x"}}}
+ ctx := newTestContext(t)
+ cap := &capture{}
+ RegisterObserver(cap)
+ UnregisterObserver(cap)
+ if err := chain.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if len(cap.events) != 0 {
+ t.Fatalf("unregistered observer still received %d events", len(cap.events))
+ }
+}
+
+// BenchmarkProcess_NoObservers guards the empty-registry fast path: the
+// observer hook must add no allocations when nothing is installed
+// (compare against removing the hasObservers guards).
+func BenchmarkProcess_NoObservers(b *testing.B) {
+ chain := &Processors{List: []Processor{obsNoopProc{"a"}, obsNoopProc{"b"}, obsNoopProc{"c"}}}
+ cfg := PipelineConfigV2{Name: "bench"}
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ ctx := AcquireContext(cfg)
+ ctx.Context = context.Background()
+ ctx.Started()
+ _ = chain.Process(ctx)
+ ReleaseContext(ctx)
+ }
+}
+
+// The flow trace must stay bounded on long-lived contexts — an uncapped
+// history makes per-step logging cost grow with total steps ever run.
+func TestAddFlowProcess_Bounded(t *testing.T) {
+ cfg := PipelineConfigV2{Name: "t"}
+ ctx := AcquireContext(cfg)
+ defer ReleaseContext(ctx)
+ for i := 0; i < 10000; i++ {
+ ctx.AddFlowProcess("p")
+ }
+ if got := len(ctx.GetFlowProcess()); got > maxProcessHistory {
+ t.Fatalf("history grew to %d, want <= %d", got, maxProcessHistory)
+ }
+ // most-recent preserved
+ h := ctx.GetFlowProcess()
+ if len(h) > 0 && h[len(h)-1] != "p" {
+ t.Fatal("latest entry lost")
+ }
+}
diff --git a/core/pipeline/pipelinelog.go b/core/pipeline/pipelinelog.go
new file mode 100644
index 000000000..f95e8b658
--- /dev/null
+++ b/core/pipeline/pipelinelog.go
@@ -0,0 +1,68 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+
+ log "github.com/cihub/seelog"
+)
+
+// pipelinelog: processor logging with automatic trace-id correlation.
+//
+// Processors emit text diagnostics via seelog; the trace_id lives on the
+// RECORD (Metadata.Labels["_trace_id"]), not the context — so plain
+// log.Errorf cannot correlate to a specific record's flight trail.
+// These helpers read the current record's trace_id from the context and
+// prefix every message, making logs grep-able by trace_id:
+//
+// pipelinelog.Warnf(ctx, "extract failed: %v", err)
+// // → "[tr-abc123] extract failed: timeout"
+//
+// The prefix format is stable: [tr-] — log panels and grep use it.
+
+// logPrefix extracts the trace prefix from the context's current record.
+func logPrefix(ctx *Context) string {
+ if ctx == nil {
+ return ""
+ }
+ rec, ok := CurrentRecord(ctx)
+ if !ok {
+ return ""
+ }
+ id := TraceID(rec)
+ if id == "" {
+ return ""
+ }
+ return "[" + id + "] "
+}
+
+func Tracef(ctx *Context, format string, args ...interface{}) {
+ log.Trace(logPrefix(ctx) + sprintf(format, args...))
+}
+
+func Debugf(ctx *Context, format string, args ...interface{}) {
+ log.Debug(logPrefix(ctx) + sprintf(format, args...))
+}
+
+func Infof(ctx *Context, format string, args ...interface{}) {
+ log.Info(logPrefix(ctx) + sprintf(format, args...))
+}
+
+func Warnf(ctx *Context, format string, args ...interface{}) {
+ log.Warn(logPrefix(ctx) + sprintf(format, args...))
+}
+
+func Errorf(ctx *Context, format string, args ...interface{}) {
+ log.Error(logPrefix(ctx) + sprintf(format, args...))
+}
+
+// sprintf avoids importing fmt at the top level for a one-liner.
+func sprintf(format string, args ...interface{}) string {
+ if len(args) == 0 {
+ return format
+ }
+ return fmt.Sprintf(format, args...)
+}
diff --git a/core/pipeline/pipelinelog_test.go b/core/pipeline/pipelinelog_test.go
new file mode 100644
index 000000000..a5dd39d41
--- /dev/null
+++ b/core/pipeline/pipelinelog_test.go
@@ -0,0 +1,53 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+)
+
+// pipelinelog helpers embed the current record's trace_id as a [tr-xxx]
+// prefix — the correlation key between text logs and flight trails.
+func TestPipelineLog_TracePrefix(t *testing.T) {
+ cfg := PipelineConfigV2{Name: "t"}
+ ctx := AcquireContext(cfg)
+ ctx.Context = context.Background()
+ ctx.Started()
+ defer ReleaseContext(ctx)
+
+ rec := &event.Event{Fields: util.MapStr{}}
+ SetTraceID(rec, "tr-correlation-42")
+ ctx.Set(RecordContextKey, rec)
+
+ prefix := logPrefix(ctx)
+ if prefix != "[tr-correlation-42] " {
+ t.Fatalf("prefix = %q", prefix)
+ }
+
+ // no record → no prefix
+ ctx.Set(RecordContextKey, nil)
+ if got := logPrefix(ctx); got != "" {
+ t.Fatalf("no-record prefix = %q", got)
+ }
+
+ // record without trace → no prefix
+ rec2 := &event.Event{Fields: util.MapStr{}}
+ ctx.Set(RecordContextKey, rec2)
+ if got := logPrefix(ctx); got != "" {
+ t.Fatalf("no-trace prefix = %q", got)
+ }
+}
+
+// log output actually contains the prefix (integration with seelog)
+func TestPipelineLog_OutputContainsTrace(t *testing.T) {
+ // capture seelog output via a trace writer is complex; the prefix
+ // function is the contract — the log package does the rest
+ _ = strings.TrimSpace // keep import honest
+}
diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go
index 5a09311fd..9819a09cf 100755
--- a/core/pipeline/processor.go
+++ b/core/pipeline/processor.go
@@ -105,7 +105,21 @@ func NewPipelineList() *Processors {
return &Processors{}
}
+// NewPipeline compiles a chain at the default batch scope with contract
+// validation for unconstrained processors only (legacy-compatible).
func NewPipeline(cfg []*config.Config) (*Processors, error) {
+ return newPipelineWithScope(cfg, ScopeInfo{Kind: ScopeBatch}, false)
+}
+
+// NewPipelineWithScope compiles a chain under an explicit scope: record
+// hosts (for_each/lane_consumer) pass their codec/lane namespace so
+// processor contracts are placement-checked at compile time. `direct`
+// marks the host's own processor array (gates batch_only granularity).
+func NewPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (*Processors, error) {
+ return newPipelineWithScope(cfg, scope, direct)
+}
+
+func newPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (*Processors, error) {
if cfg == nil || len(cfg) == 0 {
return nil, errors.Errorf("invalid processor config")
@@ -113,13 +127,39 @@ func NewPipeline(cfg []*config.Config) (*Processors, error) {
procs := NewPipelineList()
- for _, procConfig := range cfg {
+ for stepIdx, procConfig := range cfg {
// Handle if/then/else processor which has multiple top-level keys.
if procConfig.HasField("if") {
- p, err := NewIfElseThenProcessor(procConfig)
+ p, err := NewIfElseThenProcessorWithScope(procConfig, scope)
if err != nil {
return nil, errors.Wrap(err, "failed to make if/then/else processor")
}
+ // an if/else entry itself is not a BatchProcessor member; its
+ // contract (if any) still validates against this scope
+ if err := validatePlacement(p, stepIdx, scope, false); err != nil {
+ if scope.Strict {
+ return nil, err
+ }
+ log.Warnf("contract (non-strict): %v", err)
+ }
+ procs.AddProcessor(p)
+ continue
+ }
+
+ // Handle the case/branches multi-way dispatch: like if/then/else it
+ // owns its step and its arm chains must compile under the enclosing
+ // scope for contract placement.
+ if procConfig.HasField("case") {
+ p, err := NewCaseProcessorWithScope(procConfig, scope)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to make case/branches processor")
+ }
+ if err := validatePlacement(p, stepIdx, scope, false); err != nil {
+ if scope.Strict {
+ return nil, err
+ }
+ log.Warnf("contract (non-strict): %v", err)
+ }
procs.AddProcessor(p)
continue
}
@@ -157,6 +197,24 @@ func NewPipeline(cfg []*config.Config) (*Processors, error) {
p, ok := plugin.(Processor)
if ok {
+ cacheProcessorContract(p)
+ if err := validatePlacement(p, stepIdx, scope, direct); err != nil {
+ if scope.Strict {
+ return nil, err
+ }
+ log.Warnf("contract (non-strict): %v", err)
+ }
+ // hosts with nested chains (for_each, lane_consumer) compile
+ // them warn-only at construction; hand them the enclosing
+ // scope so a strict compile reaches the sub-chain as well
+ if sa, isScopeAware := p.(ScopeAwareProcessor); isScopeAware {
+ if err := sa.ApplyScope(scope); err != nil {
+ if scope.Strict {
+ return nil, err
+ }
+ log.Warnf("contract (non-strict): %v", err)
+ }
+ }
procs.AddProcessor(p)
} else {
return nil, errors.Errorf("invalid processor: [%v]", plugin.Name())
@@ -164,7 +222,7 @@ func NewPipeline(cfg []*config.Config) (*Processors, error) {
}
if len(procs.List) > 0 {
- log.Debugf("generated new processors: %v", procs)
+ log.Debugf("generated new processors: %s", procs.String())
}
return procs, nil
}
@@ -248,7 +306,7 @@ func (procs *Processors) Process(ctx *Context) error {
}()
}
- for _, p := range procs.List {
+ for stepIdx, p := range procs.List {
if !ctx.ShouldContinue() {
if global.Env().IsDebug {
log.Debugf("filter [%v] not continued", p.Name())
@@ -265,6 +323,14 @@ func (procs *Processors) Process(ctx *Context) error {
return nil
}
+ // Scope derivation for observability: the same loop drives batch
+ // chains and for_each/lane_consumer record sub-chains; the current
+ // record's presence distinguishes them.
+ scope := ScopeInfo{Kind: ScopeBatch}
+ if _, hasRec := CurrentRecord(ctx); hasRec {
+ scope = ScopeInfo{Kind: ScopeRecord}
+ }
+
log.Trace("pipeline: ", ctx.Config.Name, ", start processing:", ctx.processHistory, "->", p.Name())
ctx.AddFlowProcess(p.Name())
@@ -280,8 +346,22 @@ func (procs *Processors) Process(ctx *Context) error {
if err != nil {
stats.Increment("pipeline", ctx.Config.Name, "processor", p.Name(), "total_error")
log.Error("error on processing:", p.Name(), ",", err)
+ if hasObservers() {
+ emitStep(&StepEvent{
+ Pipeline: ctx.Config.Name, Step: stepIdx, Processor: p.Name(),
+ Scope: scope, Duration: time.Since(start),
+ Outcome: OutcomeError, Err: err.Error(),
+ }, ctx)
+ }
return err
}
+ if hasObservers() {
+ emitStep(&StepEvent{
+ Pipeline: ctx.Config.Name, Step: stepIdx, Processor: p.Name(),
+ Scope: scope, Duration: time.Since(start),
+ Outcome: OutcomeOK,
+ }, ctx)
+ }
//if event == nil {
// // Drop.
// return nil, nil
diff --git a/core/pipeline/record.go b/core/pipeline/record.go
index 3a3f6fc10..111449840 100644
--- a/core/pipeline/record.go
+++ b/core/pipeline/record.go
@@ -26,7 +26,10 @@
package pipeline
-import "infini.sh/framework/core/event"
+import (
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+)
// ──────────────────────────────────────────────────────────────────────────
// Record processing convention (framework-level; see
@@ -162,3 +165,28 @@ func TakeClones(ctx *Context) []*event.Event {
}
return nil
}
+
+// TraceIDLabel is the metadata label carrying the record-level trace id.
+// Entries (consumer / lane hosts / connectors) stamp it once; it then rides
+// the record across queues and lanes unchanged (D5 correlation).
+const TraceIDLabel = "_trace_id"
+
+// SetTraceID stamps the trace id onto the record's metadata labels.
+func SetTraceID(rec *event.Event, id string) {
+ if rec == nil || id == "" {
+ return
+ }
+ if rec.Metadata.Labels == nil {
+ rec.Metadata.Labels = util.MapStr{}
+ }
+ rec.Metadata.Labels[TraceIDLabel] = id
+}
+
+// TraceID reads the record's trace id ("" when unset).
+func TraceID(rec *event.Event) string {
+ if rec == nil || rec.Metadata.Labels == nil {
+ return ""
+ }
+ v, _ := rec.Metadata.Labels[TraceIDLabel].(string)
+ return v
+}
diff --git a/core/pipeline/register.go b/core/pipeline/register.go
index d0c9c99b5..7e36afa98 100644
--- a/core/pipeline/register.go
+++ b/core/pipeline/register.go
@@ -251,6 +251,12 @@ type Constructor func(config *config.Config) (ProcessorBase, error)
var registry = NewNamespace()
+// ProcessorConstructors returns every registered constructor keyed by
+// bare name (catalog enumeration for editors and audit tooling).
+func ProcessorConstructors() map[string]ProcessorConstructor {
+ return registry.ProcessorConstructors()
+}
+
// LookupProcessorConstructor resolves a processor constructor by its
// registered (bare) name — nil when unknown. Hosts use this to build
// ad-hoc chains (e.g. dry-run replay of a spec's processor list).
@@ -268,6 +274,35 @@ func RegisterProcessorPlugin(name string, constructor ProcessorConstructor) {
}
}
+// processorContracts carries declared record contracts per processor name
+// (populated via RegisterProcessorContract, or auto-cached when a
+// ContractProvider instance is first constructed by the compiler). It
+// feeds the catalog's scope/record_contract/granularity fields.
+var processorContracts = map[string]RecordContract{}
+
+// RegisterProcessorContract declares (or refreshes) the record contract
+// of an already-registered processor — call it from package init next to
+// the plugin registration. Deterministic and side-effect-free, unlike
+// probing a constructed instance.
+func RegisterProcessorContract(name string, c RecordContract) {
+ processorContracts[name] = c
+}
+
+// RecordContractOf returns the declared contract for a processor name
+// (zero value = unconstrained).
+func RecordContractOf(name string) RecordContract {
+ return processorContracts[name]
+}
+
+// cacheProcessorContract auto-captures contracts from constructed
+// instances so adopters without explicit registration still surface in
+// the catalog after first compile.
+func cacheProcessorContract(p Processor) {
+ if cp, ok := p.(ContractProvider); ok {
+ processorContracts[p.Name()] = cp.RecordContract()
+ }
+}
+
// processorMetadata carries the extracted config schema of processors
// registered via RegisterProcessorPluginWithConfigMetadata, so that the
// discovery API (see modules/pipeline) can render configuration forms.
@@ -283,6 +318,9 @@ var processorCategories = map[string]string{}
// records the schema of its config struct for discovery.
func RegisterProcessorPluginWithConfigMetadata(name string, constructor ProcessorConstructor, configStruct interface{}) {
RegisterProcessorPlugin(name, constructor)
+ if configStruct == nil {
+ return // no schema declared; the processor still registers and runs
+ }
processorMetadata[name] = ExtractFilterMetadata(configStruct)
}
@@ -353,14 +391,36 @@ func GetProcessorCatalog() util.MapStr {
}
catMap := grouped[cat].(util.MapStr)
x, _ := processorMetadata[name]
- catMap[name] = util.MapStr{
+ entry := util.MapStr{
"properties": x,
"category": cat,
}
+ if c, ok := processorContracts[name]; ok {
+ entry["scope"] = scopeString(c.Scope)
+ entry["record_contract"] = util.MapStr{
+ "namespace": c.Namespace,
+ "required_fields": c.RequiredFields,
+ "granularity": c.Granularity.String(),
+ }
+ } else {
+ entry["scope"] = "unconstrained"
+ }
+ catMap[name] = entry
}
return grouped
}
+func scopeString(k ScopeKind) string {
+ switch k {
+ case ScopeRecord:
+ return "record"
+ case ScopeBatch:
+ return "batch"
+ default:
+ return "any"
+ }
+}
+
func RegisterFilterPlugin(name string, constructor FilterConstructor) {
err := registry.RegisterFilter(name, constructor)
if err != nil {
diff --git a/core/pipeline/sync-runner.go b/core/pipeline/sync-runner.go
index 96f7e07a4..0091612c2 100755
--- a/core/pipeline/sync-runner.go
+++ b/core/pipeline/sync-runner.go
@@ -23,6 +23,8 @@
package pipeline
+import "time"
+
func runSync(job *Job, ctx *Context) {
if job.onFailure != nil {
@@ -31,7 +33,11 @@ func runSync(job *Job, ctx *Context) {
for _, task := range job.tasks {
- task.Process(ctx)
+ // per-step timing/error snapshot for pipeline observability
+ // (document run timelines); see Context.AddStepStat
+ started := time.Now()
+ err := task.Process(ctx)
+ ctx.AddStepStat(task.Name(), time.Since(started), err)
}
if job.onComplete != nil {
diff --git a/core/queue/api.go b/core/queue/api.go
index 6d11e81ca..bc13e7f19 100755
--- a/core/queue/api.go
+++ b/core/queue/api.go
@@ -38,6 +38,11 @@ import (
"infini.sh/framework/core/util"
)
+// ErrEmptyQueue: the queue holds no data yet (no written segment) —
+// browse/sample flows treat it as "no messages", not an error. Returned by
+// the DiskQueue advanced processor on AcquireConsumer.
+var ErrEmptyQueue = errors.New("empty queue")
+
type QueueAPI interface {
Name() string
Init(string) error
diff --git a/core/util/saferead.go b/core/util/saferead.go
index bee497814..728897b5f 100644
--- a/core/util/saferead.go
+++ b/core/util/saferead.go
@@ -115,6 +115,33 @@ func isTooBroadReadRoot(c string) bool {
return false
}
+// ValidateReadRoot applies the read policy to a configured root without
+// requiring it to exist yet: system trees are never allowed and a
+// wholesale top-level tree is treated as misconfiguration. Collection
+// roots (agent file intake) may legitimately appear after startup, so
+// unlike NewReadGuard existence is left to the caller.
+func ValidateReadRoot(root string) error {
+ c := canonicalReadPath(root)
+ if IsSystemReadPath(c) {
+ return fmt.Errorf("refusing to read from system path: %s", c)
+ }
+ if isTooBroadReadRoot(c) {
+ return fmt.Errorf("path %s is too broad, use a specific log directory instead", c)
+ }
+ return nil
+}
+
+// IsConfinedUnder reports whether path stays at or below root once both
+// sides are canonicalized (symlink-resolved). It is the collection-side
+// counterpart of ReadGuard's containment rule: a file reached through a
+// symlink that jumps out of the collection root is not confined,
+// whatever spelling the walk handed over.
+func IsConfinedUnder(root, path string) bool {
+ r := canonicalReadPath(root)
+ c := canonicalReadPath(path)
+ return c == r || strings.HasPrefix(c, r+string(filepath.Separator))
+}
+
// readPathComponent keeps all AST fragments belonging to one path component.
//
// A name such as node-* must be removed as one component by dot-dot, rather than
diff --git a/core/util/saferead_test.go b/core/util/saferead_test.go
index 9f7ad4b8c..aa087c749 100644
--- a/core/util/saferead_test.go
+++ b/core/util/saferead_test.go
@@ -275,6 +275,65 @@ func TestIsTooBroadReadRoot(t *testing.T) {
t.Error("expected specific subdir to be acceptable")
}
}
+func TestValidateReadRoot(t *testing.T) {
+ dir := t.TempDir()
+ denied := []string{
+ "/", "/etc", "/etc/nginx", "/root",
+ "/home", "/var", "/opt", // wholesale top-level trees
+ "/var/log/../../etc", // traversal into a system tree
+ }
+ for _, p := range denied {
+ if err := ValidateReadRoot(p); err == nil {
+ t.Errorf("expected [%s] to be rejected", p)
+ }
+ }
+ allowed := []string{
+ "/var/log",
+ "/var/log/elasticsearch",
+ "/opt/myapp/logs",
+ filepath.Join(dir, "logs"),
+ filepath.Join(dir, "not-created-yet"), // existence is the caller's concern
+ }
+ for _, p := range allowed {
+ if err := ValidateReadRoot(p); err != nil {
+ t.Errorf("expected [%s] to pass: %v", p, err)
+ }
+ }
+}
+
+func TestIsConfinedUnder(t *testing.T) {
+ root := t.TempDir()
+ outside := filepath.Join(t.TempDir(), "secret.log")
+
+ if !IsConfinedUnder(root, root) {
+ t.Error("the root itself must be confined")
+ }
+ if !IsConfinedUnder(root, filepath.Join(root, "app", "a.log")) {
+ t.Error("file under the root must be confined")
+ }
+ if !IsConfinedUnder(root, filepath.Join(root, "sub", "..", "a.log")) {
+ t.Error("cleaned traversal inside the root must be confined")
+ }
+ if IsConfinedUnder(root, outside) {
+ t.Error("file outside the root must not be confined")
+ }
+ if IsConfinedUnder(root, root+"-escape") {
+ t.Error("sibling sharing the prefix must not pass as confined")
+ }
+
+ link := filepath.Join(root, "jump.log")
+ // the target must exist, otherwise EvalSymlinks cannot resolve the
+ // link and the deepest-ancestor fallback would compare the raw link
+ if err := os.WriteFile(outside, []byte("secret"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outside, link); err != nil {
+ t.Skipf("symlink unavailable: %v", err)
+ }
+ if IsConfinedUnder(root, link) {
+ t.Error("symlink escaping the root must not be confined")
+ }
+}
// Helper function to create a real log file for guard authorization and resolution tests.
//
diff --git a/core/vfs/fs.go b/core/vfs/fs.go
index 32197661d..69ae366c9 100755
--- a/core/vfs/fs.go
+++ b/core/vfs/fs.go
@@ -43,6 +43,7 @@ import (
"os"
"path"
"path/filepath"
+ "regexp"
"runtime"
"sort"
"strconv"
@@ -597,9 +598,32 @@ func serveFile(w http.ResponseWriter, r *http.Request, fs http.FileSystem, name
// serveContent will check modification time
sizeFunc := func() (int64, error) { return d.Size(), nil }
+ if cc := spaCacheControl(r.URL.Path); cc != "" {
+ w.Header().Set("Cache-Control", cc)
+ }
serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f)
}
+// vfsAssetHashRE matches the content-hash segment in built frontend asset
+// names (index-DPak5aSO.js, pizza_engine_bg-B2pSYpQe.wasm).
+var vfsAssetHashRE = regexp.MustCompile(`-[A-Za-z0-9_]{8,}\.[A-Za-z0-9]+$`)
+
+// spaCacheControl is the cache policy for SPA static assets: hashed build
+// outputs under /assets/ never change in place, so they are marked
+// immutable to let browsers skip revalidation (a big win on repeat visits
+// with large bundles); the HTML entry must always revalidate so a release
+// picks up the asset manifest with the new hashes. An empty return means
+// no intervention (Last-Modified negotiation as usual).
+func spaCacheControl(p string) string {
+ if strings.HasPrefix(p, "/assets/") && vfsAssetHashRE.MatchString(p) {
+ return "public, max-age=31536000, immutable"
+ }
+ if strings.HasSuffix(p, ".html") || strings.HasSuffix(p, "/") || p == "" {
+ return "no-cache"
+ }
+ return ""
+}
+
// toHTTPError returns a non-specific HTTP error message and status code
// for a given non-nil error value. It's important that toHTTPError does not
// actually return err.Error(), since msg and httpStatus are returned to users,
diff --git a/docs/content.en/docs/development/api/_index.md b/docs/content.en/docs/development/api/_index.md
index c5a32cbf1..6dce90000 100644
--- a/docs/content.en/docs/development/api/_index.md
+++ b/docs/content.en/docs/development/api/_index.md
@@ -1,6 +1,7 @@
---
weight: 40
title: "API Development"
+bookCollapseSection: true
---
# API Development
diff --git a/docs/content.en/docs/references/_index.md b/docs/content.en/docs/references/_index.md
index 308de1b46..fe6aa7151 100644
--- a/docs/content.en/docs/references/_index.md
+++ b/docs/content.en/docs/references/_index.md
@@ -17,6 +17,8 @@ Comprehensive reference documentation for the INFINI Framework's core systems an
- [Task Scheduling](task/) — Interval-based, cron-based, and transient task execution
- [Pipeline Record Processing](pipeline_record/) — The record-processing convention: batch splitting, pluggable payload codecs, failure strategies, and batch-aware processors
- [Queue](queue/) — Pluggable message queue abstraction with disk, memory, Kafka, and Redis backends
+- [Lane](lane/) — In-process zero-serialization record channel: bounded capacity, overflow policies, `lane_consumer`, and the queue-vs-lane selection guide
+- [Glossary](glossary/) — Canonical terminology for the pipeline, record-processing, and lane subsystems
- [Key-Value Store](kv/) — Pluggable KV storage with Badger, Elasticsearch, and file-based backends
- [Statistics](stats/) — Metrics collection with counters, gauges, timings, and StatsD integration
- [Conditions](conditions/) — Declarative condition evaluation with logical operators for pipeline control flow
diff --git a/docs/content.en/docs/references/conditions/_index.md b/docs/content.en/docs/references/conditions/_index.md
index a100de493..5987de9b4 100644
--- a/docs/content.en/docs/references/conditions/_index.md
+++ b/docs/content.en/docs/references/conditions/_index.md
@@ -1,6 +1,7 @@
---
title: "Conditions"
weight: 45
+bookCollapseSection: true
---
# Conditions
diff --git a/docs/content.en/docs/references/glossary.md b/docs/content.en/docs/references/glossary.md
new file mode 100644
index 000000000..c13e383c7
--- /dev/null
+++ b/docs/content.en/docs/references/glossary.md
@@ -0,0 +1,50 @@
+---
+title: "Glossary"
+weight: 28
+---
+
+# Glossary
+
+Canonical terminology for the pipeline, record-processing, and lane subsystems. When a term appears in configuration, code, or UI, this is what it means.
+
+## Messages, Batches, Records
+
+| Term | Meaning |
+|------|---------|
+| **message** | A queue's unit of storage: an opaque byte payload (`queue.Message`). Queues do not interpret messages. |
+| **batch** | The pipeline's native input: a slice of messages delivered to a chain. All pipeline processors historically operate here (the *batch scope*). |
+| **record** | A typed, decoded event (`*event.Event`) produced inside a record host. Fields, metadata, and timestamps are mutable in place. Records exist only inside a host's scope — they never leak to the outer batch chain. |
+| **record host** (splitter host) | A processor that splits a batch into records, runs a sub-chain over them, and re-encodes: the built-in `for_each`, `lane_consumer`, or a custom host following the same skeleton. |
+| **codec** (`RecordCodec`) | Pluggable encoder/decoder between message bytes and records (`otel` default, `coco_document`, `coco_attachment`, ...). Registered by name; selected with `codec:` on `for_each`, resolved three-level for output queues. |
+| **namespace** | A record family identifier (`document`, `attachment`, ...) declared by codecs and lane configs; the compile-time currency of `RecordContract.Namespace`. |
+
+## Scopes and Contracts
+
+| Term | Meaning |
+|------|---------|
+| **batch scope** | Execution context at the chain root: processors see raw messages and pipeline parameters, not records. |
+| **record scope** | Execution context inside a record host's sub-chain: `CurrentRecord` is bound, record field paths (`document.content`) are addressable in conditions. |
+| **RecordContract** | A processor's compile-time declaration: where it may run (`Scope`), which record family it needs (`Namespace`), which paths it asserts (`RequiredFields`), and its `Granularity`. Unconstrained processors pass every placement. |
+| **granularity** | Which execution channels a processor supports: `record` (per-record path only), `batch_capable` (both, batch preferred), `batch_only` (cross-record aggregation; must sit directly in a record host's sub-chain). |
+| **strict_contract** | Pipeline config flag: placement violations become compile errors instead of warnings. The `POST /pipeline/_validate` gate reports them either way. |
+| **failure tag** | A record-level error marker appended by `on_failure: tag`; accumulated tags persist into the record's `tags` field before re-encoding so downstream stages can route on them. |
+
+## Channels
+
+| Term | Meaning |
+|------|---------|
+| **queue** | Durable, at-least-once byte channel between producers and consumers (disk, memory, Kafka, Redis backends). Cross-process safe. |
+| **lane** | In-process, zero-serialization record channel: payloads are `*event.Event` pointers. Bounded, at-most-once past a successful `Emit`. See [Lane](../lane/). |
+| **overflow policy** | A full lane's behavior: `block` (backpressure, default), `drop` (best-effort), `spill:` (degrade to disk — only for equivalent fast/slow paths). |
+| **durable bridge** (`output_queue`) | A `lane_consumer`'s optional exit: surviving records encode once into a disk queue, making the lane flow's outcome persistent and wire-compatible with existing consumers. |
+| **AckSet** | Context-borne receipt set for `output{lane:…, ack: true}` steps; the emitting consumer's commit gate waits for settlement before committing offsets (cross-lane at-least-once). |
+
+## Control Plane and Debugging
+
+| Term | Meaning |
+|------|---------|
+| **processor catalog** | Discovery view of all registered processors (scope, contract, config schema) — `GET /pipeline/processors`. Drives canvas UIs and schema forms. |
+| **dry-run** | Server-side chain execution against a sample with side effects replaced by captures: step snapshots, field diffs, per-step timings. `POST /pipeline/dry-run`. |
+| **placement** | Compile-time check that a processor's scope/granularity matches its position (e.g. a `batch_only` processor must sit directly in a record host's sub-chain). |
+| **dynamic lane** | A lane created at runtime (`lane.Create` / `GetOrCreate`) with an `owner` for cascade cleanup — e.g. one per compiled stream pipeline, never shared. |
+| **canvas_layout** | Product-owned pipeline config field holding canvas node coordinates (`map[string]{x,y}`, content-fingerprint keys). Pure presentation state; execution never reads it. |
diff --git a/docs/content.en/docs/references/lane.md b/docs/content.en/docs/references/lane.md
new file mode 100644
index 000000000..2f63cb267
--- /dev/null
+++ b/docs/content.en/docs/references/lane.md
@@ -0,0 +1,111 @@
+---
+title: "Lane"
+weight: 27
+---
+
+# Lane — In-Process Record Channel
+
+A **lane** is the in-process, zero-serialization counterpart of the [queue](../queue/): payloads are `*event.Event` **pointers** — the record *is* the message. A producer emits a record into a lane; a `lane_consumer` host drains it into a record-processing sub-chain. Nothing is encoded or decoded in transit.
+
+```
+producer ──Emit(*event.Event)──▶ [lane: bounded, in-flight] ──▶ lane_consumer
+ │ batch{max, flush_ms} │
+ │ ▼
+ overflow policy record sub-chain
+ (block/drop/spill) (same execution
+ contract as
+ for_each)
+```
+
+Use the record-processing model's [conventions](../pipeline_record/) inside the sub-chain — `CurrentRecord`, `MarkDropped`, failure tags all behave identically to `for_each`.
+
+## Declaring Lanes
+
+```yaml
+lane:
+ - name: mem_processing
+ namespace: document # record-family contract for compile checks
+ capacity: 4096 # bounded; in-flight records
+ overflow: block # block | drop | spill:
+ - name: debug_samples
+ overflow: drop
+```
+
+Configured lanes live in the `lane` section; **dynamic lanes** are created at runtime with `lane.Create(cfg)` / `lane.GetOrCreate(name, cfg)` and carry an `owner` for cascade cleanup (e.g. one lane per compiled stream pipeline, never shared — sharing a lane between consumers is a semantic error). Dynamic lanes default to a small capacity (512) and are lazy: an idle lane costs nothing until first emit. A global `total_budget` caps aggregate in-flight records across dynamic lanes; when the budget is hit each lane applies its own overflow policy.
+
+## Overflow Policies
+
+| Policy | Behavior | When it is correct |
+|--------|----------|-------------------|
+| `block` (default) | `Emit` waits; backpressure reaches the producer | Whenever the producer can safely slow down (task goroutines, HTTP handlers answering 429). The honest default — data is never risked. |
+| `drop` | Discard and count (`drops` metric) | Best-effort telemetry only: tap sampling, debug indexes. |
+| `spill:` | Degrade the record to the disk queue | **Only** when the fast and slow paths run *equivalent* chains — spill means "this record takes the slow version of the same path". |
+
+Two hidden costs of `spill`: a required fast-path step (e.g. redaction) is bypassed for spilled records — a correctness bug if the paths differ; and it can mask backpressure until the spill queue grows unbounded. If the producer cannot block **and** the data must be durable, the honest answer is queue-first (persist, then ACK), not spill.
+
+Spilled records **never flow back into the lane**. The slow path's consumer picks them up on its regular polling cycle, so: a record's path is decided once (`TryEmit` failure is an atomic fork — no double delivery); spilled records survive crashes while in-lane records do not; and completion order across the two paths is not guaranteed — use `block` (lane is FIFO) when ordering matters.
+
+## lane_consumer
+
+```yaml
+processor:
+ - lane_consumer:
+ lane: mem_processing
+ batch_max: 200 # flush on size
+ batch_flush_ms: 1000 # flush on idle
+ on_failure: tag # tag (default) | dead_letter: | ignore
+ failure_tag: _processing_failed
+ output_queue: indexing_documents # optional durable bridge (encode once)
+ processor: [...] # record sub-chain
+```
+
+The execution contract matches `for_each` exactly (stage-wise, order as written, drop/clone/failure-tag semantics identical). Two differences: the entry is zero-decode (records are constructed by the emitting side, not decoded from bytes), and the exit encodes **only** when `output_queue` is configured — `queue.Message.Data = codec.Encode(record)`, wire-compatible with every existing consumer of that codec.
+
+Codec resolution for `output_queue` is three-level: the record's `_codec` hint → the lane's `namespace` primary codec → the default `otel`.
+
+## The Unified `output` Step
+
+Emit into a lane from inside a chain with the [unified output](../pipeline/) step — exactly one target field, enforced at config unpacking:
+
+```yaml
+- output:
+ lane: mem_processing
+ ack: true # attach a receipt to the context's AckSet
+# or:
+- output:
+ queue: indexing_documents
+```
+
+In **batch scope** a queue target pushes raw message bytes; a lane target is a compile error (point the consumer's `codec` + `output_lane` bridging at the lane instead). In **record scope** a queue target encodes the current record; a lane target emits the pointer. `queue_output` / `lane_output` remain as aliases.
+
+## Queue or Lane? — Selection Guide
+
+**The boundary in one sentence:** a successful `Emit` return is the end of the producer's responsibility — lanes are **at-most-once**. A lane flow is safe when at least one of these holds:
+
+1. the upstream can recompute (re-collect, upstream retries, data is disposable), or
+2. an uncommitted queue offset covers the whole flow (a crash replays it), or
+3. the traffic is best-effort by nature (debugging, sampling).
+
+| Signal | Choose |
+|--------|--------|
+| Every record must survive crashes | **Queue** — disk persistence, at-least-once redelivery |
+| Chain contains non-replayable steps (LLM enrichment, external calls with side effects) | **Queue** (durable path; `for_each` decodes once on each side) |
+| Cross-process or cross-machine delivery | **Queue** — lanes are in-process only |
+| Light, deterministic pre-processing between an in-process producer and disk | **Lane** (`lane_consumer` + `output_queue` durable bridge; `block` overflow) |
+| Webhook-style intake where the caller retries | **Lane** with `block` (backpressure surfaces as 429/slow response) or `drop` |
+| Observation taps, sampled debug feeds | **Lane** with `drop` |
+| Fan-out after a durable point | **Lane** with `block` — `output{lane:…}` branches after the offset-covered durable write |
+
+### Commit Correctness Across Lanes
+
+`output{lane:…}` is **not** a commit point; a consumer's offset only covers durable writes inside its own sub-chain. Three correct postures, by strength needed:
+
+| Posture | Shape | Guarantee |
+|---------|-------|-----------|
+| durable-first (default) | `output{queue:…}` inside the sub-chain (the commit covers it); the lane branch only does optional extras | Correct at zero cost — a crash replays from the queue |
+| ack receipts | `output{lane:…, ack: true}` settles into the context **AckSet**; the emitting consumer's commit gate waits for settlement (success → `output_queue`, failure → `dead_letter`, explicit drop — all count), undecided or past the consumer's `ack_timeout_ms` (default 30s) → no commit → the disk queue replays | Cross-lane at-least-once: never lost, possibly duplicated (make sinks idempotent — ES `_id` writes are) |
+| inline | When "commit only if processed" must be strict, don't decouple: run `for_each` with the durable write in the sub-chain | Strongest and simplest |
+
+## Observability
+
+`GET /pipeline/lanes` returns a live snapshot of every lane: `depth`, `capacity`, in/out counters (`emitted`, `delivered`), `drops`, `spills`, `blocked_waits` (how often `block` overflow actually waited), `owner`, and `closed` — the in-process counterpart of queue-depth observability.
diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md
index bffe0dd5a..157f56084 100644
--- a/docs/content.en/docs/references/pipeline.md
+++ b/docs/content.en/docs/references/pipeline.md
@@ -308,6 +308,65 @@ processor:
message: "non-POST request"
```
+### Multi-branch Dispatch with `case`
+
+Chaining `if`/`then`/`else` blocks gets unwieldy once the same input routes to three or more destinations. The `case` block is an ordered multi-way dispatch: the **first** matching branch runs (exactly one arm executes), and an optional `default` chain handles everything else.
+
+```yaml
+processor:
+ - case:
+ field: log_level
+ branches:
+ - values: [error, fatal]
+ then:
+ - echo:
+ message: "route to the alerting pipeline"
+ - values: [warn]
+ then:
+ - echo:
+ message: "route to the warning pipeline"
+ default:
+ - echo:
+ message: "keep on the default route"
+```
+
+Branch arms support two match styles, freely mixed within one block:
+
+| Field | Matches when |
+|-------|--------------|
+| `values` | The current value of the case block's `field` equals any entry (same semantics as the [`in`](conditions/in/) condition). Requires `field` on the case block. |
+| `when` | Any condition operator from the table above (`equals`, `regexp`, `and`, ...). |
+| both on one branch | Both must hold (AND). |
+| `then` | The processor (or list of processors) to run when the branch matches — required on every branch. |
+
+Branches without `values` do not need `field`, so a `case` block also works as a pure ordered condition ladder:
+
+```yaml
+processor:
+ - case:
+ branches:
+ - when:
+ and:
+ - equals:
+ env: prod
+ - regexp:
+ file.path: '\.json$'
+ then:
+ - echo:
+ message: "structured prod logs"
+ - when:
+ exists:
+ - file.path
+ then:
+ - echo:
+ message: "plain file logs"
+ default:
+ - echo:
+ message: "everything else"
+```
+
+Value resolution follows `if`/`then`/`else`: inside a per-record sub-chain (e.g. `for_each`) branches see the current record's fields; at pipeline level they read the context through the `_ctx.` prefix. Which arm ran is recorded in the flow trace as `case:1`, `case:2`, ..., `case:default`, or `case:no-match`.
+
## Error Handling
Pipeline error handling follows these rules:
@@ -446,8 +505,200 @@ import _ "your_app/plugins/health"
## Built-in Processors
-The framework ships with a growing set of built-in processors registered via `RegisterProcessorPlugin` / `RegisterDomainProcessor` — framework primitives (`echo`, `dag`, `for_each`), queue bridges (`consumer`, `queue_output`), Elasticsearch indexing (`bulk_indexing`, `json_indexing`, `indexing_merge`, `merge_to_bulk`), output transports (`otlp_export`), general-purpose helpers (`http`, `smtp`, `replay`), and a wide catalog of record processors for parsing, transforming, enriching, routing and governing records inside `for_each` sub-chains (`dissect`, `grok`, `mutate`, `geoip`, `throttle`, `script`, ...).
+The framework ships with a growing set of built-in processors registered via `RegisterProcessorPlugin` / `RegisterDomainProcessor` — framework primitives (`echo`, `dag`, `for_each`), queue bridges (`consumer`, `queue_output`, `lane_consumer`), Elasticsearch indexing (`bulk_indexing`, `json_indexing`, `indexing_merge`, `merge_to_bulk`, and the in-memory tail `merge_to_bulk_lane` + `bulk_indexing_lane`), unified output (`output` — exactly one of queue/lane), output transports (`otlp_export`), general-purpose helpers (`http`, `smtp`, `replay`), and a wide catalog of record processors for parsing, transforming, enriching, routing and governing records inside `for_each` sub-chains (`dissect`, `grok`, `mutate`, `geoip`, `throttle`, `script`, ...).
Every processor has its own reference page — description, configuration parameters with types and defaults, and an example — under the **Processor Reference** section. Record processors operate on the current record inside `for_each` sub-chains (most are no-ops outside a record scope); they are registered by domain category, the bare name is the YAML key, and `category:name` is an explicit alias.
The same catalog is discoverable at runtime via the `GET /pipeline/processors` API (`?grouped=1` groups the catalog by category).
+
+## Configuration Validation
+
+Invalid pipeline configs — unknown processors, missing required parameters, malformed DSL — are rejected **before** they are persisted or distributed, not discovered on the instance after deployment. `ValidatePipelineConfig` (`modules/pipeline/validate.go`) compiles the chain without running it and returns structured findings:
+
+```go
+type ConfigError struct {
+ Stage int `json:"stage"` // 0-based processor index; -1 = chain/config level
+ Processor string `json:"processor,omitempty"` // offending processor key
+ Field string `json:"field,omitempty"`
+ Message string `json:"message"`
+}
+```
+
+The compile pass covers:
+
+- **DSL shape** — exactly one action per step, well-formed `if`/`then`/`else` and `case`/`branches` blocks, nested sub-chains (`for_each`, `lane_consumer` inner chains compile recursively);
+- **Registry existence** — unknown processor names fail with the list of valid names;
+- **Parameter legality** — each constructor's own validation runs (missing required fields, bad enum values, out-of-range numbers);
+- **Record contracts** — placement mismatches surface as warnings, or as errors when the config opts in with `strict_contract: true`.
+
+Constructor panics are recovered per step and reported as validation errors — a broken plugin can never crash the gate.
+
+### Where the gate is enforced
+
+| Entry point | Behavior on invalid config |
+|-------------|---------------------------|
+| `POST /pipelines/` (create) | `400` with the findings; nothing persisted |
+| `PUT /pipelines/:id` (update) | `400`; the working config and its version history stay untouched |
+| `POST /pipeline/tasks/` (start task) | `400` — including disabled pipelines, which would otherwise defer the failure to enable-time |
+| `POST /pipelines/:id/versions/:version/rollback` | `400`; a version referencing a processor that was renamed or removed since must not be restored |
+| `POST /pipeline/deployments/` (Fleet) | `400`; the referenced pipeline must exist **and** compile (read failures reject too — fail-closed) |
+| Deployment reconciler | Refuses to distribute a config that stopped compiling (e.g. a plugin removed after an upgrade) |
+
+Rejection bodies carry both a standard error wrapper (UIs toast `error.reason`) and the structured detail:
+
+```json
+{
+ "error": { "reason": "invalid pipeline config: http processor requires at least one host (hosts) (+1 more)" },
+ "status": 400,
+ "errors": [
+ { "stage": 1, "processor": "http", "message": "http processor requires at least one host (hosts)" }
+ ],
+ "warnings": []
+}
+```
+
+### Pre-flight endpoint
+
+`POST /pipeline/_validate` compiles a submitted config without persisting anything. Always `200`; the `valid` flag carries the verdict:
+
+```bash
+curl -X POST /pipeline/_validate -H 'Content-Type: application/json' -d '{
+ "name": "my-pipeline",
+ "processor": [{ "dissect": { "pattern": "%{a} %{b}" } }]
+}'
+```
+
+```json
+{ "valid": true, "errors": null, "warnings": null }
+```
+
+### Constructor contract for processor authors
+
+The gate is only as strong as constructor validation. Two rules, enforced by the `TestEmptyConfigAudit` test:
+
+1. **Return errors, never panic** — a config problem is not a runtime fault; `panic` in a constructor crashes the host that compiles the chain.
+2. **Either require your inputs or default them safely** — every processor is constructed with an empty config in CI; one that succeeds must land in the audit's documented allowlist (44 entries, each annotated with why empty-accept is legitimate), otherwise it must fail with a clear required-field error. Variable templates such as `$[[env.HOSTS]]` remain non-empty strings at validation time and do not conflict with required-field checks.
+
+## Stream indexing tail: lanes vs disk bridge
+
+Compiled stream pipelines (LogPilot → Gateway) choose between two tail shapes via the stream spec `sink.transport`:
+
+| `sink.transport` | Shape | Durability |
+|------------------|-------|------------|
+| `""` / `"lane"` (default) | `consumer(staging)` → `for_each(parallel)` → `output{lane}` → `merge_to_bulk_lane{input_lane}` → `bulk_indexing_lane` → ES. The staging queue is the **only** disk write. | At-most-once: in-flight records are lost if the gateway exits. |
+| `"disk"` | `consumer(staging)` → `output{queue: indexing}` → `indexing_merge` → `bulk` queue → `bulk_indexing` → ES. | At-least-once: bulk payloads survive a gateway restart. |
+
+Both tails process records in parallel (`for_each.parallel`, `worker_size`); log workloads are order-insensitive.
+
+## Pipeline Configuration API
+
+Beyond YAML files, pipelines are managed as first-class resources (`PipelineConfigV2`) over HTTP:
+
+| Method & Path | Description |
+|---------------|-------------|
+| `POST /pipelines/` | Create a config (validated, versioned). |
+| `GET /pipelines/:id` / `PUT /pipelines/:id` / `DELETE /pipelines/:id` | Read / update (validated) / delete (archives the last version first). |
+| `POST /pipelines/_search` | Search configs. |
+| `POST /pipelines/:id/_start` / `_stop` | Start / stop the pipeline task. |
+| `GET /pipelines/:id/versions` | List versions. |
+| `GET /pipelines/:id/versions/:version` | Read one version snapshot. |
+| `POST /pipelines/:id/versions/:version/rollback` | Restore a version (validated; snapshots the current config first so rollbacks are themselves revertible). |
+
+Every update auto-snapshots the previous config as a `PipelineVersion` — there is no explicit publish step; each edit is a version.
+
+## Pipeline Deployment (Fleet)
+
+`PipelineDeployment` is the control-plane desired state: which pipeline runs on which instances. A reconciler (15s interval) converges desired → reported:
+
+1. Loads the pipeline config, re-validates it compiles, and renders it as **managed config** rows — one per target instance (`singleton` marks the config for instance-side lock resolution).
+2. Instances pull the managed config on their next `_sync`, write it to their config directory, and hot-reload the pipeline.
+3. Pushes are **converged**: when the rendered content is unchanged, the reconciler skips the version bump — instances don't rewrite files every cycle.
+
+### Placement scope (targeting)
+
+`placement` decides **where** the pipeline runs — resolved against the registered instance fleet on every cycle:
+
+```json
+{
+ "placement": {
+ "strategy": "all",
+ "selector": { "env": "prod" },
+ "instances": ["gw-2"]
+ }
+}
+```
+
+| Field | Semantics |
+|-------|-----------|
+| `selector` | Label match (AND across entries) — an instance is in scope when its labels carry every key=value. The reserved `group` key matches console-assigned group membership instead: values are comma-separated group names, any of which puts the instance in scope. |
+| `instances` | Explicit instance IDs (union with the selector match; unknown IDs are ignored until they register). |
+| *(neither set)* | Broadcast: a single `*` row every instance pulls. |
+
+Convergence is **exact** — the row set always equals the desired set:
+
+- Instances that fall out of every scope get their row **deleted**; their next sync reports the file `Deleted`, the config watcher removes it and stops the task. No duplicate deployments linger after a scope change A→B.
+- Label membership is re-evaluated every cycle: an instance that gains/loses a matching label is added/removed automatically.
+- Several deployments may share one pipeline; their target sets merge (an enabled deployment outranks a disabled one on the same instance, `singleton` wins over `all`). Any broadcast deployment subsumes targeting for that pipeline.
+- Row versions are monotonic **per config name** — the instance sync applies updates strictly by version (`newest wins`), so a migrated row always lands above what clients previously pulled.
+
+| Method & Path | Description |
+|---------------|-------------|
+| `POST /pipeline/deployments/` | Create (requires pipeline exists + compiles; strategy limited to `all`\|`singleton`). |
+| `GET /pipeline/deployments/` | List. |
+| `GET/PUT/DELETE /pipeline/deployments/:id` | Read / update (name, pipeline rebind, placement, enable — re-validated) / delete (cascades managed-config recall when no other deployment references the pipeline). |
+| `GET /pipeline/deployments/:id/status` | Desired-vs-reported status (`desired_count` is the resolved scope size). |
+
+### Runtime report and failure visibility
+
+Every managed instance reports its live pipeline states with each config sync (keyed by **pipeline name**). The report persists on the instance record and feeds the deployment status:
+
+```json
+{
+ "instances": [
+ {
+ "instance_id": "agent-1",
+ "state": "FAILED",
+ "reason": "the processor nope does not exist. valid processors: ...",
+ "last_seen": "2026-10-15T04:50:54Z"
+ }
+ ],
+ "desired_count": 1,
+ "ready_count": 0
+}
+```
+
+A pipeline whose config fails to compile **on the instance** (e.g. the instance's processor catalog differs from the control plane's — a plugin missing from that binary) is reported as `FAILED` with the compile error as `reason`, instead of silently disappearing from the report. Reports carry `"stale": true` when the instance still runs the pipeline but the placement scope no longer includes it (recall in flight, or a recall that failed). Instances without an ORM backend (lightweight agents) skip the reconciler side but still report runtime state.
+
+## Runtime APIs
+
+| Method & Path | Description |
+|---------------|-------------|
+| `GET /pipeline/tasks/` | Running pipeline tasks and states. |
+| `GET /pipeline/task/:id/stages` | Per-stage metrics: invocations, errors, latency. |
+| `GET /pipeline/task/:id/flight` | Sampled execution trace (`?trace_id=`, `?errors=1`, `&limit=`). |
+| `POST /pipeline/task/:id/_inject` | Inject a test sample (raw record or batch) into a running pipeline. |
+| `POST /pipeline/task/:id/_pause` / `_resume` | Pause / resume a task. |
+| `POST /pipeline/dry-run` | Unified dry-run: execute a processor chain against sample input with side-effect capture, field diffs, and scope/contract checks. |
+| `GET /pipeline/processors` | Processor catalog (`?grouped=1`). |
+| `GET /pipeline/codecs` | Registered record codecs. |
+| `GET /pipeline/lanes` | In-process lane channels: depth, emitted/delivered counters, drops, backpressure waits. |
+| `GET /pipeline/task/:id/stages?stage=` | Per-stage metrics for one stage. |
+| `DELETE /pipeline/task/:id` | Stop and remove a task (cancels consumers first, waits for release). |
+
+### Queue consumer management
+
+Queue consumers (offset registrations) are manageable per queue — stale consumer groups from removed pipelines are the typical cleanup target:
+
+| Method & Path | Description |
+|---------------|-------------|
+| `GET /queue/:id/consumer/:consumer_id/offset` | Read a consumer's committed offset. |
+| `PUT /queue/:id/consumer/:consumer_id/offset?offset=,` | Reset the offset to an explicit segment/position. Rewinds re-ingest (duplicates); jumps skip backlog. An actively committing consumer overwrites manual resets at its next commit cycle — pause the pipeline first for a sticky reset. |
+| `DELETE /queue/:id/consumer/:consumer_id` | Remove the consumer registration and its offset. Active consumers re-register at `0,0` (from head); stale registrations clear permanently. |
+
+Notes:
+
+- Address the **queue by its registry UUID** (the key in `/queue/stats`) — consumer/offset lookups walk the per-queue KV bucket and do not resolve queue names.
+- The consumer ID is the `id` field of the entry in `/queue/stats` (`?use_key=true&consumers=true`).
+- All three are surfaced in the Console queue browser per consumer (reset-to-earliest / reset-to-latest / delete).
+
+`GET /pipeline/cluster/overview` aggregates fleet-wide pipeline health for control-plane dashboards.
diff --git a/docs/content.en/docs/references/pipeline_record.md b/docs/content.en/docs/references/pipeline_record.md
index 8a385a42d..3e7023d2c 100644
--- a/docs/content.en/docs/references/pipeline_record.md
+++ b/docs/content.en/docs/references/pipeline_record.md
@@ -124,6 +124,40 @@ A sub-chain may itself contain another splitter bound to a different message fie
5. Record pass → set `RecordContextKey` (+ `FailureTagsKey` when tagging), run plain processors, apply `on_failure`.
6. Encode pass → honor `IsDropped`, persist failure tags (`Fields["tags"]`), re-encode, write back to the batch.
+## Record Contracts
+
+Processors may declare a `RecordContract` (`core/pipeline/contract.go`) — where they may run and what they need. The declaration is **compile-time**: placement is validated when the chain is built, never detected at runtime. Unconstrained processors (the entire historical inventory) implement nothing and pass every placement.
+
+```go
+type RecordContract struct {
+ Scope ScopeKind // ScopeRecord demands a record host; ScopeBatch demands the chain root
+ Namespace string // record family ("document", "attachment"); "" accepts any
+ RequiredFields []string // hard assertions on record paths ("document.content")
+ Granularity Granularity // execution granularity (see below)
+}
+
+// implement ContractProvider on the processor:
+func (p *MyProcessor) RecordContract() RecordContract {
+ return RecordContract{Scope: ScopeRecord, ...}
+}
+```
+
+### Granularity
+
+`Granularity` declares which execution channels a processor supports — the compiler routes by it and the catalog surfaces it for canvas UIs:
+
+| Value | Meaning |
+|-------|---------|
+| `record` (default) | Only the per-record path is meaningful (`grok`, `mutate`, ...). |
+| `batch_capable` | Batch and single-record paths are both valid; batch preferred when the host provides one. Implementing `BatchProcessor` implies the single-record fallback contract. |
+| `batch_only` | Only the batch channel is meaningful (cross-record aggregation). Must be a **direct** member of a record host's sub-chain — nesting under `if`/`then` is a compile error there, because the batch channel cannot reach it. |
+
+### Enforcement
+
+- **Placement** (`Scope`, `Granularity`) is checked by `NewPipelineWithScope` at compile time. By default a mismatch is a warning; the pipeline config's `strict_contract: true` turns it into a compile error, and the `POST /pipeline/_validate` gate reports it accordingly.
+- **`RequiredFields`** are data assertions — enforced by the runtime guard `CheckContract(ctx, contract)` (defense in depth for chains that bypass compile-time validation) and by dry-run sample validation, never by placement (data does not exist at compile time).
+- Record hosts (`for_each`, `lane_consumer`) pass their codec/lane namespace into the compile scope, so a processor requiring `Namespace: "document"` fails to compile inside a `codec: otel` chain.
+
## Compatibility Notes
- Sub-processors written against the original for_each (pre-convention) work unchanged: per-record `Process`, drop markers, and `ignore`-on-error semantics (error skips the rest of the sub-chain for that record) are preserved exactly.
diff --git a/docs/content.en/docs/references/processors/_index.md b/docs/content.en/docs/references/processors/_index.md
index 8787a9bb2..ac5854282 100644
--- a/docs/content.en/docs/references/processors/_index.md
+++ b/docs/content.en/docs/references/processors/_index.md
@@ -1,6 +1,7 @@
---
title: "Processor Reference"
weight: 16
+bookCollapseSection: true
---
# Processor Reference
@@ -15,6 +16,8 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea
| [`echo`](echo/) | Logs a configured message. |
| [`dag`](dag/) | Executes a DAG of processors with parallel and dependency-based execution. |
| [`for_each`](for_each/) | Splits a batch into records, runs a sub-chain per record, re-encodes the batch. |
+| [`if`](if/) | Conditional branch: matching records run the `then` sub-chain, the rest the optional `else`. |
+| [`case`](case/) | Ordered multi-way dispatch: the first matching branch runs, with an optional `default` chain. |
## Queue
@@ -22,6 +25,7 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea
|-----------|-------------|
| [`consumer`](consumer/) | Consumes a queue (by name or label selector) and exposes the batch to the pipeline. |
| [`queue_output`](queue_output/) | Appends the processed batch onto a target queue — the two-stage pipeline bridge. |
+| [`lane_consumer`](lane_consumer/) | Consumes an in-memory lane in competing micro-batches, runs a record sub-chain, optional durable bridge. |
## Elasticsearch
@@ -31,12 +35,15 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea
| [`json_indexing`](json_indexing/) | Indexes JSON documents into Elasticsearch. |
| [`indexing_merge`](indexing_merge/) | Merges small documents into combined indexing requests before writing. |
| [`merge_to_bulk`](merge_to_bulk/) | Merges events into Elasticsearch bulk requests. |
+| [`merge_to_bulk_lane`](merge_to_bulk_lane/) | Aggregates queue/lane input into bulk payloads written to an in-memory lane — the memory tail of the indexing pipeline. |
+| [`bulk_indexing_lane`](bulk_indexing_lane/) | Ships combined bulk payloads from an in-memory lane to Elasticsearch. |
## Output transport
| Processor | Description |
|-----------|-------------|
| [`otlp_export`](otlp_export/) | Ships the processed batch to any OTLP/gRPC collector. |
+| [`output`](output/) | Unified output step: writes to exactly one target — a durable queue (batch scope) or an in-memory lane (record scope). |
## General
diff --git a/docs/content.en/docs/references/processors/bulk_indexing_lane.md b/docs/content.en/docs/references/processors/bulk_indexing_lane.md
new file mode 100644
index 000000000..9de984bed
--- /dev/null
+++ b/docs/content.en/docs/references/processors/bulk_indexing_lane.md
@@ -0,0 +1,34 @@
+---
+title: "bulk_indexing_lane"
+weight: 306
+---
+
+# bulk_indexing_lane
+
+| Category | Scope |
+|----------|-------|
+| Elasticsearch | pipeline (consumer-form) |
+
+Consumes combined bulk payloads from an in-memory lane and ships them to Elasticsearch. Paired with [`merge_to_bulk_lane`](merge_to_bulk_lane/) — the send half of the all-memory indexing tail. One batch's payloads are sent concurrently across `worker_size` workers; the batch function waits for every send before returning, keeping receipt settlement exact.
+
+## Configuration
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `lane` | string | required | Lane carrying combined bulk payloads (from `merge_to_bulk_lane.output_lane`). |
+| `elasticsearch` | string | required | Cluster reference (id resolved Gateway-side; no credentials inlined). |
+| `worker_size` | int | | Send one batch's payloads concurrently across workers. |
+| `bulk` | object | | Bulk processor tuning: `batch_size_in_mb`, `batch_size_in_docs`, `retry_delay_in_seconds`, `max_reject_retry_times`, `request_timeout_in_second`, dead-letter queues. |
+| `failure_queue` | string | | Dead-letter queue for payloads that exhausted retries. |
+| `max_retries` | int | | Send retry attempts before dead-lettering. |
+| `retry_delay_in_ms` | int | | Delay between send retries. |
+
+## Example
+
+```yaml
+processor:
+ - bulk_indexing_lane:
+ lane: bulk-lane-
+ elasticsearch: cls-1
+ worker_size: 4
+```
diff --git a/docs/content.en/docs/references/processors/case.md b/docs/content.en/docs/references/processors/case.md
new file mode 100644
index 000000000..7f196c8b7
--- /dev/null
+++ b/docs/content.en/docs/references/processors/case.md
@@ -0,0 +1,100 @@
+---
+title: "case"
+weight: 208
+---
+
+# case
+
+| Category | Scope |
+|----------|-------|
+| Routing | record (sub-chain host) |
+
+Multi-way dispatch: the block declares an ordered list of `branches`, the **first** branch whose match holds runs its `then` sub-chain, and exactly one arm executes. An optional `default` chain handles records no branch matched. It replaces the `if`/`then`/`else` ladder once the same input has to route to three or more destinations. Unlike [`if`](if/), everything — `field`, `branches`, `default` — is nested under the `case` key.
+
+A branch matches on an enumerated value set (`values`, compared against the block's `field`), on a full [condition](../conditions/) (`when`), or on both (AND). Conditions use the same DSL as `if`.
+
+## Structure
+
+```yaml
+- case:
+ field: # optional; required when any branch uses `values`
+ branches:
+ - values: [, ] # match: field value equals any entry
+ then:
+ - ...
+ - when:
+ # match: any condition operator
+ then:
+ - ...
+ default: # optional; runs when no branch matched
+ - ...
+```
+
+| Key | Description |
+|-----|-------------|
+| `field` | Field read for `values` matching (record field inside a per-record sub-chain, `_ctx.`-prefixed context field at pipeline level). Required when any branch declares `values`; omit it for a pure condition ladder. |
+| `branches` | Ordered list of arms; at least one is required. Evaluation stops at the first match. |
+| `branches[].values` | Match when the `field` value equals any entry — the same typed membership test as the [`in`](../conditions/in/) condition. |
+| `branches[].when` | Match when the condition holds (`equals`, `regexp`, `range`, `and`, ...). |
+| `branches[].then` | Sub-chain for a matching record — a processor or a list of processors, required on every branch. |
+| `default` | Optional sub-chain for records no branch matched. Without it, unmatched records pass through unchanged. |
+
+Each branch needs `values` or `when`; declaring both means both must hold. Compile validation rejects a `case` with no branches, a branch without `then`, and a `values` branch in a block that declares no `field`.
+
+## Example
+
+Route by severity, with a prefix match for the `warn*` variants and a catch-all default:
+
+```yaml
+- case:
+ field: log_level
+ branches:
+ - values: [error, fatal]
+ then:
+ - mutate:
+ add:
+ route: alerting
+ - when:
+ prefix:
+ log_level: warn
+ then:
+ - mutate:
+ add:
+ route: warnings
+ default:
+ - mutate:
+ add:
+ route: archive
+```
+
+Branches without `values` do not need `field`, so the block also works as an ordered condition ladder:
+
+```yaml
+- case:
+ branches:
+ - when:
+ and:
+ - equals:
+ env: prod
+ - regexp:
+ file.path: '\.json$'
+ then:
+ - json:
+ field: message
+ - when:
+ exists:
+ - file.path
+ then:
+ - dissect:
+ field: message
+ pattern: "%{ts} %{level} %{msg}"
+```
+
+## Notes
+
+- Exactly one arm runs per record; after it completes the event returns to the main chain. A `terminal` processor inside an arm (e.g. [`drop_event`](drop_event/)) ends the event instead.
+- No match and no `default` is not an error — the record continues down the main chain untouched.
+- Value resolution follows `if`: inside a per-record sub-chain ([`for_each`](for_each/), [`lane_consumer`](lane_consumer/)) arms see the current record's fields; at pipeline level they read the context through the `_ctx.` prefix.
+- Arm chains compile under the enclosing scope, so record-scope contracts keep applying inside nested chains and batch-only processors stay rejected there, exactly as for `if`.
+- The arm that ran is recorded in the flow trace as `case:1`, `case:2`, ..., `case:default`, or `case:no-match`.
+- Sub-chain processors must be safe for concurrent `Process` calls when the surrounding `for_each` runs with `parallel > 1`.
diff --git a/docs/content.en/docs/references/processors/for_each.md b/docs/content.en/docs/references/processors/for_each.md
index 352962cea..76ae06bd3 100644
--- a/docs/content.en/docs/references/processors/for_each.md
+++ b/docs/content.en/docs/references/processors/for_each.md
@@ -14,6 +14,7 @@ weight: 103
| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `message_field` | string | "messages" | Context key holding the message batch. |
+| `parallel` | int | 1 | Runs the record sub-chain with N workers concurrently. Log workloads are order-insensitive — with a serial per-record cost of ~330µs, `parallel: 4` measured ~2× end-to-end throughput. Processors must be safe for concurrent `Process` calls on **distinct** records (they mutate the record, never shared state); every in-repo record processor satisfies this. Cancellation still propagates from the parent context. |
| `codec` | string | | Payload codec used to decode/encode records (default `otel`). |
| `on_failure` | string | | Sub-chain failure strategy: `ignore`, `tag` or `fail`. |
| `failure_tag` | string | "_processing_failed" | Tag appended to the record when `on_failure` is `tag`. |
diff --git a/docs/content.en/docs/references/processors/if.md b/docs/content.en/docs/references/processors/if.md
new file mode 100644
index 000000000..c5a3615fd
--- /dev/null
+++ b/docs/content.en/docs/references/processors/if.md
@@ -0,0 +1,51 @@
+---
+title: "if"
+weight: 207
+---
+
+# if
+
+| Category | Scope |
+|----------|-------|
+| Routing | record (sub-chain host) |
+
+Conditional branch: when the condition matches, records run the `then` sub-chain; otherwise the `else` sub-chain. After the branch completes, the event returns to the main chain. The condition body is written directly under the `if` key; `then`/`else` are sibling keys holding processor objects.
+
+Conditions use the [conditions DSL](../conditions/) — field comparisons, patterns, ranges, network membership, and logical `and`/`or`/`not` combinations, nestable arbitrarily.
+
+## Structure
+
+```yaml
+- if:
+ # any condition operator, see Conditions
+ then:
+ - ... # runs when the condition matches
+ else:
+ - ... # optional; runs when it does not
+```
+
+| Key | Description |
+|-----|-------------|
+| `` | Condition operator at the `if` key (`equals`, `regexp`, `range`, `and`, ...). |
+| `then` | Sub-chain for matching records. An empty `then` fails compile validation. |
+| `else` | Optional sub-chain for non-matching records. |
+| `when` | Per-processor conditional execution is the same mechanism declared on any processor (evaluates the condition, skips the step when it does not match). |
+
+## Example
+
+```yaml
+- if:
+ equals:
+ log_level: error
+ then:
+ - drop_event:
+ else:
+ - mutate:
+ add:
+ routed: non-error
+```
+
+## Notes
+
+- Branch execution returns to the main chain afterwards; a branch `terminal` processor (e.g. [`drop_event`](drop_event/)) ends the event instead.
+- Sub-chain processors must be safe for concurrent `Process` calls when the surrounding [`for_each`](for_each/) runs with `parallel > 1` (each processor only touches its own record).
diff --git a/docs/content.en/docs/references/processors/lane_consumer.md b/docs/content.en/docs/references/processors/lane_consumer.md
new file mode 100644
index 000000000..b461b4076
--- /dev/null
+++ b/docs/content.en/docs/references/processors/lane_consumer.md
@@ -0,0 +1,42 @@
+---
+title: "lane_consumer"
+weight: 211
+---
+
+# lane_consumer
+
+| Category | Scope |
+|----------|-------|
+| Queue | pipeline (batch host) |
+
+Consumes an in-memory lane in competing micro-batches and executes a record sub-chain per batch, stage-wise in written order. Records arrive as live pointers (no codec decode). Surviving records are optionally encoded once into an `output_queue` durable bridge — the memory-to-disk counterpart of `output{lane}`.
+
+Use it to parallelize or re-shape the tail of a lane pipeline: several `lane_consumer` steps (or multiple workers) compete on the same lane, each record is delivered to exactly one of them.
+
+## Configuration
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `lane` | string | required | Lane to consume. Must exist (`lane:` stanza or created by an upstream `output{lane}`). |
+| `batch_max` | int | 64 | Records per micro-batch (flush on size). |
+| `batch_flush_ms` | int | 100 | Flush an idle partial batch after this window. |
+| `on_failure` | string | "tag" | Sub-chain failure strategy: `tag` (default), `dead_letter:`, or `ignore`. |
+| `failure_tag` | string | "_processing_failed" | Tag appended to the record when `on_failure` is `tag`. |
+| `output_queue` | string | | Optional durable bridge: surviving records encode once into this queue. |
+| `output_codec` | string | | Override the codec used for the `output_queue` encode. |
+| `processor` | list (processor configs) | required | Record-scope sub-chain executed per batch, stage-wise in written order. |
+
+## Example
+
+Parallel aggregation tail: two workers compete on the raw lane, each aggregates bulk payloads and hands them to the send lane (paired with [`bulk_indexing_lane`](bulk_indexing_lane/)):
+
+```yaml
+processor:
+ - lane_consumer:
+ lane: ingest-lane-
+ batch_max: 64
+ batch_flush_ms: 100
+ processor:
+ - merge_to_bulk:
+ output_queue: { name: bulk- }
+```
diff --git a/docs/content.en/docs/references/processors/merge_to_bulk_lane.md b/docs/content.en/docs/references/processors/merge_to_bulk_lane.md
new file mode 100644
index 000000000..3017939ce
--- /dev/null
+++ b/docs/content.en/docs/references/processors/merge_to_bulk_lane.md
@@ -0,0 +1,47 @@
+---
+title: "merge_to_bulk_lane"
+weight: 305
+---
+
+# merge_to_bulk_lane
+
+| Category | Scope |
+|----------|-------|
+| Elasticsearch | pipeline (consumer-form) |
+
+Reads documents from an `input_queue` (disk) **or** an `input_lane` (memory), aggregates them into combined Elasticsearch bulk payloads, and writes the payloads into an in-memory lane. Paired with [`bulk_indexing_lane`](bulk_indexing_lane/) — together they replace the disk-queue bridge (`indexing_merge` + `bulk` queue + `bulk_indexing`) with an all-memory tail: the staging queue is the only disk write. In-flight payloads are lost if the node exits; the lane applies block-on-full backpressure instead of unbounded buffering.
+
+Exactly one of `input_queue` / `input_lane` must be set. The output lane is auto-created (block-on-full, capacity 64) when no `lane:` stanza defines it — define an explicit stanza to tune the memory budget (capacity × `bulk_size_in_mb` is the worst case).
+
+## Configuration
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `input_queue` | string | | Disk queue to read documents from. |
+| `input_lane` | string | | Lane to read records from — each record's fields are serialized to a document. Exactly one of `input_queue` / `input_lane`. |
+| `output_lane` | string | required | Lane receiving the combined bulk payloads. |
+| `index_name` | string | required | Target index / data stream name written into the bulk action line. |
+| `type_name` | string | | `_type` for clusters that still require it (defaults from the target cluster's doc type). |
+| `write_op_type` | string | "index" | Bulk action (`index` / `create`). Data streams only accept `create`. |
+| `key_field` / `key_fields` | string / list | | Deterministic `_id`: fingerprint of the given field(s) makes replays land on version conflicts instead of duplicates. Without a file source, falls back to hashing the whole document. |
+| `bulk_size_in_mb` | int | 10 | Flush the aggregate payload at this size. |
+| `bulk_size_in_kb` | int | | Size trigger in KB (overrides MB). |
+| `idle_timeout_in_seconds` | int | 5 | Flush a partial payload after this idle window. |
+| `worker_size` | int | 1 | Competing aggregation workers on the input (lane input: workers subscribe in parallel; queue input: workers pop concurrently). |
+| `elasticsearch` | string | | Cluster reference — only used to default `type_name`; the merge never talks to Elasticsearch. |
+| `failure_queue` | string | `_failure` | Dead-letter queue for failed payloads. |
+
+## Example
+
+Lane-to-lane aggregation tail (memory-only between staging and the send lane):
+
+```yaml
+processor:
+ - merge_to_bulk_lane:
+ input_lane: ingest-lane-
+ output_lane: bulk-lane-
+ index_name: nginx2
+ write_op_type: create
+ worker_size: 2
+ elasticsearch: cls-1
+```
diff --git a/docs/content.en/docs/references/processors/output.md b/docs/content.en/docs/references/processors/output.md
new file mode 100644
index 000000000..0d116cc52
--- /dev/null
+++ b/docs/content.en/docs/references/processors/output.md
@@ -0,0 +1,49 @@
+---
+title: "output"
+weight: 210
+---
+
+# output
+
+| Category | Scope |
+|----------|-------|
+| Output transport | batch (queue target) · record (lane target) |
+
+Unified output step: writes the processed batch or record to **exactly one** target — a durable queue or an in-memory lane. Config unpacking rejects zero or multiple targets. `queue_output` is a registered alias for the queue form.
+
+## Configuration
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `queue` | string or object | | Target queue: bare name, or `{ name, label }` object. Queue and lane are mutually exclusive. |
+| `lane` | string | | Target lane name. Record scope only — the lane carries live event pointers. Auto-created (block-on-full, default capacity) when no `lane:` stanza defines it. |
+| `ack` | bool | | Lane target only: attach an acknowledgment receipt to the context's AckSet, so the emitting consumer's commit gate waits for settlement (cross-lane at-least-once). |
+| `codec` | string | | Queue target: override the codec resolution. |
+| `message_field` | string | "messages" | Batch scope (queue target): context key holding the batch. |
+
+## Scope rules
+
+| Target | Scope | Notes |
+|--------|-------|-------|
+| `queue` | batch (top level) | Encodes the batch through the 3-level codec resolution. |
+| `lane` | record (inside `for_each` / `lane_consumer`) | Placing it at the consumer top level errors per batch with "requires record scope". |
+
+## Example
+
+Lane hand-off — the main pipeline writes processed records into an in-memory lane instead of a second disk queue (staging stays the only disk write):
+
+```yaml
+processor:
+ - consumer:
+ queue_selector:
+ keys: ["staging-"]
+ message_field: messages
+ processor:
+ - for_each:
+ message_field: messages
+ parallel: 4
+ processor:
+ - grok: { field: message, patterns: ["..."] }
+ - output:
+ lane: ingest-lane-
+```
diff --git a/docs/content.en/docs/references/processors/tap.md b/docs/content.en/docs/references/processors/tap.md
new file mode 100644
index 000000000..652e9e91d
--- /dev/null
+++ b/docs/content.en/docs/references/processors/tap.md
@@ -0,0 +1,30 @@
+---
+title: "tap"
+weight: 212
+---
+
+# tap
+
+| Category | Scope |
+|----------|-------|
+| General | record |
+
+Side-channel sampler: copies a sampled fraction of records to a debug queue or lane. The main flow is never affected — it is an in-flight **copy**, not a divert. Sanctioned side-channel for debug indices, shadow runs and observation sampling. Exactly one target (`queue` or `lane`).
+
+## Configuration
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `sample_ratio` | float | 1.0 | Fraction of records copied, within (0, 1]. `1.0` copies everything. |
+| `queue` | object | | Side target queue (`{ name }`). Mutually exclusive with `lane`. |
+| `lane` | string | | Side target lane. Mutually exclusive with `queue`. |
+
+## Example
+
+Copy 10% of records into a debug lane:
+
+```yaml
+- tap:
+ sample_ratio: 0.1
+ lane: debug-samples
+```
diff --git a/docs/content.en/docs/references/websocket_reverse.md b/docs/content.en/docs/references/websocket_reverse.md
index 20fb6e6a6..e9c3a62c4 100644
--- a/docs/content.en/docs/references/websocket_reverse.md
+++ b/docs/content.en/docs/references/websocket_reverse.md
@@ -210,6 +210,7 @@ responses. Agent implementations should always terminate via
The reverse channel is a **transport**, not an authenticator. Compose it with the framework's existing security layers:
- **Control plane → agent authorization**: the agent's own API authentication applies to the proxied request. The `AccessToken` field carries the caller's bearer token end-to-end, so the agent's API layer can validate it exactly as if the call were direct.
+- **Loopback token (managed agents)**: the framework's reverse client (`modules/managed/reverseclient`) executes proxied requests as loopback HTTP calls against the agent's own web port, authenticated with an `AGENT_API_ACCESS_TOKEN` from the keystore. When no token is provisioned — or the stored one has no matching record in the security KV bucket (stale from a wiped data dir or an earlier build) — the client mints one automatically: an `AccessToken` carrying every instance-ops permission plus all registered permissions, never expiring, cached in the keystore for restart reuse. The declared instance-ops permission keys are registered at reverse-channel setup, so route-level permission resolution succeeds even though those keys are constants that no route registers itself.
- **Agent → control plane**: the outbound websocket rides the agent's existing authenticated channel to the control plane (e.g. Console's managed websocket with instance credentials).
- **Session binding**: the handshake check (`RegisterPendingSession` → `HandleHelloPayload`) prevents an agent from claiming another peer's session slot.
diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md
index a2c7e68ff..57b9651b3 100644
--- a/docs/content.en/docs/release-notes/_index.md
+++ b/docs/content.en/docs/release-notes/_index.md
@@ -11,7 +11,15 @@ Information about release notes of INFINI Framework is provided here.
## Latest (In development)
### ❌ Breaking changes
- `ReadGuard` rules now interpret glob metacharacters. Literal names containing these characters require escaping or singleton character groups. On Unix, literal backslashes also require escaping. #434
+- pipeline configs are now validated at compile time before they are persisted or distributed: unknown processors, missing required fields, constructor errors and scope-violating sub-chains are rejected up front instead of surfacing at runtime, and every registered processor constructor is audited to return an error (never panic) on an empty config. #433
### 🚀 Features
+- feat(pipeline): unified pipeline engine foundation — record contracts declared per processor (`ContractProvider` with `Scope`/`Namespace`/`Granularity`, enforced by `NewPipelineWithScope`, `batch_only` processors restricted to direct members of record hosts), payload-agnostic record codecs with namespace declaration and in-place upgrade registration (`RegisterRecordCodecOrReplace`, `ListRecordCodecs`), in-memory lanes with self-healing `lane.GetOrCreate`, and per-step stats (`Context.StepStat`) for sequential runs feeding document-level timelines. #433
+- feat(pipeline): distributed deployment management — `PipelineConfigV2`/`PipelineDeployment` ORM models and a deployment reconciler that pushes configs to instances through configs-center: instance-targeted selectors with exact per-instance convergence, disabling a deployment propagates to instances, deleting one cascades config recall, name-based status matching, and instance runtime reporting in `POST /configs/_sync`. #433
+- feat(pipeline): pipeline rollback gate — a config distribution that fails on an instance rolls the deployment back instead of leaving instances half-applied, with instance-side failure visibility in the deployment status. #433
+- feat(pipeline): lane-based bulk indexing, the memory-path twin of the disk-queue bulk pipeline — `merge_to_bulk_lane` aggregates queue/lane input into combined bulk payloads written to an in-memory lane, `bulk_indexing_lane` ships them to Elasticsearch and settles the batch receipt on success. #433
+- feat(pipeline): add the `case` processor — ordered multi-way dispatch for enumerated routing: the first matching branch runs (exactly one arm executes) with an optional `default` chain, unmatched records pass through unchanged when no `default` is set; arms match on `values` (typed `in`-style membership against the block's `field`) or on any condition (`when`), combinable with AND, and evaluate against the current record inside per-record sub-chains. #433
+- feat(configs): per-instance config rows shadow group defaults, enabling edge-rule variants per instance without forking the group config; heartbeat upserts preserve server-owned display metadata (`alias`) and group assignments. #433
+- feat(queue): disk-queue consumer self-heal — offsets parked at a missing write head re-open the segment (or keep waiting for it) instead of erroring forever, so consumers closing their lane on exit no longer wedge producers. #433
- feat: support glob rules in `ReadGuard` to authorize matching directories without enumerating the filesystem. Rules support relative paths and path component cleanup, with documented escaping and symbolic-link behavior. #434
- feat: add MCP (Model Context Protocol) server support — auto-expose UI routes as MCP tools via `api.MCPTool()` annotation
- feat: add MCP configuration section (`web.mcp.enabled`, `web.mcp.base_path`, `web.mcp.stateless`, `web.mcp.disable_streaming`)
@@ -40,6 +48,9 @@ Information about release notes of INFINI Framework is provided here.
- perf(pipeline): probe the current record via `Parameters.Peek` — the `GetValue` miss path built a stack-capturing error for every `CurrentRecord` call outside a record scope (µs + 6 allocs per if/then/else evaluation); the bare locked map read restores the no-record path to pre-fix allocation counts #408
### 🐛 Bug fix
+- fix(pipeline): the unified `output` processor accepts a bare queue name or a full config object. #433
+- fix(agent): mint the reverse-channel loopback API token agents were missing, so agent-initiated loopback calls authenticate again. #433
+- fix(agent): skip the deployment reconcile loop and guard ORM access on lightweight agent builds that carry no metadata store. #433
- fix(queue): `disk_queue.max_used_bytes` now counts only `data/queue`, so unrelated application data no longer blocks queue writes or prevents startup. #431
- fix: expand configs.template when loading templated config files #391
- fix: register elasticsearch instance even when version probe fails #393
@@ -57,7 +68,7 @@ Information about release notes of INFINI Framework is provided here.
- refactor: rate limit the large cluster warning in metric collection #394
- docs: catalog built-in processors and condition operators in the pipeline/conditions references #414
- docs: per-processor and per-condition reference sections — one page for each of the 51 registered processors and 16 condition operators, in a uniform format (category/scope, configuration parameters with types and defaults, examples) #414
-- fixdoc: replace relref shortcodes embedded in link destinations with plain relative page links, silencing the docs build's "Page 'HAHAHUGOSHORTCODE-...' not found" warnings #415
+- fixdoc: replace relref shortcodes embedded in link destinations with plain relative page links, silencing the docs build's broken-relref "Page not found" warnings #415
## 1.4.2 (2026-06-23)
### ❌ Breaking changes
diff --git a/lib/go-ucfg/types.go b/lib/go-ucfg/types.go
index af3315d2e..1c833f7d6 100755
--- a/lib/go-ucfg/types.go
+++ b/lib/go-ucfg/types.go
@@ -376,7 +376,13 @@ func (c cfgSub) reify(opts *options) (interface{}, error) {
// preserve empty arrays
return []interface{}{}, nil
case len(fields) == 0 && len(arr) == 0:
- return nil, nil
+ // preserve empty dicts: returning nil here used to drop the KEY
+ // entirely when the parent unpacked into map[string]interface{}
+ // (reflect.ValueOf(nil) is invalid, so reifyMap skipped
+ // SetMapIndex) — pipeline configs like `processor: [{echo: {}}]`
+ // lost the processor name and failed to load. An empty dict must
+ // reify to an empty map, mirroring the empty-array case above.
+ return map[string]interface{}{}, nil
case len(fields) > 0 && len(arr) == 0:
m := make(map[string]interface{})
for k, v := range fields {
diff --git a/lib/go-ucfg/unpack_test.go b/lib/go-ucfg/unpack_test.go
index 6acf10742..f7d7dad18 100755
--- a/lib/go-ucfg/unpack_test.go
+++ b/lib/go-ucfg/unpack_test.go
@@ -237,3 +237,40 @@ func TestReifyUnpackersPtr(t *testing.T) {
assertSubConfig(to.C.c)
assertSubConfig(to.R.c)
}
+
+// An empty dict value must keep its key when unpacked into a generic
+// map[string]interface{}: `processor: [{echo: {}}]` losing `echo` broke
+// every config-file-loaded pipeline with an empty processor body.
+func TestUnpackPreservesEmptyDictKeys(t *testing.T) {
+ cfg, err := NewFrom(map[string]interface{}{
+ "processor": []interface{}{
+ map[string]interface{}{"echo": map[string]interface{}{}},
+ map[string]interface{}{"node_stamp": map[string]interface{}{"field": "x"}},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ type raw struct {
+ Processor []map[string]interface{} `config:"processor"`
+ }
+ out := raw{}
+ if err := cfg.Unpack(&out); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(out.Processor) != 2 {
+ t.Fatalf("expected 2 processors, got %d", len(out.Processor))
+ }
+ echoCfg, exists := out.Processor[0]["echo"]
+ if !exists {
+ t.Fatalf("empty dict key dropped: %v", out.Processor[0])
+ }
+ if m, ok := echoCfg.(map[string]interface{}); !ok || len(m) != 0 {
+ t.Fatalf("expected echo to be an empty map, got %#v", echoCfg)
+ }
+ if _, exists := out.Processor[1]["node_stamp"]; !exists {
+ t.Fatalf("node_stamp key dropped: %v", out.Processor[1])
+ }
+}
diff --git a/lib/go-ucfg/yaml/yaml_test.go b/lib/go-ucfg/yaml/yaml_test.go
index 4c4ff020b..18d29af3c 100755
--- a/lib/go-ucfg/yaml/yaml_test.go
+++ b/lib/go-ucfg/yaml/yaml_test.go
@@ -134,9 +134,15 @@ func TestEmptyCollections(t *testing.T) {
to: &struct {
A interface{}
}{},
+ // an empty mapping reifies to an empty map (mirroring `a: []`
+ // → []interface{}{}), NOT nil: nil made the KEY disappear when
+ // the parent unpacked into map[string]interface{}, breaking
+ // configs like `processor: [{echo: {}}]`
want: &struct {
A interface{}
- }{},
+ }{
+ A: map[string]interface{}{},
+ },
},
"struct with empty map into struct with map of interfaces": {
input: "a: {}",
diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go
index af662d486..7ec8aacfa 100644
--- a/modules/configs/client/client.go
+++ b/modules/configs/client/client.go
@@ -53,6 +53,31 @@ import (
"infini.sh/framework/modules/configs/config"
)
+// SetPipelineRuntimeSnapshotProvider wires the pipeline module's snapshot
+// into the sync client (called from PipeModule.Setup).
+func SetPipelineRuntimeSnapshotProvider(fn func() map[string]interface{}) {
+ getPipelineRuntimeSnapshot = fn
+}
+
+// SetInstanceFlagsProvider wires instance-level maintenance flags into the
+// sync client's runtime report (called from PipeModule.Setup; the pipeline
+// module owns the cordon flag).
+func SetInstanceFlagsProvider(fn func() map[string]interface{}) {
+ getInstanceFlags = fn
+}
+
+var getInstanceFlags func() map[string]interface{}
+
+// SetStageStatsProvider wires the pipeline module's per-stage counters
+// into the sync report (called from PipeModule.Setup). The manager
+// persists them so cluster aggregation covers instances that do not
+// share its datastore.
+func SetStageStatsProvider(fn func() []common.StageStat) {
+ getStageStats = fn
+}
+
+var getStageStats func() []common.StageStat
+
const bucketName = "instance_registered"
const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS"
const legacyManagedRegisterCompatMaxVersion = "1.30.4"
@@ -429,6 +454,12 @@ func ListenConfigChanges() error {
req.Configs = cfgs
req.Hash = util.MD5digestString(util.MustToJSONBytes(cfgs))
+ // runtime pipeline state: report which tasks are running
+ // so the deployment reconciler can compute health status
+ if snapshot := runtimePipelineSnapshot(); snapshot != nil {
+ req.Runtime = snapshot
+ }
+
//fetch configs from manager
request := util.Request{Method: util.Verb_POST}
request.ContentType = "application/json"
@@ -576,3 +607,43 @@ func saveKeystore(k string, v string) error {
err = ks.Save()
return err
}
+
+// runtimePipelineSnapshot builds the RuntimeReport for the sync request.
+// It delegates to the PipeModule if loaded; nil when not (agent-only etc).
+func runtimePipelineSnapshot() *common.RuntimeReport {
+ snapshot := getPipelineRuntimeSnapshot()
+ if snapshot == nil {
+ return nil
+ }
+ report := &common.RuntimeReport{Pipelines: map[string]common.PipelineRuntimeState{}}
+ if flags := getInstanceFlags(); flags != nil {
+ if v, ok := flags["cordoned"].(bool); ok {
+ report.Cordoned = v
+ }
+ }
+ for name, state := range snapshot {
+ m, _ := state.(map[string]interface{})
+ stateStr, _ := m["state"].(string)
+ source, _ := m["config_source"].(string)
+ reason, _ := m["reason"].(string)
+ digest, _ := m["config_digest"].(string)
+ report.Pipelines[name] = common.PipelineRuntimeState{
+ State: stateStr,
+ ConfigSource: source,
+ ConfigDigest: digest,
+ Reason: reason,
+ }
+ }
+ // per-stage counters ride the same heartbeat — cumulative rows the
+ // manager snapshots; zero rows are noise, skip them
+ if getStageStats != nil {
+ if rows := getStageStats(); len(rows) > 0 {
+ report.StageStats = rows
+ }
+ }
+ return report
+}
+
+// getPipelineRuntimeSnapshot is set by the pipeline module on startup;
+// nil when the pipeline module is not loaded.
+var getPipelineRuntimeSnapshot func() map[string]interface{}
diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go
index 46ecbbcaf..bf49380b3 100644
--- a/modules/configs/common/domain.go
+++ b/modules/configs/common/domain.go
@@ -83,6 +83,59 @@ type ConfigSyncRequest struct {
Hash string `json:"hash"`
Client model.Instance `json:"client"`
Configs ConfigList `json:"configs"`
+
+ // Runtime reports the instance's live pipeline state (which tasks are
+ // running, what config version they loaded). The deployment reconciler
+ // reads this to compute DeploymentStatus (desired vs reported).
+ Runtime *RuntimeReport `json:"runtime,omitempty"`
+}
+
+// RuntimeReport is the pipeline execution state an instance reports on
+// every sync — the "reported" half of the reconcile loop.
+type RuntimeReport struct {
+ // Pipelines maps task name → state + the managed-config version the
+ // instance last loaded (0 = unknown/not from managed config).
+ Pipelines map[string]PipelineRuntimeState `json:"pipelines,omitempty"`
+ // Cordoned mirrors the instance-local maintenance flag (#121/#128):
+ // running tasks drain but nothing starts. Reported so the control
+ // plane can see WHO is cordoned without a proxy round-trip — the
+ // placement side keeps configs flowing (a cordoned instance still
+ // syncs; recall-on-cordon would strand its running tasks).
+ Cordoned bool `json:"cordoned,omitempty"`
+ // StageStats carries the instance's cumulative per-stage counters on
+ // the same heartbeat — the same rows the local metrics exporter
+ // persists. Instances without a shared datastore (gateway-type, local
+ // sqlite) still appear in the manager's cluster aggregation.
+ StageStats []StageStat `json:"stage_stats,omitempty"`
+}
+
+// StageStat is one per-stage counter row in the sync report — the wire
+// shape of the pipeline module's StageStats (cumulative counters since
+// process start; the manager persists them as pipeline-stage-stats
+// snapshots and the cluster fold takes each instance's latest tick).
+type StageStat struct {
+ Pipeline string `json:"pipeline"`
+ InstanceID string `json:"instance_id"`
+ Stage string `json:"stage"` // processor name or lane name
+ Carrier string `json:"carrier"` // "processor" | "lane:"
+ Count int64 `json:"count"`
+ Errors int64 `json:"errors"`
+ AvgLatency int64 `json:"avg_latency_ns"`
+}
+
+type PipelineRuntimeState struct {
+ State string `json:"state"` // STARTED|STOPPED|FAILED|...
+ ConfigSource string `json:"config_source,omitempty"` // managed config name
+ ConfigVersion int64 `json:"config_version,omitempty"` // managed config version loaded
+ // ConfigDigest is the running task's content digest (the six fields a
+ // deployment renders). The control plane compares it against the digest
+ // it pushed to verify rollout readiness — state alone cannot prove the
+ // instance runs the NEW content (#125).
+ ConfigDigest string `json:"config_digest,omitempty"`
+ // Reason explains a FAILED state (e.g. the config references a
+ // processor missing from this instance's catalog) — surfaced in the
+ // deployment status so operators see why without instance shell access
+ Reason string `json:"reason,omitempty"`
}
type ConfigSyncResponse struct {
diff --git a/modules/configs/config/config.go b/modules/configs/config/config.go
index 8dff7780d..af39787b1 100755
--- a/modules/configs/config/config.go
+++ b/modules/configs/config/config.go
@@ -7,11 +7,11 @@ import (
"fmt"
log "github.com/cihub/seelog"
"infini.sh/framework/core/api"
- "infini.sh/framework/core/security"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/env"
"infini.sh/framework/core/errors"
"infini.sh/framework/core/global"
+ "infini.sh/framework/core/security"
"infini.sh/framework/core/util"
"infini.sh/framework/core/util/file"
"infini.sh/framework/modules/configs/common"
@@ -178,6 +178,17 @@ func deleteConfigAction(w http.ResponseWriter, req *http.Request, params httprou
var versionRegexp = regexp.MustCompile(`#MANAGED_CONFIG_VERSION:\s*(?P\d+)`)
var managedRegexp = regexp.MustCompile(`#MANAGED:\s*(?P\w+)`)
+// managedMarkerLine matches whole trailer lines (`#MANAGED...`) so the
+// content digest can exclude client-local metadata stamps.
+var managedMarkerLine = regexp.MustCompile(`(?m)^\s*#MANAGED(_CONFIG_VERSION)?:[^\n]*$`)
+
+// stripManagedMarkers removes the managed trailer lines: the config server
+// digests the raw row content, while the local file carries the version/
+// managed stamps the client appends — hashing the stamped bytes would make
+// every managed file look permanently changed (rewrite loop each sync).
+func stripManagedMarkers(input string) string {
+ return strings.TrimSpace(managedMarkerLine.ReplaceAllString(input, ""))
+}
func parseConfigVersion(input string) int64 {
matches := versionRegexp.FindStringSubmatch(util.TrimSpaces(input))
@@ -313,6 +324,12 @@ func GetConfigFromFile(cfgDir, filename string) (*common.ConfigFile, error) {
Updated: f.ModTime().Unix(),
Size: f.Size(),
}
+ // Content digest: the config server diffs by hash when present —
+ // per-row version counters aren't comparable across the same-name rows
+ // an instance may be assigned (instance variant vs group base). The
+ // digest must match the server's (raw row content): strip the managed
+ // trailer lines first.
+ cfg.Hash = util.MD5digest(stripManagedMarkers(content))
cfg.Content = content
cfg.Managed = parseConfigManaged(content, cfg.Version > 0 || global.Env().SystemConfig.Configs.ConfigFileManagedByDefault)
diff --git a/modules/configs/config/config_test.go b/modules/configs/config/config_test.go
index 4e46f283d..eb38875c4 100644
--- a/modules/configs/config/config_test.go
+++ b/modules/configs/config/config_test.go
@@ -77,3 +77,23 @@ func TestSaveConfigStr(t *testing.T) {
t.Fatalf("Failed to remove test file: %v", err)
}
}
+
+// TestStripManagedMarkersSymmetry — the hash computed after the client
+// strips the marker lines must equal the server-side one (raw lines,
+// TrimSpace'd); otherwise every sync cycle would see a content change and
+// rewrite the file.
+func TestStripManagedMarkersSymmetry(t *testing.T) {
+ row := "{\"pipeline\":[{\"name\":\"x\"}]}"
+ stamped := row + "\n#MANAGED_CONFIG_VERSION: 15\n#MANAGED: true"
+ if got := stripManagedMarkers(stamped); got != row {
+ t.Fatalf("strip = %q, want %q", got, row)
+ }
+ // marker glued onto the row (historic stacking): the whole line goes
+ glued := row + "\n#MANAGED: trueMANAGED_CONFIG_VERSION: 15"
+ if got := stripManagedMarkers(glued); got != row {
+ t.Fatalf("strip glued = %q, want %q", got, row)
+ }
+ if stripManagedMarkers(row) != row {
+ t.Fatal("bare content must pass through")
+ }
+}
diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go
index d639e65b7..1edcdf5bc 100644
--- a/modules/configs/server/server.go
+++ b/modules/configs/server/server.go
@@ -44,6 +44,17 @@ import (
"infini.sh/framework/modules/configs/common"
)
+// persistStageStatsSink stores an instance's reported per-stage counters.
+// Registered by the pipeline module (which owns the pipeline-stage-stats
+// schema); nil on apps that don't load it.
+var persistStageStatsSink func(octx *orm.Context, rows []common.StageStat)
+
+// SetStageStatsSink installs the reported-stage-stats persistence hook
+// (called from the pipeline module's Setup).
+func SetStageStatsSink(fn func(octx *orm.Context, rows []common.StageStat)) {
+ persistStageStatsSink = fn
+}
+
// Config holds the server-side settings (configs.server.* in YAML).
// Disabled by default; products opt in via Setup().
type Config struct {
@@ -436,6 +447,24 @@ func (h *APIHandler) registerInstance(w http.ResponseWriter, req *http.Request,
h.WriteJSON(w, resp, http.StatusOK)
}
+// normalizeVersionDates blanks client-reported version date placeholders
+// ("N/A", "unknown", ...) that are not RFC3339 dates. Indices created
+// before the keyword mapping pin carry a dynamic `date` mapping for
+// version.build_date/eol_date (the first writer sent real ISO dates), and
+// ES rejects placeholder strings with document_parsing_exception — which
+// silently killed every heartbeat upsert for instances built without
+// full version metadata (plain `go build`, no ldflags).
+func normalizeVersionDates(app *env.Application) {
+ for _, d := range []*string{&app.Version.BuildDate, &app.Version.EolDate} {
+ if *d == "" {
+ continue
+ }
+ if _, err := time.Parse(time.RFC3339, *d); err != nil {
+ *d = ""
+ }
+ }
+}
+
// upsertInstance persists/refreshes the registration. Returns true when
// the instance was newly created.
func upsertInstance(instance *model.Instance) (bool, error) {
@@ -449,6 +478,8 @@ func upsertInstance(instance *model.Instance) (bool, error) {
// not-found is the normal first-registration path, not an error
exists = err == nil && exists
+ normalizeVersionDates(&instance.Application)
+
now := strconv.FormatInt(time.Now().UnixMilli(), 10)
if instance.Labels == nil {
instance.Labels = map[string]string{}
@@ -472,6 +503,10 @@ func upsertInstance(instance *model.Instance) (bool, error) {
// Groups are server-owned (UI-managed targeting): the heartbeat
// payload does not carry them — keep the stored value.
instanceCopy.Groups = existing.Groups
+ // Alias/Description are server-owned display metadata (UI-managed):
+ // keep the stored value across heartbeats.
+ instanceCopy.Alias = existing.Alias
+ instanceCopy.Description = existing.Description
// Heartbeat syncs carry no access token; keep the registered one
// (wiping it here would 401 the very next sync).
if instanceCopy.AccessToken == nil {
@@ -528,8 +563,39 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt
// Heartbeat: refresh the instance record on every sync.
if _, err := upsertInstance(&obj.Client); err != nil {
- log.Debugf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err)
- }
+ log.Warnf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err)
+ }
+
+ // Runtime report: persist the instance's pipeline state for the
+ // deployment reconciler (desired-vs-reported comparison). The
+ // cordon flag rides the same report (#128) — the upsert must run
+ // even when no pipelines are running, or the un-cordon transition
+ // would never reach the control plane.
+ if obj.Runtime != nil {
+ updated := false
+ if len(obj.Runtime.Pipelines) > 0 {
+ rtJSON, rerr := json.Marshal(obj.Runtime.Pipelines)
+ if rerr == nil {
+ obj.Client.RuntimePipelines = string(rtJSON)
+ updated = true
+ }
+ }
+ if obj.Runtime.Cordoned != obj.Client.Cordoned {
+ obj.Client.Cordoned = obj.Runtime.Cordoned
+ updated = true
+ }
+ if updated {
+ _, _ = upsertInstance(&obj.Client) // second upsert with runtime
+ }
+
+ // Stage stats: the instance's cumulative counters snapshot into
+ // the manager's store — fleet-wide cluster aggregation without
+ // every instance sharing the datastore. The pipeline module
+ // registers the sink; nil on apps without it.
+ if persistStageStatsSink != nil && len(obj.Runtime.StageStats) > 0 {
+ persistStageStatsSink(orm.NewContext().DirectAccess(), obj.Runtime.StageStats)
+ }
+ }
assigned, aerr := loadAssignedConfigs(obj.Client.ID)
if aerr != nil {
@@ -569,7 +635,8 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt
// loadAssignedConfigs returns the server-side config files assigned to the
// instance: its own + the "*" catch-all + any group-targeted config whose
-// Groups intersect the instance's groups. Newest version per name.
+// Groups intersect the instance's groups. Same-name rows resolve by
+// specificity: per-instance rows shadow group/catch-all rows.
// A query FAILURE returns an error — callers must NOT treat it as "no
// configs": a transiently empty read (e.g. manager just booting, ORM not
// ready) would diff every client file as Deleted and wipe managed configs
@@ -620,15 +687,38 @@ func loadAssignedConfigs(instanceID string) ([]common.ConfigFile, error) {
}
}
- out := make([]common.ConfigFile, 0, len(stored))
- for _, mc := range stored {
+ // Same-name resolution: a per-instance row (edge-rule variant) must
+ // SHADOW the group/catch-all rows of the same file for that instance —
+ // per-row version counters are not comparable across rows, so
+ // specificity decides and version only breaks ties within the same
+ // specificity. Without this the winner is query-order luck and an
+ // instance variant may never apply.
+ rank := func(mc *ManagedConfig) int {
+ if mc.InstanceID == instanceID {
+ return 2 // per-instance override
+ }
+ return 1 // "*" catch-all or group-targeted base
+ }
+ byName := make(map[string]ManagedConfig, len(stored))
+ for i := range stored {
+ mc := stored[i]
+ cur, ok := byName[mc.Name]
+ if !ok || rank(&mc) > rank(&cur) || (rank(&mc) == rank(&cur) && mc.Version > cur.Version) {
+ byName[mc.Name] = mc
+ }
+ }
+
+ out := make([]common.ConfigFile, 0, len(byName))
+ for _, mc := range byName {
out = append(out, common.ConfigFile{
Name: mc.Name,
Location: mc.Location,
Content: mc.Content,
Version: mc.Version,
Managed: true,
- Hash: util.MD5digest(mc.Content),
+ // TrimSpace mirrors the client's digest (managed trailer
+ // lines stripped on the client side before hashing).
+ Hash: util.MD5digest(strings.TrimSpace(mc.Content)),
Size: int64(len(mc.Content)),
Updated: time.Now().UnixMilli(),
})
@@ -686,6 +776,18 @@ func diffConfigs(assigned []common.ConfigFile, clientConfigs map[string]common.C
if !cc.Managed {
continue // client opted this file out of management
}
+ // Content-hash comparison when the client reports one (newer
+ // clients): the assigned row may be a DIFFERENT row than the one
+ // the client's file came from (instance variant vs group base),
+ // and per-row version counters cannot be compared across rows —
+ // the hash says exactly whether the bytes differ. Version
+ // comparison remains the fallback for old clients.
+ if sc.Hash != "" && cc.Hash != "" {
+ if sc.Hash != cc.Hash {
+ resp.Configs.UpdatedConfigs[name] = sc
+ }
+ continue
+ }
if sc.Version > cc.Version {
resp.Configs.UpdatedConfigs[name] = sc
}
diff --git a/modules/configs/server/server_test.go b/modules/configs/server/server_test.go
index 65a138641..986c52b67 100644
--- a/modules/configs/server/server_test.go
+++ b/modules/configs/server/server_test.go
@@ -10,6 +10,7 @@ import (
"testing"
httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/env"
"infini.sh/framework/lib/go-ucfg"
"infini.sh/framework/modules/configs/common"
)
@@ -119,7 +120,9 @@ func TestConfigsHash(t *testing.T) {
if h1 != h2 {
t.Fatal("hash must be stable")
}
- // 顺序无关不应成立? 协议要求两侧列表一致 — 同序序列化, 顺序变化视为变更
+ // order-insensitivity must NOT hold: the protocol requires both sides
+ // to carry the same list — same-order marshaling, so a reorder counts
+ // as a change
reordered := []common.ConfigFile{files[1], files[0]}
if ConfigsHash(reordered) == h1 {
t.Log("note: hash is order-sensitive (both sides marshal the same list)")
@@ -225,3 +228,74 @@ func TestHashTokenOneWay(t *testing.T) {
t.Fatal("different inputs must hash differently")
}
}
+
+// TestDiffConfigs_HashAware — content-hash comparison (new clients):
+// version numbers are not comparable across rows (an instance variant at
+// v1 vs the base row at v12 held by the client); a different hash means
+// update, and an equal hash suppresses the resend even when the version is
+// smaller. Legacy clients (no hash) fall back to version comparison.
+func TestDiffConfigs_HashAware(t *testing.T) {
+ base := cfg("s.yml", 12)
+ base.Hash = "hash-base"
+ variant := cfg("s.yml", 1)
+ variant.Hash = "hash-variant"
+
+ t.Run("variant applied over higher-versioned base content", func(t *testing.T) {
+ client := map[string]common.ConfigFile{"s.yml": base}
+ resp := diffConfigs([]common.ConfigFile{variant}, client)
+ if !resp.Changed || resp.Configs.UpdatedConfigs["s.yml"].Hash != "hash-variant" {
+ t.Fatalf("variant must win by hash, got %+v", resp.Configs.UpdatedConfigs)
+ }
+ })
+
+ t.Run("identical hash, lower version → no churn", func(t *testing.T) {
+ same := cfg("s.yml", 3)
+ same.Hash = "hash-base"
+ client := map[string]common.ConfigFile{"s.yml": base}
+ resp := diffConfigs([]common.ConfigFile{same}, client)
+ if resp.Changed {
+ t.Fatal("same content must not re-send")
+ }
+ })
+
+ t.Run("old client without hash falls back to version", func(t *testing.T) {
+ oldClient := cfg("s.yml", 12) // no Hash
+ client := map[string]common.ConfigFile{"s.yml": oldClient}
+ resp := diffConfigs([]common.ConfigFile{variant}, client)
+ if resp.Changed {
+ t.Fatal("version 1 must not beat 12 for hashless clients (logpilot bumps row versions instead)")
+ }
+ })
+}
+
+// Instances built without version ldflags report placeholder dates ("N/A").
+// An index whose mapping predates the keyword pin maps these fields as
+// `date`, and ES rejects the placeholder — every heartbeat upsert failed
+// silently (Debug log) while syncs kept returning 200. normalizeVersionDates
+// must blank anything that is not an RFC3339 date.
+func TestNormalizeVersionDates(t *testing.T) {
+ app := env.Application{}
+ app.Version.BuildDate = "N/A"
+ app.Version.EolDate = "unknown"
+ app.Version.BuildCommitHash = "abc123"
+
+ normalizeVersionDates(&app)
+
+ if app.Version.BuildDate != "" {
+ t.Fatalf("placeholder build_date not blanked: %q", app.Version.BuildDate)
+ }
+ if app.Version.EolDate != "" {
+ t.Fatalf("placeholder eol_date not blanked: %q", app.Version.EolDate)
+ }
+ if app.Version.BuildCommitHash != "abc123" {
+ t.Fatalf("non-date field must be untouched: %q", app.Version.BuildCommitHash)
+ }
+
+ // real dates survive
+ app.Version.BuildDate = "2026-10-21T12:58:27Z"
+ app.Version.EolDate = "2027-12-31T10:10:10Z"
+ normalizeVersionDates(&app)
+ if app.Version.BuildDate != "2026-10-21T12:58:27Z" || app.Version.EolDate != "2027-12-31T10:10:10Z" {
+ t.Fatalf("real dates must survive: %+v", app.Version)
+ }
+}
diff --git a/modules/easysearch/module.go b/modules/easysearch/module.go
index cdfe1962f..e692c367b 100644
--- a/modules/easysearch/module.go
+++ b/modules/easysearch/module.go
@@ -21,6 +21,13 @@ import (
// registration is a later refactor.
type Module struct {
moduleConfig moduleConfig
+
+ // EnabledByDefault lets the registering app opt the module in when the
+ // yml carries no `easysearch:` section. The gateway sets this: its
+ // documented role includes receiving LogPilot's sink-cluster pushes on
+ // /easysearch/ at publish time, and a missing section silently broke
+ // that flow with bare 404s (found live in #130 acceptance).
+ EnabledByDefault bool
}
type moduleConfig struct {
@@ -35,6 +42,9 @@ func (m *Module) Setup() {
if exists && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError {
panic(err)
}
+ if !exists && m.EnabledByDefault {
+ m.moduleConfig.Enabled = true
+ }
if !m.moduleConfig.Enabled {
return
diff --git a/modules/elastic/api.go b/modules/elastic/api.go
index b32afded6..a4495076a 100644
--- a/modules/elastic/api.go
+++ b/modules/elastic/api.go
@@ -29,9 +29,9 @@ package elastic
import (
"infini.sh/framework/core/api"
- "infini.sh/framework/core/security"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/security"
"infini.sh/framework/core/util"
"net/http"
)
diff --git a/modules/elastic/cluster_loader.go b/modules/elastic/cluster_loader.go
index f1db9bd06..b5bd7b92c 100644
--- a/modules/elastic/cluster_loader.go
+++ b/modules/elastic/cluster_loader.go
@@ -23,6 +23,14 @@ import (
// created via the /easysearch/ API (served by modules/easysearch) usable as
// live clients after (re)start.
func LoadClustersFromORM() {
+ // Lightweight instances (e.g. agents) run without any ORM backend —
+ // getHandler() panics when none is registered, and the agent's module
+ // start order may also reach here before the backend registers. Nothing
+ // to load in either case; skip quietly.
+ if !orm.IsHandlerRegistered() {
+ log.Debug("no ORM backend registered, skip loading clusters from ORM")
+ return
+ }
ctx := orm.NewContext().DirectAccess()
orm.WithModel(ctx, &elastic.ElasticsearchConfig{})
res, err := orm.SearchV2(ctx, orm.NewQuery().Size(1000))
diff --git a/modules/elastic/contract_integration_test.go b/modules/elastic/contract_integration_test.go
index ef3df3bb8..e0de280cc 100644
--- a/modules/elastic/contract_integration_test.go
+++ b/modules/elastic/contract_integration_test.go
@@ -29,10 +29,10 @@ func TestContract_Elastic(t *testing.T) {
if err := client.CreateIndex("contractmodel", map[string]interface{}{
"mappings": map[string]interface{}{
"properties": map[string]interface{}{
- "name": map[string]interface{}{"type": "keyword"},
- "status": map[string]interface{}{"type": "keyword"},
- "body": map[string]interface{}{"type": "text"},
- "age": map[string]interface{}{"type": "integer"},
+ "name": map[string]interface{}{"type": "keyword"},
+ "status": map[string]interface{}{"type": "keyword"},
+ "body": map[string]interface{}{"type": "text"},
+ "age": map[string]interface{}{"type": "integer"},
"created": map[string]interface{}{"type": "date"},
},
},
diff --git a/modules/elastic/module.go b/modules/elastic/module.go
index 7bad426fe..b9f6ad0cd 100755
--- a/modules/elastic/module.go
+++ b/modules/elastic/module.go
@@ -423,6 +423,15 @@ var ormInited bool
func (module *ElasticModule) Start() error {
if !moduleConfig.Enabled {
+ // The client provider and the ORM change hook are registered even
+ // when this module is disabled, so runtime cluster pushes via the
+ // /easysearch/ API work without it. A restart must not silently lose
+ // that persisted state: restore dynamic clusters as live clients
+ // here too. Best-effort — the sqlite ORM backend starts before this
+ // module, failures just log a warning.
+ if global.Env().SystemConfig.ORMConfig.Enabled {
+ LoadClustersFromORM()
+ }
return nil
}
diff --git a/modules/elastic/orm/query_dsl.go b/modules/elastic/orm/query_dsl.go
index 67d8ed606..325a3020b 100644
--- a/modules/elastic/orm/query_dsl.go
+++ b/modules/elastic/orm/query_dsl.go
@@ -242,6 +242,14 @@ func BuildQueryDSL(q *orm.QueryBuilder) map[string]interface{} {
dsl["aggs"] = aggs
}
+ // body-level parameters land at the DSL root next to query/from/size;
+ // in the OnTopOfDSL paths this runs inside the override map, so an
+ // explicit BodyParameter also beats a raw request-body key of the same
+ // name
+ for k, v := range q.BodyParametersVal() {
+ dsl[k] = v
+ }
+
return dsl
}
diff --git a/modules/elastic/orm/query_dsl_test.go b/modules/elastic/orm/query_dsl_test.go
index 62ac79220..cd0e4922a 100644
--- a/modules/elastic/orm/query_dsl_test.go
+++ b/modules/elastic/orm/query_dsl_test.go
@@ -1652,3 +1652,41 @@ func TestToDSL_NestedQuery_SimpleMatch(t *testing.T) {
actual, _ := json.Marshal(dsl)
assert.JSONEq(t, expected, string(actual))
}
+
+func TestBuildQueryDSL_BodyParameter(t *testing.T) {
+ q := orm.NewQuery().
+ Must(orm.MatchQuery("title", "elastic")).
+ BodyParameter("min_score", 0.35)
+
+ dsl := BuildQueryDSL(q)
+
+ // body-level parameter must sit at the DSL root, next to query —
+ // not inside the bool clause where ES rejects it with
+ // "unknown field [min_score]"
+ // a single must clause flattens to a bare match (flattenBoolClauses)
+ expected := `{
+ "query": {
+ "match": {"title": {"query": "elastic"}}
+ },
+ "min_score": 0.35
+ }`
+ actual, _ := json.Marshal(dsl)
+ assert.JSONEq(t, expected, string(actual))
+}
+
+func TestBuildQueryDSLOnTopOfDSL_BodyParameterOverridesRawBody(t *testing.T) {
+ q := orm.NewQuery().
+ Must(orm.MatchQuery("title", "elastic")).
+ BodyParameter("min_score", 0.35)
+
+ raw := []byte(`{"query": {"match_all": {}}, "min_score": 0.05, "size": 20}`)
+ dsl := BuildQueryDSLOnTopOfDSL(q, raw)
+
+ // the explicit BodyParameter beats the raw request-body key
+ if v, ok := dsl["min_score"].(float64); !ok || v != 0.35 {
+ t.Fatalf("min_score = %v, want 0.35", dsl["min_score"])
+ }
+ if dsl["size"] != float64(20) {
+ t.Fatalf("size = %v, want 20 (raw body keys survive unless overridden)", dsl["size"])
+ }
+}
diff --git a/modules/elastic/orm_aggregate.go b/modules/elastic/orm_aggregate.go
index 004389417..462643c18 100644
--- a/modules/elastic/orm_aggregate.go
+++ b/modules/elastic/orm_aggregate.go
@@ -4,9 +4,9 @@ package elastic
import (
"encoding/json"
+ "fmt"
"strconv"
"time"
- "fmt"
"infini.sh/framework/core/aggregate"
api "infini.sh/framework/core/orm"
@@ -179,7 +179,6 @@ func parseESAggNode(m map[string]interface{}) *api.AggNode {
return node
}
-
// normalizeESBucketKey renders ES date_histogram key_as_string values in the
// canonical second-precision layout (2006-01-02T15:04:05, UTC, no millis)
// used by the cross-backend conformance suite — ES emits RFC3339 with
diff --git a/modules/keystore/api/api.go b/modules/keystore/api/api.go
index fde85be24..7a7fdb8d8 100644
--- a/modules/keystore/api/api.go
+++ b/modules/keystore/api/api.go
@@ -80,7 +80,6 @@ func (h *APIHandler) setKeystoreValue(w http.ResponseWriter, req *http.Request,
}, http.StatusOK)
}
-
// listKeystoreKeys — GET /keystore — key names only. Values are secrets:
// the API is write-only for them by design (nothing can read them back).
func (h *APIHandler) listKeystoreKeys(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
diff --git a/modules/managed/reverseclient/reverse.go b/modules/managed/reverseclient/reverse.go
index 3d08364b1..670049dd7 100644
--- a/modules/managed/reverseclient/reverse.go
+++ b/modules/managed/reverseclient/reverse.go
@@ -24,6 +24,8 @@ import (
"infini.sh/framework/core/api/websocket/reverse"
"infini.sh/framework/core/global"
+ "infini.sh/framework/core/kv"
+ "infini.sh/framework/core/security"
"infini.sh/framework/core/util"
common "infini.sh/framework/modules/configs/common"
)
@@ -46,6 +48,10 @@ func Setup() {
if !global.Env().SystemConfig.Configs.Managed {
return
}
+ // instance-ops keys are declared as constants and never route-
+ // registered; register them now so the permission filter can
+ // resolve the system:* route options on manager callbacks
+ security.GetOrInitPermissionKeys(security.InstanceOpsPermissionKeys()...)
go run()
})
}
@@ -246,8 +252,14 @@ func execute(reqMsg reverse.RequestMessage) (int, []byte) {
req.Header.Set("Content-Type", util.ContentTypeJson)
}
// Authenticate the loopback call with the agent's own API token so
- // it passes the access_token realm.
- if tok, _ := common.LoadTokenFromKeystore("AGENT_API_ACCESS_TOKEN"); tok != "" {
+ // it passes the access_token realm. A keystore token with no matching
+ // record in the security KV bucket (stale from a wiped data dir, or
+ // provisioned by another flow) is re-minted.
+ tok, _ := common.LoadTokenFromKeystore("AGENT_API_ACCESS_TOKEN")
+ if tok == "" || !loopbackTokenKnown(tok) {
+ tok = ensureLoopbackToken()
+ }
+ if tok != "" {
req.Header.Set("X-API-Token", strings.TrimSpace(tok))
}
client := &http.Client{Timeout: requestDeadline}
@@ -260,6 +272,72 @@ func execute(reqMsg reverse.RequestMessage) (int, []byte) {
return resp.StatusCode, data
}
+// loopbackTokenKnown reports whether the token string has a usable record
+// in the "access_token" KV bucket (the security filter's X-API-TOKEN
+// lookup table — modules/security/access_token.KVAccessTokenBucket): the
+// record must exist AND carry every instance-ops permission, so tokens
+// minted by older builds get upgraded.
+func loopbackTokenKnown(tok string) bool {
+ v, err := kv.GetValue("access_token", []byte(tok))
+ if err != nil || len(v) == 0 {
+ return false
+ }
+ at := security.AccessToken{}
+ if util.FromJSONBytes(v, &at) != nil {
+ return false
+ }
+ have := map[security.PermissionKey]bool{}
+ for _, k := range at.Permissions {
+ have[k] = true
+ }
+ for _, k := range security.InstanceOpsPermissionKeys() {
+ if !have[k] {
+ return false
+ }
+ }
+ return true
+}
+
+// ensureLoopbackToken mints (once) a local API token carrying every
+// registered permission, so manager callbacks proxied onto the agent's
+// own HTTP API pass the access_token realm. The record goes into the KV
+// bucket the security filter reads; the token string is cached in the
+// keystore so restarts reuse it instead of accumulating records.
+var (
+ loopbackOnce sync.Once
+ loopbackToken string
+)
+
+func ensureLoopbackToken() string {
+ loopbackOnce.Do(func() {
+ // union the declared instance-ops keys (registered at Setup) with
+ // everything route-registration has already added, for the widest
+ // callback compatibility
+ perms := append(security.InstanceOpsPermissionKeys(), security.GetAllPermissionKeys()...)
+
+ tok := util.GetUUID() + util.GenerateRandomString(64)
+ at := security.AccessToken{
+ Name: "reverse-channel-loopback",
+ Description: "auto-minted local token for manager reverse-channel callbacks",
+ Permissions: perms,
+ ExpireIn: -1, // never
+ }
+ at.ID = util.GetUUID()
+ at.SetOwnerID(global.Env().SystemConfig.NodeConfig.ID)
+ if err := kv.AddValue("access_token", []byte(tok), util.MustToJSONBytes(&at)); err != nil {
+ log.Errorf("agent reverse channel: mint loopback token failed: %v", err)
+ return
+ }
+ if err := common.SaveTokenToKeystore("AGENT_API_ACCESS_TOKEN", tok); err != nil {
+ log.Errorf("agent reverse channel: persist loopback token failed: %v", err)
+ // still usable this run; next start mints again
+ }
+ loopbackToken = tok
+ log.Info("agent reverse channel: loopback API token minted for manager callbacks")
+ })
+ return loopbackToken
+}
+
// localBaseURL derives the agent's own serving address.
func localBaseURL() string {
// Prefer the web app port; some binaries (gateway) serve their API
diff --git a/modules/pipeline/ai_generate.go b/modules/pipeline/ai_generate.go
new file mode 100644
index 000000000..5f198d472
--- /dev/null
+++ b/modules/pipeline/ai_generate.go
@@ -0,0 +1,169 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "sort"
+ "strings"
+ "sync"
+
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/util"
+)
+
+// Chain generation is product logic (LLM backend, prompts, processor
+// catalog wording); the endpoint is not. Products register a
+// ChainGenerator at init and the unified POST /pipeline/ai-generate stays
+// product-agnostic — the same shape as the unified dry-run endpoint.
+
+// ChainGenerateRequest is the product-agnostic input: parsed sample
+// documents, a natural-language requirement and (refine mode) the chain
+// to edit.
+type ChainGenerateRequest struct {
+ Context context.Context
+ Documents []util.MapStr
+ Requirements string
+ // Mode: "generate" (default) | "refine"
+ Mode string
+ CurrentChain []map[string]interface{}
+}
+
+// ChainGenerator turns samples plus a requirement into a processor chain
+// (entries in the same shape dry-run takes: single-key processor maps or
+// if/then/else). Generators return raw entries; validation and replay
+// stay with the caller (the unified dry-run endpoint).
+type ChainGenerator interface {
+ Name() string
+ Generate(req ChainGenerateRequest) ([]map[string]interface{}, error)
+}
+
+var (
+ chainGenMu sync.RWMutex
+ chainGens = map[string]ChainGenerator{}
+)
+
+// RegisterChainGeneratorOrReplace registers (or replaces) a generator by
+// name; the empty name is reserved for the framework.
+func RegisterChainGeneratorOrReplace(g ChainGenerator) error {
+ if g == nil || g.Name() == "" {
+ return fmt.Errorf("chain generator must have a non-empty name")
+ }
+ chainGenMu.Lock()
+ defer chainGenMu.Unlock()
+ chainGens[g.Name()] = g
+ return nil
+}
+
+// GetChainGenerator resolves a generator by name.
+func GetChainGenerator(name string) (ChainGenerator, bool) {
+ chainGenMu.RLock()
+ defer chainGenMu.RUnlock()
+ g, ok := chainGens[name]
+ return g, ok
+}
+
+// ChainGeneratorNames lists registered generators (sorted).
+func ChainGeneratorNames() []string {
+ chainGenMu.RLock()
+ defer chainGenMu.RUnlock()
+ names := make([]string, 0, len(chainGens))
+ for n := range chainGens {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ return names
+}
+
+// postAIGenerateHandler — POST /pipeline/ai-generate.
+// body: {generator?, documents: [raw-JSON-string...], requirements?,
+// mode?, current_chain?} → {chain} or {chain, generators} on 501/400.
+// 501 when no generator is registered; 400 for bad input or an ambiguous
+// empty generator name with multiple registrations; 502 when the
+// generator fails. Validation/replay is the caller's follow-up call to
+// /pipeline/dry-run.
+func (module *PipeModule) postAIGenerateHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
+ var body struct {
+ Generator string `json:"generator"`
+ Documents []string `json:"documents"`
+ Requirements string `json:"requirements"`
+ Mode string `json:"mode"`
+ CurrentChain []map[string]interface{} `json:"current_chain"`
+ }
+ if err := module.DecodeJSON(req, &body); err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if len(body.Documents) == 0 {
+ module.WriteError(w, "documents (non-empty string array of sample document JSON) is required", http.StatusBadRequest)
+ return
+ }
+
+ name := strings.TrimSpace(body.Generator)
+ if name == "" {
+ names := ChainGeneratorNames()
+ switch len(names) {
+ case 0:
+ module.WriteError(w, "no chain generator registered in this build", http.StatusNotImplemented)
+ return
+ case 1:
+ name = names[0]
+ default:
+ module.WriteError(w, "multiple chain generators registered, specify one of: "+strings.Join(names, ", "), http.StatusBadRequest)
+ return
+ }
+ }
+ gen, ok := GetChainGenerator(name)
+ if !ok {
+ module.WriteError(w, "unknown chain generator "+name+" (registered: "+strings.Join(ChainGeneratorNames(), ", ")+")", http.StatusBadRequest)
+ return
+ }
+
+ mode := body.Mode
+ if mode == "" {
+ mode = "generate"
+ }
+ if mode != "generate" && mode != "refine" {
+ module.WriteError(w, "mode must be generate or refine", http.StatusBadRequest)
+ return
+ }
+ if mode == "refine" && len(body.CurrentChain) == 0 {
+ module.WriteError(w, "refine requires current_chain", http.StatusBadRequest)
+ return
+ }
+
+ // sample documents must be JSON objects — generators promise real docs
+ docs := make([]util.MapStr, 0, len(body.Documents))
+ for i, raw := range body.Documents {
+ var doc util.MapStr
+ if err := util.FromJSONBytes([]byte(raw), &doc); err != nil {
+ module.WriteError(w, fmt.Sprintf("documents[%d] is not a valid JSON object", i), http.StatusBadRequest)
+ return
+ }
+ docs = append(docs, doc)
+ }
+
+ chain, err := gen.Generate(ChainGenerateRequest{
+ Context: req.Context(),
+ Documents: docs,
+ Requirements: body.Requirements,
+ Mode: mode,
+ CurrentChain: body.CurrentChain,
+ })
+ if err != nil {
+ module.WriteError(w, truncateGenError(err.Error()), http.StatusBadGateway)
+ return
+ }
+ module.WriteJSON(w, util.MapStr{"chain": chain, "generator": name}, http.StatusOK)
+}
+
+func truncateGenError(s string) string {
+ const cap = 300
+ if len(s) <= cap {
+ return s
+ }
+ return s[:cap] + "…"
+}
diff --git a/modules/pipeline/ai_generate_test.go b/modules/pipeline/ai_generate_test.go
new file mode 100644
index 000000000..de4bf0b03
--- /dev/null
+++ b/modules/pipeline/ai_generate_test.go
@@ -0,0 +1,138 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/util"
+)
+
+// stubGenerator is the ChainGenerator test double.
+type stubGenerator struct {
+ name string
+ err error
+}
+
+func (s stubGenerator) Name() string { return s.name }
+func (s stubGenerator) Generate(req ChainGenerateRequest) ([]map[string]interface{}, error) {
+ if s.err != nil {
+ return nil, s.err
+ }
+ return []map[string]interface{}{{"test_upper": util.MapStr{}}}, nil
+}
+
+func TestChainGeneratorRegistry(t *testing.T) {
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_a"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := RegisterChainGeneratorOrReplace(nil); err == nil {
+ t.Fatal("nil generator must be rejected")
+ }
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: ""}); err == nil {
+ t.Fatal("empty name must be rejected")
+ }
+ if g, ok := GetChainGenerator("test_gen_a"); !ok || g.Name() != "test_gen_a" {
+ t.Fatal("registered generator must resolve")
+ }
+ if _, ok := GetChainGenerator("missing"); ok {
+ t.Fatal("unregistered name must not resolve")
+ }
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_a", err: fmt.Errorf("v2")}); err != nil {
+ t.Fatal(err)
+ }
+ if g, _ := GetChainGenerator("test_gen_a"); g.(stubGenerator).err == nil {
+ t.Fatal("replace must overwrite")
+ }
+}
+
+func callAIGenerate(t *testing.T, body string) *httptest.ResponseRecorder {
+ t.Helper()
+ m := &PipeModule{}
+ req := httptest.NewRequest(http.MethodPost, "/pipeline/ai-generate", strings.NewReader(body))
+ w := httptest.NewRecorder()
+ m.postAIGenerateHandler(w, req, httprouter.Params{})
+ return w
+}
+
+func TestAIGenerateHandler_UnimplementedWithoutGenerator(t *testing.T) {
+ // no generator carries this unique name; simulate a bare build by
+ // asking for a name that cannot exist
+ w := callAIGenerate(t, `{"generator":"__none__","documents":["{}"]}`)
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("unknown generator must be 400, got %d", w.Code)
+ }
+ if !strings.Contains(w.Body.String(), "unknown chain generator") {
+ t.Fatalf("unexpected body: %s", w.Body.String())
+ }
+}
+
+func TestAIGenerateHandler_HappyPathAndValidation(t *testing.T) {
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai"}); err != nil {
+ t.Fatal(err)
+ }
+
+ w := callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{\"title\":\"a\"}"],"requirements":"add tags"}`)
+ if w.Code != http.StatusOK {
+ t.Fatalf("generate must succeed, got %d: %s", w.Code, w.Body.String())
+ }
+ if !strings.Contains(w.Body.String(), `"test_upper"`) || !strings.Contains(w.Body.String(), `"generator":"test_gen_ai"`) {
+ t.Fatalf("unexpected body: %s", w.Body.String())
+ }
+
+ // empty documents → 400
+ if w := callAIGenerate(t, `{"documents":[]}`); w.Code != http.StatusBadRequest {
+ t.Fatalf("empty documents must be 400, got %d", w.Code)
+ }
+ // bad document JSON → 400
+ if w := callAIGenerate(t, `{"documents":["not-json"]}`); w.Code != http.StatusBadRequest {
+ t.Fatalf("invalid document JSON must be 400, got %d", w.Code)
+ }
+ // bad mode → 400
+ if w := callAIGenerate(t, `{"documents":["{}"],"mode":"nope"}`); w.Code != http.StatusBadRequest {
+ t.Fatalf("bad mode must be 400, got %d", w.Code)
+ }
+ // refine without current_chain → 400
+ if w := callAIGenerate(t, `{"documents":["{}"],"mode":"refine"}`); w.Code != http.StatusBadRequest {
+ t.Fatalf("refine without chain must be 400, got %d", w.Code)
+ }
+ // generator failure → 502
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai", err: fmt.Errorf("llm down")}); err != nil {
+ t.Fatal(err)
+ }
+ if w := callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{}"]}`); w.Code != http.StatusBadGateway {
+ t.Fatalf("generator error must be 502, got %d", w.Code)
+ }
+ // restore the working stub
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai"}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestAIGenerateHandler_AmbiguousDefault: with a second generator the
+// empty-name default must refuse instead of guessing.
+func TestAIGenerateHandler_AmbiguousDefault(t *testing.T) {
+ if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_b"}); err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ chainGenMu.Lock()
+ delete(chainGens, "test_gen_b")
+ chainGenMu.Unlock()
+ }()
+ w := callAIGenerate(t, `{"documents":["{}"]}`)
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("ambiguous default must be 400, got %d: %s", w.Code, w.Body.String())
+ }
+ // explicit name still works
+ w = callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{}"]}`)
+ if w.Code != http.StatusOK {
+ t.Fatalf("explicit generator must work, got %d", w.Code)
+ }
+}
diff --git a/modules/pipeline/cluster_stages_test.go b/modules/pipeline/cluster_stages_test.go
new file mode 100644
index 000000000..c61139c9c
--- /dev/null
+++ b/modules/pipeline/cluster_stages_test.go
@@ -0,0 +1,75 @@
+/* Copyright © INFINI LTD. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "testing"
+ "time"
+)
+
+// statRow builds a persisted snapshot row shorthand.
+func statRow(inst, pipeline, stage string, count, errors int64, avg int64, ts time.Time) PipelineStageStat {
+ return PipelineStageStat{
+ InstanceID: inst, Pipeline: pipeline, Stage: stage, Carrier: "processor",
+ Count: count, Errors: errors, AvgLatency: avg, Ts: ts,
+ }
+}
+
+// TestAggregateStageRows_ClusterSumMatchesParts covers the #130
+// acceptance math: the cluster aggregate equals the sum of each
+// instance's LATEST tick (older ticks superseded), and latency is the
+// invocation-weighted mean across instances.
+func TestAggregateStageRows_ClusterSumMatchesParts(t *testing.T) {
+ t0 := time.Now()
+ t1 := t0.Add(10 * time.Second) // instance A's second tick supersedes the first
+ rows := []PipelineStageStat{
+ // instance A: tick1 (stale), tick2 (latest) — only tick2 may count
+ statRow("inst-a", "p1", "echo", 100, 1, 1_000_000, t0),
+ statRow("inst-a", "p1", "echo", 300, 2, 2_000_000, t1),
+ // instance B: single tick
+ statRow("inst-b", "p1", "echo", 100, 0, 6_000_000, t0),
+ // a different stage stays its own row
+ statRow("inst-b", "p1", "grok", 50, 5, 3_000_000, t0),
+ }
+ // deliberately shuffle: the aggregator must not depend on input order
+ rows = []PipelineStageStat{rows[1], rows[3], rows[0], rows[2]}
+
+ out := aggregateStageRows(rows)
+ if len(out) != 2 {
+ t.Fatalf("want 2 stage rows, got %d: %+v", len(out), out)
+ }
+
+ var echo *ClusterStageStat
+ for i := range out {
+ if out[i].Stage == "echo" {
+ echo = &out[i]
+ }
+ }
+ if echo == nil {
+ t.Fatal("echo stage row missing")
+ }
+ // aggregate = latest tick of A + tick of B (stale A tick1 ignored)
+ if echo.Count != 400 || echo.Errors != 2 {
+ t.Fatalf("echo aggregate = %d/%d, want 400/2", echo.Count, echo.Errors)
+ }
+ // weighted latency: (2ms*300 + 6ms*100) / 400 = 3ms
+ if echo.AvgLatency != 3_000_000 {
+ t.Fatalf("weighted avg latency = %d, want 3000000", echo.AvgLatency)
+ }
+ // per-instance detail present and consistent with the aggregate
+ if len(echo.PerInstance) != 2 {
+ t.Fatalf("per-instance detail missing: %+v", echo.PerInstance)
+ }
+ if a := echo.PerInstance["inst-a"]; a.Count != 300 {
+ t.Fatalf("inst-a detail = %+v, want the latest tick (300)", a)
+ }
+ var sum int64
+ for _, st := range echo.PerInstance {
+ sum += st.Count
+ }
+ if sum != echo.Count {
+ t.Fatalf("aggregate %d != sum of parts %d", echo.Count, sum)
+ }
+}
diff --git a/modules/pipeline/control_test.go b/modules/pipeline/control_test.go
new file mode 100644
index 000000000..e9380da18
--- /dev/null
+++ b/modules/pipeline/control_test.go
@@ -0,0 +1,264 @@
+/* Copyright © INFINI LTD. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "net/http/httptest"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/kv"
+ "infini.sh/framework/core/locker"
+ "infini.sh/framework/core/pipeline"
+)
+
+// ── generation guard (#120) ─────────────────────────────────────────────
+
+func TestRequestedGeneration(t *testing.T) {
+ req := httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=2", nil)
+ if n, ok := requestedGeneration(req); !ok || n != 2 {
+ t.Fatalf("generation=2 parsed as (%d, %v)", n, ok)
+ }
+ req = httptest.NewRequest("POST", "/pipeline/task/x/_pause", nil)
+ if _, ok := requestedGeneration(req); ok {
+ t.Fatal("absent param must be unguarded")
+ }
+ req = httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=abc", nil)
+ if _, ok := requestedGeneration(req); ok {
+ t.Fatal("malformed param must be unguarded")
+ }
+ req = httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=0", nil)
+ if _, ok := requestedGeneration(req); ok {
+ t.Fatal("non-positive param must be unguarded")
+ }
+}
+
+// ── drain (#121) ────────────────────────────────────────────────────────
+
+func drainableCfg(id, procName string) pipeline.PipelineConfigV2 {
+ cfg := hotCfg(id, procName, "x")
+ cfg.KeepRunning = true
+ cfg.RetryDelayInMs = 1
+ return cfg
+}
+
+func TestDrainTask_ParksStoppedAndResumes(t *testing.T) {
+ proc := hotReloadTestProcessor()
+ m := &PipeModule{}
+ defer func() {
+ m.stopAndWaitForRelease([]string{"drain-1"}, time.Minute)
+ m.deleteTask("drain-1")
+ }()
+
+ if err := m.createPipelineTask(drainableCfg("drain-1", proc), false); err != nil {
+ t.Fatalf("create: %v", err)
+ }
+
+ if !m.drainTask("drain-1") {
+ t.Fatal("drainTask reported task missing")
+ }
+ state := m.waitTaskSettled("drain-1", 5*time.Second)
+ if state == pipeline.STARTED || state == pipeline.STARTING || state == pipeline.STOPPING {
+ t.Fatalf("task not settled after drain, state=%v", state)
+ }
+ // drained task stays registered (resumable), unlike delete
+ if _, ok := m.contexts.Load("drain-1"); !ok {
+ t.Fatal("drained task was removed from the registry")
+ }
+
+ // restart clears the drain flag so the task runs again
+ if !m.startTask("drain-1") {
+ t.Fatal("startTask on drained task failed")
+ }
+ deadline := time.Now().Add(3 * time.Second)
+ for m.taskRunningState("drain-1") == pipeline.STOPPED {
+ if time.Now().After(deadline) {
+ t.Fatal("task did not resume after start")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ v, _ := m.contexts.Load("drain-1")
+ if ctx, ok := v.(*pipeline.Context); !ok || ctx.IsDraining() {
+ t.Fatal("restart did not clear the drain flag")
+ }
+}
+
+func TestDrainAllTasks_SettlesEveryTask(t *testing.T) {
+ proc := hotReloadTestProcessor()
+ m := &PipeModule{}
+ for _, id := range []string{"drainall-1", "drainall-2"} {
+ if err := m.createPipelineTask(drainableCfg(id, proc), false); err != nil {
+ t.Fatalf("create %s: %v", id, err)
+ }
+ defer func(id string) {
+ m.stopAndWaitForRelease([]string{id}, time.Minute)
+ m.deleteTask(id)
+ }(id)
+ }
+
+ states := m.drainAllTasks(5 * time.Second)
+ if len(states) != 2 {
+ t.Fatalf("drainAllTasks saw %d tasks, want 2: %v", len(states), states)
+ }
+ for id, s := range states {
+ if s == string(pipeline.STARTED) || s == string(pipeline.STARTING) || s == string(pipeline.STOPPING) {
+ t.Fatalf("task %s not settled: %v", id, s)
+ }
+ }
+}
+
+// ── cordon (#121) ───────────────────────────────────────────────────────
+
+func TestCordon_BlocksTaskStarts(t *testing.T) {
+ proc := hotReloadTestProcessor()
+ m := &PipeModule{}
+ defer func() {
+ m.stopAndWaitForRelease([]string{"cordon-1"}, time.Minute)
+ m.deleteTask("cordon-1")
+ }()
+
+ m.cordoned.Store(true)
+ if err := m.createPipelineTask(drainableCfg("cordon-1", proc), false); err != nil {
+ t.Fatalf("create under cordon: %v", err)
+ }
+ // auto_start task on a cordoned instance parks STOPPED instead of running
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ s := m.taskRunningState("cordon-1")
+ if s == pipeline.STOPPED {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("cordoned task did not park, state=%v", s)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ time.Sleep(100 * time.Millisecond)
+ if s := m.taskRunningState("cordon-1"); s != pipeline.STOPPED {
+ t.Fatalf("cordoned task left the parked state: %v", s)
+ }
+
+ m.cordoned.Store(false)
+ if !m.startTask("cordon-1") {
+ t.Fatal("startTask after uncordon failed")
+ }
+ deadline = time.Now().Add(3 * time.Second)
+ for m.taskRunningState("cordon-1") == pipeline.STOPPED {
+ if time.Now().After(deadline) {
+ t.Fatal("task did not resume after uncordon")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+// ── lock takeover (#122) ────────────────────────────────────────────────
+
+// controlMemKV is the minimal in-memory KVStore backing the locker probes.
+type controlMemKV struct {
+ mu sync.Mutex
+ kv map[string][]byte
+}
+
+func (m *controlMemKV) Open() error { return nil }
+func (m *controlMemKV) Close() error { return nil }
+
+func (m *controlMemKV) key(bucket string, key []byte) string {
+ return bucket + ":" + string(key)
+}
+
+func (m *controlMemKV) GetValue(bucket string, key []byte) ([]byte, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ v, ok := m.kv[m.key(bucket, key)]
+ if !ok {
+ return nil, nil
+ }
+ return v, nil
+}
+
+func (m *controlMemKV) GetCompressedValue(bucket string, key []byte) ([]byte, error) {
+ return m.GetValue(bucket, key)
+}
+
+func (m *controlMemKV) AddValue(bucket string, key []byte, value []byte) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.kv[m.key(bucket, key)] = value
+ return nil
+}
+
+func (m *controlMemKV) AddValueCompress(bucket string, key []byte, value []byte) error {
+ return m.AddValue(bucket, key, value)
+}
+
+func (m *controlMemKV) ExistsKey(bucket string, key []byte) (bool, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ _, ok := m.kv[m.key(bucket, key)]
+ return ok, nil
+}
+
+func (m *controlMemKV) DeleteKey(bucket string, key []byte) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ delete(m.kv, m.key(bucket, key))
+ return nil
+}
+
+var controlKVOnce sync.Once
+
+// placeLockRow writes a raw locker row ("clientID/") under the
+// locker's private kv bucket; the bucket literal mirrors core/locker's
+// parentBucket.
+func placeLockRow(t *testing.T, name, clientID string, ts time.Time) {
+ v := clientID + "/" + strconv.FormatInt(ts.Unix(), 10)
+ if err := kv.AddValue("dis_locker", locker.GetKey(pipelineSingleton, name), []byte(v)); err != nil {
+ t.Fatalf("place lock row: %v", err)
+ }
+}
+
+func TestProbeLockTakeover(t *testing.T) {
+ controlKVOnce.Do(func() {
+ kv.Register("control-test-mem", &controlMemKV{kv: map[string][]byte{}})
+ })
+
+ const name = "takeover-probe-1"
+ ttl := 60 * time.Second
+ expired := time.Now().Add(-2 * ttl)
+
+ if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" {
+ t.Fatalf("no lock placed, probe returned %q", got)
+ }
+
+ placeLockRow(t, name, "node-A", expired)
+ if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "node-A" {
+ t.Fatalf("expired foreign lock: probe returned %q, want node-A", got)
+ }
+
+ placeLockRow(t, name, "node-B", time.Now())
+ if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" {
+ t.Fatalf("fresh foreign lock must not report takeover, got %q", got)
+ }
+
+ placeLockRow(t, name, global.Env().SystemConfig.NodeConfig.ID, expired)
+ if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" {
+ t.Fatalf("self-owned lock must not report takeover, got %q", got)
+ }
+}
+
+func TestRecordPipelineEvent_TakeoverLandsInFlight(t *testing.T) {
+ installFlight()
+ recordPipelineEvent("evt-takeover-1", pipeline.OutcomeTakeover, "singleton lock taken over from instance node-A")
+ evs := flightQuery("evt-takeover-1", "", true, 10)
+ if len(evs) != 1 {
+ t.Fatalf("takeover event not retained: %+v", evs)
+ }
+ if evs[0].Outcome != pipeline.OutcomeTakeover {
+ t.Fatalf("outcome = %v, want %v", evs[0].Outcome, pipeline.OutcomeTakeover)
+ }
+}
diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go
new file mode 100644
index 000000000..5ae4bc967
--- /dev/null
+++ b/modules/pipeline/deployment.go
@@ -0,0 +1,970 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+ "time"
+
+ log "github.com/cihub/seelog"
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/model"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+ configs_server "infini.sh/framework/modules/configs/server"
+)
+
+// PipelineDeployment is the DESIRED state: which pipeline runs where and
+// how. The reconciler translates this into managed config rows that
+// gateway/agent instances pull through the configs-center channel.
+//
+// This is the control-plane object — instances never see it; they only
+// receive the translated pipeline configs.
+type PipelineDeployment struct {
+ orm.ORMObjectBase `json:",inline" elastic_mapping:"-"`
+
+ Name string `json:"name" elastic_mapping:"name:{type:keyword}"`
+ PipelineID string `json:"pipeline_id" elastic_mapping:"pipeline_id:{type:keyword}"`
+
+ Placement PlacementSpec `json:"placement" elastic_mapping:"placement:{type:object}"`
+
+ // Enabled gates the reconciler: disabled = no configs pushed.
+ Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"`
+
+ // Version increments on every spec change (reconciler compares with
+ // the last-pushed version to detect drift).
+ Version int `json:"version" elastic_mapping:"version:{type:integer}"`
+
+ // State is the reconciler-owned progress of a staged rollout (#125):
+ // which content digest is being staged, who already carries it and
+ // what a rollback would fall back to. Managed by rollout.go.
+ State RolloutState `json:"rollout_state,omitempty" elastic_mapping:"rollout_state:{type:object}"`
+}
+
+// PlacementSpec declares WHERE and HOW the pipeline runs.
+type PlacementSpec struct {
+ // Selector picks the target instance set (label match).
+ Selector map[string]string `json:"selector,omitempty" config:"selector"`
+
+ // Instances lists explicit target instance IDs. When non-empty, the
+ // reconciler creates per-instance managed configs; empty = broadcast.
+ Instances []string `json:"instances,omitempty" config:"instances"`
+
+ // Strategy: "singleton" (one active runner, lock-contested) |
+ // "all" (every instance runs a copy)
+ // "sharded" requires kafka queue backend (deferred, D-12)
+ Strategy string `json:"strategy" config:"strategy"`
+
+ // Rollout controls staged delivery.
+ Rollout RolloutSpec `json:"rollout,omitempty" config:"rollout"`
+}
+
+// RolloutSpec gates how many instances get the new version at once.
+type RolloutSpec struct {
+ Canary int `json:"canary,omitempty" config:"canary"` // first N instances
+ Batch int `json:"batch,omitempty" config:"batch"` // then batches of N
+ MaxUnavailable int `json:"max_unavailable,omitempty" config:"max_unavailable"` // concurrent down cap
+}
+
+// DeploymentStatus is the REPORTED state per deployment (what instances
+// actually acknowledged running).
+type DeploymentStatus struct {
+ DeploymentID string `json:"deployment_id"`
+ Version int `json:"version"`
+ Instances []InstanceReport `json:"instances"`
+ DesiredCount int `json:"desired_count"`
+ ReadyCount int `json:"ready_count"`
+ // Rollout mirrors the staged-delivery progress when one is (or was)
+ // active — the rollout progress block of the cluster overview.
+ Rollout *RolloutState `json:"rollout,omitempty"`
+}
+
+// runtimePipelineReport is one pipeline entry of an instance's
+// RuntimePipelines JSON (what _sync upserts on every heartbeat).
+type runtimePipelineReport struct {
+ State string `json:"state"`
+ Reason string `json:"reason"`
+ ConfigDigest string `json:"config_digest"`
+}
+
+// runtimePipelineStates parses the instance's reported pipeline states;
+// absent/invalid JSON yields an empty map (unreported, blocks readiness).
+func runtimePipelineStates(raw string) map[string]runtimePipelineReport {
+ out := map[string]runtimePipelineReport{}
+ if raw == "" {
+ return out
+ }
+ var parsed map[string]runtimePipelineReport
+ if json.Unmarshal([]byte(raw), &parsed) != nil {
+ return out
+ }
+ return parsed
+}
+
+type InstanceReport struct {
+ InstanceID string `json:"instance_id"`
+ State string `json:"state"` // running | stopped | failed | unknown
+ Version int `json:"version"`
+ LastSeen time.Time `json:"last_seen"`
+ // Reason carries the instance-reported failure explanation (compile
+ // error, missing processor on that instance)
+ Reason string `json:"reason,omitempty"`
+ // Stale marks an instance that still runs the pipeline although the
+ // deployment scope no longer includes it (recall in flight/failed)
+ Stale bool `json:"stale,omitempty"`
+ // Cordoned mirrors the instance's maintenance flag (reported via
+ // _sync, #128): running tasks drain, nothing new starts there
+ Cordoned bool `json:"cordoned,omitempty"`
+}
+
+// ─── reconciler ──────────────────────────────────────────────────────────
+
+// reconcilePoke requests an immediate reconciler pass outside the 15s tick.
+// Buffered so pokers never block; a poke landing while a pass is already
+// running just coalesces into one follow-up pass.
+var reconcilePoke = make(chan struct{}, 1)
+
+// RequestReconcile asks the deployment reconciler to run a pass as soon as
+// possible (async, best-effort). Publishers call it after upserting
+// PipelineDeployment/PipelineConfigV2 so convergence starts immediately
+// instead of on the next tick (#128).
+func RequestReconcile() {
+ select {
+ case reconcilePoke <- struct{}{}:
+ default:
+ }
+}
+
+// reconcileDeployments runs on a ticker: for each enabled deployment,
+// compare desired state with the managed-config rows that exist, and
+// create/update/delete configs to converge.
+func (module *PipeModule) startDeploymentReconciler() {
+ go func() {
+ ticker := time.NewTicker(15 * time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ case <-reconcilePoke:
+ }
+ if module.closed.Load() {
+ return
+ }
+ module.reconcileOnce()
+ }
+ }()
+ log.Debugf("deployment reconciler started (interval 15s)")
+}
+
+func (module *PipeModule) reconcileOnce() {
+ // reconcile is a control-plane operation — skip on instances without
+ // an ORM backend (lightweight agents that only run pushed configs)
+ if !orm.IsHandlerRegistered() {
+ return
+ }
+ octx := orm.NewContext().DirectAccess()
+
+ // registered instances — the universe that selector/explicit scopes
+ // resolve against (labels refresh on every sync heartbeat)
+ orm.WithModel(octx, &model.Instance{})
+ instRes, ierr := orm.SearchV2(octx, orm.NewQuery().Size(500))
+ if ierr != nil {
+ return
+ }
+ instances, _, _ := elastic.DecodeHits[model.Instance](instRes)
+
+ // ALL deployments, disabled included: a disabled deployment keeps
+ // converging with enabled:false in the rendered config, so instances
+ // STOP the task while KEEPING the config (instant re-enable). Skipping
+ // disabled ones would leave the pipeline running on every instance.
+ orm.WithModel(octx, &PipelineDeployment{})
+ res, err := orm.SearchV2(octx, orm.NewQuery().Size(200))
+ if err != nil {
+ return
+ }
+ deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res)
+
+ // group by pipeline: managed-config rows are named per PIPELINE
+ // (pipeline-deploy-.yml), so every deployment of the same
+ // pipeline must converge together — one adds instances, another's
+ // scope-shrink must not delete rows the first one still needs
+ byPipeline := map[string][]*PipelineDeployment{}
+ for i := range deployments {
+ byPipeline[deployments[i].PipelineID] = append(byPipeline[deployments[i].PipelineID], &deployments[i])
+ }
+ for pipelineID, deps := range byPipeline {
+ if rerr := module.reconcilePipeline(octx, pipelineID, deps, instances); rerr != nil {
+ log.Warnf("reconciler: %v", rerr)
+ }
+ }
+}
+
+// renderSpec is how ONE target instance should run the pipeline — the
+// merge result of every deployment of that pipeline targeting it.
+type renderSpec struct {
+ enabled bool
+ singleton bool
+}
+
+// resolveTargets maps a deployment's placement onto registered instances:
+// label selector matches AND explicit instance IDs union. Neither set =
+// broadcast (every instance, the v1 semantics).
+func resolveTargets(dep *PipelineDeployment, instances []model.Instance) (map[string]bool, bool) {
+ if len(dep.Placement.Selector) == 0 && len(dep.Placement.Instances) == 0 {
+ return nil, true
+ }
+ known := map[string]bool{}
+ targets := map[string]bool{}
+ for i := range instances {
+ known[instances[i].ID] = true
+ // an empty selector must not match everything — when only
+ // explicit IDs are set, label matching is skipped entirely
+ if len(dep.Placement.Selector) > 0 && instanceMatchesLabels(&instances[i], dep.Placement.Selector) {
+ targets[instances[i].ID] = true
+ }
+ }
+ // explicit IDs resolve against REGISTERED instances only — a row for
+ // an unknown ID is invisible garbage no one pulls; when the instance
+ // later registers, the next cycle picks it up (self-healing scope)
+ for _, id := range dep.Placement.Instances {
+ if known[id] {
+ targets[id] = true
+ }
+ }
+ return targets, false
+}
+
+// instanceMatchesLabels resolves one selector entry per instance: plain
+// keys match Labels; the reserved `group` key matches console-assigned
+// group membership instead (any of the comma-separated groups), since
+// groups are server-owned data that instances never report as labels.
+func instanceMatchesLabels(inst *model.Instance, selector map[string]string) bool {
+ for k, v := range selector {
+ if k == "group" {
+ if !instanceInAnyGroup(inst, v) {
+ return false
+ }
+ continue
+ }
+ if inst.Labels[k] != v {
+ return false
+ }
+ }
+ return true
+}
+
+// instanceInAnyGroup reports whether the instance belongs to any of the
+// comma-separated group names. Empty segments are ignored so "a,, b"
+// behaves like "a,b"; an empty list matches nothing (never broadcast).
+func instanceInAnyGroup(inst *model.Instance, groupsCSV string) bool {
+ for _, g := range strings.Split(groupsCSV, ",") {
+ g = strings.TrimSpace(g)
+ if g == "" {
+ continue
+ }
+ for _, member := range inst.Groups {
+ if member == g {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// reconcilePipeline converges ONE pipeline's deployments into the EXACT
+// set of managed-config rows: one row per target instance (or a single
+// broadcast "*" row). Rows for instances that fell out of every scope are
+// deleted — the instance's next sync reports the file Deleted and the
+// config watcher removes it and stops the task. Nothing more, nothing
+// less: converging means the row set equals the desired set.
+func (module *PipeModule) reconcilePipeline(octx *orm.Context, pipelineID string, deps []*PipelineDeployment, instances []model.Instance) error {
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = pipelineID
+ exists, err := orm.GetV2(octx, &cfg)
+ if err != nil || !exists {
+ return fmt.Errorf("pipeline %s not found (deployments: %s)", pipelineID, deploymentNames(deps))
+ }
+
+ // defensive gate: never distribute a config that no longer compiles
+ // (e.g. a processor plugin went missing after an upgrade)
+ if errs, _ := ValidatePipelineConfig(&cfg); len(errs) > 0 {
+ return fmt.Errorf("pipeline %s failed validation, refusing to distribute: %s",
+ cfg.Name, errs[0].Message)
+ }
+
+ // merge the target set + render flags across all deployments of this
+ // pipeline: an enabled deployment outranks a disabled one targeting
+ // the same instance; singleton wins over all. Any broadcast
+ // deployment subsumes targeting (a "*" row already reaches everyone).
+ broadcast := false
+ desired := map[string]renderSpec{}
+ for _, dep := range deps {
+ targets, bcast := resolveTargets(dep, instances)
+ if bcast {
+ broadcast = true
+ continue
+ }
+ spec := renderSpec{enabled: dep.Enabled, singleton: dep.Placement.Strategy == "singleton"}
+ for id := range targets {
+ prev := desired[id]
+ prev.enabled = prev.enabled || spec.enabled
+ prev.singleton = prev.singleton || spec.singleton
+ desired[id] = prev
+ }
+ }
+ if broadcast {
+ spec := renderSpec{}
+ for _, dep := range deps {
+ spec.enabled = spec.enabled || dep.Enabled
+ spec.singleton = spec.singleton || dep.Placement.Strategy == "singleton"
+ }
+ desired = map[string]renderSpec{configs_server.AllInstancesID: spec}
+ }
+
+ configName := fmt.Sprintf("pipeline-deploy-%s.yml", pipelineID)
+
+ // existing rows of this config name (deployment-owned convention)
+ orm.WithModel(octx, &configs_server.ManagedConfig{})
+ rowsRes, rerr := orm.SearchV2(octx, orm.NewQuery().
+ Filter(orm.TermQuery("name", configName)).Size(500))
+ if rerr != nil {
+ return fmt.Errorf("query managed config %s: %w", configName, rerr)
+ }
+ existingRows, _, _ := elastic.DecodeHits[configs_server.ManagedConfig](rowsRes)
+ existing := map[string]configs_server.ManagedConfig{}
+ for _, row := range existingRows {
+ existing[row.InstanceID] = row
+ }
+
+ // instance syncs apply updates strictly by version ("newest version
+ // per name" wins on the server too) — every write must land ABOVE the
+ // name's current max version or clients keep stale content forever
+ // (e.g. migrating the legacy broadcast row onto per-instance rows)
+ maxVersion := int64(0)
+ for _, row := range existingRows {
+ if row.Version > maxVersion {
+ maxVersion = row.Version
+ }
+ }
+
+ // staged rollout gate (#125): a rollout-bearing deployment stages the
+ // content — only the current wave (canary first, then batches once
+ // the wave reports running it, see rollout.go) may receive it.
+ var gateAllowed map[string]bool
+ if owner := rolloutGateOwner(deps, desired); owner != nil {
+ var sampleSpec renderSpec
+ for _, spec := range desired {
+ sampleSpec = spec
+ break
+ }
+ content := string(renderDeployedPipeline(&cfg, sampleSpec))
+ digest := deployedContentDigest(deployedPipelineMap(&cfg, sampleSpec))
+ // the fallback a rollback would restore: what rows that do not
+ // carry the new content still hold ("" when none exist yet)
+ prevContent := ""
+ for _, row := range existingRows {
+ if row.Content != content {
+ prevContent = row.Content
+ break
+ }
+ }
+ byID := map[string]model.Instance{}
+ for i := range instances {
+ byID[instances[i].ID] = instances[i]
+ }
+ singleton := owner.Placement.Strategy == "singleton"
+ ready := func(id string) bool {
+ inst, ok := byID[id]
+ if !ok {
+ return false
+ }
+ // a frozen report must not count forever: an instance that
+ // died keeps its last STARTED+digest — require a heartbeat
+ // within 3 default sync intervals (30s each) or the wave
+ // holds until a live holder proves the content
+ if inst.Updated != nil && time.Since(*inst.Updated) > 90*time.Second {
+ return false
+ }
+ st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name]
+ if !reported || st.State == "FAILED" {
+ return false
+ }
+ if st.ConfigDigest != digest {
+ return false // still running the old content
+ }
+ if singleton {
+ return true // non-FAILED holders count: the lock picks the runner
+ }
+ return st.State == "STARTED"
+ }
+ targets := make([]string, 0, len(desired))
+ for id := range desired {
+ targets = append(targets, id)
+ }
+ module.rolloutMu.Lock()
+ allowed, next, changed := rolloutPlan(owner.Placement.Rollout, owner.State, targets, digest, prevContent, ready, false)
+ if changed {
+ owner.State = next
+ dctx := orm.NewContext().DirectAccess()
+ orm.WithModel(dctx, &PipelineDeployment{})
+ if serr := orm.Save(dctx, owner); serr != nil {
+ log.Warnf("reconciler: persist rollout state of %s failed: %v (gate held this pass)", owner.Name, serr)
+ } else {
+ log.Debugf("reconciler: rollout of %s now at %d/%d targets (active=%v)", owner.Name, len(next.Rolled), len(targets), next.Active)
+ }
+ }
+ module.rolloutMu.Unlock()
+ gateAllowed = allowed
+ }
+
+ for instID, spec := range desired {
+ if gateAllowed != nil && !gateAllowed[instID] {
+ continue // staged rollout: this instance's wave has not opened yet
+ }
+ content := string(renderDeployedPipeline(&cfg, spec))
+ row, ok := existing[instID]
+ // converged: the pushed content already matches — skip the
+ // version bump and re-push so instances don't rewrite files
+ if ok && row.Content == content {
+ continue
+ }
+ row.Name = configName
+ row.InstanceID = instID
+ row.Content = content
+ row.Version = maxVersion + 1
+ row.Readonly = true // deployment-owned
+ if row.ID == "" {
+ // broadcast keeps the legacy v1 ID so an existing row
+ // updates in place instead of duplicating
+ if instID == configs_server.AllInstancesID {
+ row.ID = fmt.Sprintf("deploy-%s", pipelineID)
+ } else {
+ row.ID = fmt.Sprintf("deploy-%s-%s", pipelineID, instID)
+ }
+ }
+ if serr := orm.Save(octx, &row); serr != nil {
+ return fmt.Errorf("push managed config %s→%s: %w", configName, instID, serr)
+ }
+ maxVersion = row.Version
+ log.Debugf("reconciler: %s → instance %s v%d (%d bytes, enabled=%v singleton=%v)",
+ configName, instID, row.Version, len(content), spec.enabled, spec.singleton)
+ }
+
+ // garbage-collect rows that no deployment targets anymore — the
+ // instance recalls the file (and stops the task) on its next sync
+ for instID, row := range existing {
+ if _, ok := desired[instID]; !ok {
+ if derr := orm.Delete(octx, &row); derr != nil {
+ return fmt.Errorf("recall managed config %s→%s: %w", configName, instID, derr)
+ }
+ log.Infof("reconciler: %s dropped instance %s from scope — recalling config", configName, instID)
+ }
+ }
+ return nil
+}
+
+func deploymentNames(deps []*PipelineDeployment) string {
+ names := ""
+ for i, dep := range deps {
+ if i > 0 {
+ names += ", "
+ }
+ names += dep.Name
+ }
+ return names
+}
+
+// renderDeployedPipeline serializes the pipeline config handed to
+// instances. A disabled spec renders enabled:false + auto_start:false —
+// the instance's config watcher stops (and cleans) the running task on
+// the enabled→disabled transition while keeping the file for instant
+// re-enable.
+func renderDeployedPipeline(cfg *pipeline.PipelineConfigV2, spec renderSpec) []byte {
+ return util.MustToJSONBytes(map[string]interface{}{
+ "pipeline": []interface{}{deployedPipelineMap(cfg, spec)},
+ })
+}
+
+// deployedPipelineMap is the inner object a deployment renders — the six
+// fields that define the deployed behavior. The same shape is digested on
+// BOTH sides (control plane here, instance in RuntimeSnapshot), so a
+// digest comparison proves which content an instance actually runs.
+func deployedPipelineMap(cfg *pipeline.PipelineConfigV2, spec renderSpec) map[string]interface{} {
+ pipelineEnabled := cfg.Enabled
+ pipelineAutoStart := cfg.AutoStart
+ if !spec.enabled {
+ // explicit false pointer (nil would mean "default enabled" at
+ // the instance's isPipelineEnabled check)
+ f := false
+ pipelineEnabled = &f
+ pipelineAutoStart = false
+ }
+ return map[string]interface{}{
+ "name": cfg.Name,
+ "enabled": pipelineEnabled,
+ "auto_start": pipelineAutoStart,
+ "keep_running": cfg.KeepRunning,
+ "singleton": spec.singleton,
+ "processor": cfg.Processors,
+ }
+}
+
+// deployedContentDigest digests a rendered pipeline map — the identity the
+// staged rollout compares instance reports against.
+func deployedContentDigest(m map[string]interface{}) string {
+ return util.MD5digest(string(util.MustToJSONBytes(m)))
+}
+
+// deployedContentDigestOf digests FULL rendered config bytes (the
+// {"pipeline":[{...}]} wrapper a managed-config row carries) by
+// re-canonicalizing the inner object — comparable with
+// deployedContentDigest.
+func deployedContentDigestOf(rendered []byte) string {
+ var wrapper struct {
+ Pipeline []map[string]interface{} `json:"pipeline"`
+ }
+ if err := util.FromJSONBytes(rendered, &wrapper); err != nil || len(wrapper.Pipeline) == 0 {
+ return ""
+ }
+ return deployedContentDigest(wrapper.Pipeline[0])
+}
+
+// loadRegisteredInstances lists the instance universe selectors and
+// explicit targeting resolve against.
+func (h *PipeModule) loadRegisteredInstances(octx *orm.Context) []model.Instance {
+ orm.WithModel(octx, &model.Instance{})
+ res, err := orm.SearchV2(octx, orm.NewQuery().Size(500))
+ if err != nil {
+ return nil
+ }
+ instances, _, _ := elastic.DecodeHits[model.Instance](res)
+ return instances
+}
+
+// deploymentTargetIDs resolves a deployment's scope to concrete instance
+// IDs (broadcast = every registered instance).
+func deploymentTargetIDs(dep *PipelineDeployment, instances []model.Instance) []string {
+ targets, broadcast := resolveTargets(dep, instances)
+ if broadcast {
+ ids := make([]string, 0, len(instances))
+ for i := range instances {
+ ids = append(ids, instances[i].ID)
+ }
+ return ids
+ }
+ ids := make([]string, 0, len(targets))
+ for id := range targets {
+ ids = append(ids, id)
+ }
+ return ids
+}
+
+// rollbackDeploymentContent rewrites EVERY target's managed-config row to
+// the deployment's recorded fallback content at once — a rollback is
+// urgent, no staging (#125 _rollout action=rollback).
+func (h *PipeModule) rollbackDeploymentContent(octx *orm.Context, dep *PipelineDeployment) error {
+ instances := h.loadRegisteredInstances(octx)
+ targets, broadcast := resolveTargets(dep, instances)
+ if broadcast {
+ targets = map[string]bool{}
+ for i := range instances {
+ targets[instances[i].ID] = true
+ }
+ }
+ if len(targets) == 0 {
+ return fmt.Errorf("deployment %s has no targets in scope right now", dep.Name)
+ }
+
+ configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID)
+ orm.WithModel(octx, &configs_server.ManagedConfig{})
+ rowsRes, rerr := orm.SearchV2(octx, orm.NewQuery().
+ Filter(orm.TermQuery("name", configName)).Size(500))
+ if rerr != nil {
+ return fmt.Errorf("query managed config %s: %w", configName, rerr)
+ }
+ rows, _, _ := elastic.DecodeHits[configs_server.ManagedConfig](rowsRes)
+ existing := map[string]configs_server.ManagedConfig{}
+ maxVersion := int64(0)
+ for _, row := range rows {
+ existing[row.InstanceID] = row
+ if row.Version > maxVersion {
+ maxVersion = row.Version
+ }
+ }
+
+ for instID := range targets {
+ row := existing[instID]
+ row.Name = configName
+ row.InstanceID = instID
+ row.Content = dep.State.RollbackContent
+ row.Version = maxVersion + 1
+ row.Readonly = true
+ if row.ID == "" {
+ if instID == configs_server.AllInstancesID {
+ row.ID = fmt.Sprintf("deploy-%s", dep.PipelineID)
+ } else {
+ row.ID = fmt.Sprintf("deploy-%s-%s", dep.PipelineID, instID)
+ }
+ }
+ if err := orm.Save(octx, &row); err != nil {
+ return fmt.Errorf("rollback row %s→%s: %w", configName, instID, err)
+ }
+ maxVersion = row.Version
+ }
+ return nil
+}
+
+// getDeploymentStatus builds the REPORTED state from instance runtime
+// data (desired vs reported comparison for the cluster overview).
+func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStatus {
+ status := DeploymentStatus{
+ DeploymentID: dep.ID,
+ Version: dep.Version,
+ }
+
+ // query instances that have runtime pipeline data
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &model.Instance{})
+ q := orm.NewQuery().Size(200)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ return status
+ }
+ instances, _, _ := elastic.DecodeHits[model.Instance](res)
+
+ // resolve the pipeline NAME — instances run pipelines under their
+ // config name (YAML configs default ID = Name), not the config row ID
+ taskCtx := orm.NewContext().DirectAccess()
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = dep.PipelineID
+ if found, gerr := orm.GetV2(taskCtx, &cfg); gerr != nil || !found {
+ return status
+ }
+
+ targets, broadcast := resolveTargets(dep, instances)
+
+ for _, inst := range instances {
+ inScope := broadcast || targets[inst.ID]
+ // the honest "when did this instance last check in" — the ORM
+ // record's updated stamp refreshes on every _sync heartbeat.
+ // Read-time now() would show dead instances as freshly reporting
+ // forever (#130: a killed gateway kept displaying STARTED with a
+ // brand-new last_seen).
+ lastSeen := time.Now().UTC()
+ if inst.Updated != nil {
+ lastSeen = inst.Updated.UTC()
+ }
+ if inst.RuntimePipelines == "" {
+ // in-scope instances that have not reported yet are part of
+ // the desired count — surface them as pending instead of
+ // silently counting toward a number the list never explains
+ if inScope && dep.Enabled {
+ status.Instances = append(status.Instances, InstanceReport{
+ InstanceID: inst.ID,
+ State: "PENDING",
+ Version: dep.Version,
+ LastSeen: lastSeen,
+ })
+ }
+ continue
+ }
+ var pipelines map[string]struct {
+ State string `json:"state"`
+ Reason string `json:"reason"`
+ }
+ if json.Unmarshal([]byte(inst.RuntimePipelines), &pipelines) != nil {
+ continue
+ }
+ // check if this instance runs the deployment's pipeline
+ if st, ok := pipelines[cfg.Name]; ok {
+ status.Instances = append(status.Instances, InstanceReport{
+ InstanceID: inst.ID,
+ State: st.State,
+ Version: dep.Version,
+ LastSeen: lastSeen,
+ Reason: st.Reason,
+ // stale = the instance still runs the pipeline but the
+ // scope no longer includes it (config recall in flight,
+ // or a recall that failed — visible instead of silent)
+ Stale: !inScope,
+ Cordoned: inst.Cordoned,
+ })
+ }
+ }
+ status.DesiredCount = len(instances)
+ if !broadcast {
+ status.DesiredCount = len(targets)
+ }
+ if dep.Placement.Strategy == "singleton" {
+ status.DesiredCount = 1
+ }
+ status.ReadyCount = 0
+ for _, ir := range status.Instances {
+ if ir.Stale {
+ continue
+ }
+ if ir.State == "STARTED" {
+ status.ReadyCount++
+ }
+ }
+ if dep.State.Active || len(dep.State.Rolled) > 0 {
+ st := dep.State
+ status.Rollout = &st
+ }
+ return status
+}
+
+// ─── Fleet API ───────────────────────────────────────────────────────────
+
+// createDeploymentHandler — POST /pipeline/deployments/
+func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ var dep PipelineDeployment
+ if err := h.DecodeJSON(req, &dep); err != nil {
+ h.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if dep.Name == "" || dep.PipelineID == "" {
+ h.WriteError(w, "name and pipeline_id are required", http.StatusBadRequest)
+ return
+ }
+ if dep.Placement.Strategy == "" {
+ dep.Placement.Strategy = "all" // v1 default (D-12: no sharded)
+ }
+ if dep.Placement.Strategy != "all" && dep.Placement.Strategy != "singleton" {
+ h.WriteError(w, fmt.Sprintf("unknown strategy %q (all|singleton)", dep.Placement.Strategy), http.StatusBadRequest)
+ return
+ }
+ for k, v := range dep.Placement.Selector {
+ if k == "" || v == "" {
+ delete(dep.Placement.Selector, k)
+ }
+ }
+
+ // the referenced pipeline must exist AND compile — a deployment is a
+ // distribution promise; accepting one that ships a broken config to
+ // every instance just moves the failure to the far end
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = dep.PipelineID
+ // GetV2 reports a missing row as an error ("record not found"); both
+ // that and a genuine backend failure fail CLOSED here — a control
+ // plane that cannot read the pipeline must not accept deployments
+ exists, err := orm.GetV2(octx, &cfg)
+ if err != nil || !exists {
+ h.WriteError(w, fmt.Sprintf("pipeline %s not found", dep.PipelineID), http.StatusBadRequest)
+ return
+ }
+ if !h.validateAndReject(w, &cfg) {
+ return
+ }
+
+ dep.Version = 1
+ dep.ID = util.GetUUID()
+
+ if err := orm.Save(octx, &dep); err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ RequestReconcile()
+ h.WriteJSON(w, dep, http.StatusCreated)
+}
+
+// listDeploymentsHandler — GET /pipeline/deployments/
+func (h *PipeModule) listDeploymentsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ q := orm.NewQuery().Size(100)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ deployments, total, _ := elastic.DecodeHits[PipelineDeployment](res)
+ h.WriteJSON(w, util.MapStr{"total": total, "items": deployments}, http.StatusOK)
+}
+
+// getDeploymentHandler — GET /pipeline/deployments/:id
+func (h *PipeModule) getDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ dep := PipelineDeployment{}
+ dep.ID = id
+ exists, err := orm.GetV2(octx, &dep)
+ if err != nil || !exists {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ h.WriteJSON(w, dep, http.StatusOK)
+}
+
+// updateDeploymentHandler — PUT /pipeline/deployments/:id
+func (h *PipeModule) updateDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ var newDep PipelineDeployment
+ if err := h.DecodeJSON(req, &newDep); err != nil {
+ h.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ old := PipelineDeployment{}
+ old.ID = id
+ exists, err := orm.GetV2(octx, &old)
+ if err != nil || !exists {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+
+ // re-validation on update: the edit can rebind pipeline_id — the same
+ // distribution promise as create (must exist AND compile) holds
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = newDep.PipelineID
+ pexists, perr := orm.GetV2(octx, &cfg)
+ if perr != nil || !pexists {
+ h.WriteError(w, fmt.Sprintf("pipeline %s not found", newDep.PipelineID), http.StatusBadRequest)
+ return
+ }
+ if !h.validateAndReject(w, &cfg) {
+ return
+ }
+
+ // normalize placement: default strategy, drop empty selector keys
+ if newDep.Placement.Strategy == "" {
+ newDep.Placement.Strategy = "all"
+ }
+ for k, v := range newDep.Placement.Selector {
+ if k == "" || v == "" {
+ delete(newDep.Placement.Selector, k)
+ }
+ }
+
+ // preserve identity + bump version
+ newDep.ID = id
+ newDep.Version = old.Version + 1
+
+ if err := orm.Save(octx, &newDep); err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ RequestReconcile()
+ h.WriteJSON(w, newDep, http.StatusOK)
+}
+
+// deleteDeploymentHandler — DELETE /pipeline/deployments/:id
+func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ dep := PipelineDeployment{}
+ dep.ID = id
+ if exists, gerr := orm.GetV2(octx, &dep); gerr != nil || !exists {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ if err := orm.Delete(octx, &dep); err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ // cascade: drop the pushed managed config when no OTHER deployment
+ // still references the same pipeline — instances see the file Deleted
+ // on their next sync and the config watcher removes + stops the task.
+ // Rows are per-instance (or the "*" broadcast) — remove them all.
+ others := orm.NewQuery().
+ Filter(orm.TermQuery("pipeline_id", dep.PipelineID)).Size(10)
+ if res, qerr := orm.SearchV2(octx, others); qerr == nil && res != nil {
+ if rows, _, derr := elastic.DecodeHits[PipelineDeployment](res); derr == nil && len(rows) == 0 {
+ configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID)
+ // switch the ORM context to the managed-config model — the
+ // queries above ran against PipelineDeployment
+ orm.WithModel(octx, &configs_server.ManagedConfig{})
+ mcQ := orm.NewQuery().
+ Filter(orm.TermQuery("name", configName)).Size(500)
+ removed := 0
+ if mcRes, merr := orm.SearchV2(octx, mcQ); merr == nil && mcRes != nil {
+ if mcRows, _, mderr := elastic.DecodeHits[configs_server.ManagedConfig](mcRes); mderr == nil {
+ for _, row := range mcRows {
+ _ = orm.Delete(octx, &row)
+ removed++
+ }
+ if removed > 0 {
+ log.Infof("deployment %s deleted; cascaded %d managed config rows of %s (instances will recall the file)", dep.Name, removed, configName)
+ }
+ }
+ }
+ }
+ }
+ RequestReconcile()
+ h.WriteJSON(w, util.MapStr{"deleted": true, "id": id}, http.StatusOK)
+}
+
+// getDeploymentStatusHandler — GET /pipeline/deployments/:id/status
+func (h *PipeModule) getDeploymentStatusHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ dep := PipelineDeployment{}
+ dep.ID = id
+ exists, err := orm.GetV2(octx, &dep)
+ if err != nil || !exists {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ status := h.getDeploymentStatus(&dep)
+ h.WriteJSON(w, status, http.StatusOK)
+}
+
+// getClusterOverviewHandler — GET /pipeline/cluster/overview
+// Matrix of deployments × instances (who runs what, what version).
+func (h *PipeModule) getClusterOverviewHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ q := orm.NewQuery().Filter(orm.TermQuery("enabled", true)).Size(100)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res)
+
+ type overviewEntry struct {
+ Deployment string `json:"deployment"`
+ Pipeline string `json:"pipeline"`
+ Strategy string `json:"strategy"`
+ Version int `json:"version"`
+ Selector map[string]string `json:"selector,omitempty"`
+ }
+ out := make([]overviewEntry, 0, len(deployments))
+ for _, dep := range deployments {
+ out = append(out, overviewEntry{
+ Deployment: dep.Name,
+ Pipeline: dep.PipelineID,
+ Strategy: dep.Placement.Strategy,
+ Version: dep.Version,
+ Selector: dep.Placement.Selector,
+ })
+ }
+ // enrich with reported status
+ type overviewWithStatus struct {
+ overviewEntry
+ Status *DeploymentStatus `json:"status,omitempty"`
+ }
+ outWithStatus := make([]overviewWithStatus, 0, len(deployments))
+ for _, dep := range deployments {
+ entry := overviewEntry{
+ Deployment: dep.Name,
+ Pipeline: dep.PipelineID,
+ Strategy: dep.Placement.Strategy,
+ Version: dep.Version,
+ Selector: dep.Placement.Selector,
+ }
+ st := h.getDeploymentStatus(&dep)
+ outWithStatus = append(outWithStatus, overviewWithStatus{entry, &st})
+ }
+ h.WriteJSON(w, util.MapStr{"deployments": outWithStatus}, http.StatusOK)
+}
diff --git a/modules/pipeline/deployment_targeting_test.go b/modules/pipeline/deployment_targeting_test.go
new file mode 100644
index 000000000..4f840d556
--- /dev/null
+++ b/modules/pipeline/deployment_targeting_test.go
@@ -0,0 +1,313 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/model"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/modules/configs/server"
+ sqlite "infini.sh/framework/modules/sqlite"
+)
+
+// deployTestEnv wires an in-memory control plane: sqlite ORM with the
+// three schemas the reconciler touches (deployments, instances, managed
+// configs) plus one saved, compilable pipeline config. The ORM handler
+// is process-global, so every env gets unique instance/label/pipeline
+// IDs and tests stay isolated.
+type deployTestEnv struct {
+ t *testing.T
+ pid string
+ pipeName string // the pipeline's config NAME (what instances run it under)
+ instA string
+ instB string
+ instC string
+ prod string // label value shared by instA+instC
+ stage string // label value of instB
+ edge string // extra label of instC
+}
+
+var (
+ deployEnvSeq int
+ deployORMOnce sync.Once
+)
+
+// ensureDeployORMHandler installs the ONE sqlite handler for the whole
+// test binary (orm.Register is a process-global last-wins registry — a
+// second registration anywhere would flip later tests onto a handler
+// missing their schemas). Every schema any test in this package needs
+// registers here; side-effect-free so stats tests can call it without
+// minting instances.
+func ensureDeployORMHandler(t *testing.T) {
+ t.Helper()
+ deployORMOnce.Do(func() {
+ handler := &sqlite.SQLiteORM{Config: sqlite.SQLiteConfig{Enabled: true,
+ DBPath: filepath.Join(os.TempDir(), fmt.Sprintf("pipeline-deploy-test-%d.db", time.Now().UnixNano()))}}
+ if err := handler.Open(); err != nil {
+ t.Fatal(err)
+ }
+ orm.Register("deploy-test", handler)
+ orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs-test")
+ orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments-test")
+ orm.MustRegisterSchemaWithIndexName(model.Instance{}, "instance-test")
+ orm.MustRegisterSchemaWithIndexName(server.ManagedConfig{}, "managed-configs-test")
+ orm.MustRegisterSchemaWithIndexName(PipelineStageStat{}, "pipeline-stage-stats-test")
+ if err := orm.InitSchema(); err != nil {
+ t.Fatal(err)
+ }
+ })
+}
+
+func newDeployTestEnv(t *testing.T) *deployTestEnv {
+ deployEnvSeq++
+ sfx := fmt.Sprintf("%d", deployEnvSeq)
+ ensureDeployORMHandler(t)
+
+ env := &deployTestEnv{
+ t: t, pid: "pipe-" + sfx, pipeName: "deploy-test-pipeline-" + sfx,
+ instA: "gw-a-" + sfx, instB: "gw-b-" + sfx, instC: "gw-c-" + sfx,
+ prod: "prod-" + sfx, stage: "staging-" + sfx, edge: "edge-" + sfx,
+ }
+ env.saveInstance(env.instA, map[string]string{"env": env.prod})
+ env.saveInstance(env.instB, map[string]string{"env": env.stage})
+ env.saveInstance(env.instC, map[string]string{"env": env.prod, "tier": env.edge})
+
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &pipeline.PipelineConfigV2{})
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = env.pid
+ cfg.Name = env.pipeName
+ cfg.Processors = []map[string]interface{}{
+ {"validate_test_echo": map[string]interface{}{"message": "hi"}},
+ }
+ if err := orm.Save(octx, &cfg); err != nil {
+ t.Fatal(err)
+ }
+ return env
+}
+
+func (e *deployTestEnv) saveInstance(id string, labels map[string]string) {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &model.Instance{})
+ inst := model.Instance{}
+ inst.ID = id
+ inst.Labels = labels
+ if err := orm.Save(octx, &inst); err != nil {
+ e.t.Fatal(err)
+ }
+}
+
+func (e *deployTestEnv) saveDeployment(dep *PipelineDeployment) {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ if err := orm.Save(octx, dep); err != nil {
+ e.t.Fatal(err)
+ }
+}
+
+// rows returns the deployment-owned managed-config rows keyed by
+// instance_id.
+func (e *deployTestEnv) rows() map[string]server.ManagedConfig {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &server.ManagedConfig{})
+ res, err := orm.SearchV2(octx, orm.NewQuery().Filter(orm.TermQuery("name", fmt.Sprintf("pipeline-deploy-%s.yml", e.pid))).Size(100))
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ rows, _, _ := elastic.DecodeHits[server.ManagedConfig](res)
+ out := map[string]server.ManagedConfig{}
+ for _, r := range rows {
+ out[r.InstanceID] = r
+ }
+ return out
+}
+
+func (e *deployTestEnv) reconcile() {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &model.Instance{})
+ instRes, err := orm.SearchV2(octx, orm.NewQuery().Size(100))
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ instances, _, _ := elastic.DecodeHits[model.Instance](instRes)
+
+ orm.WithModel(octx, &PipelineDeployment{})
+ res, err := orm.SearchV2(octx, orm.NewQuery().Size(100))
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res)
+ for i := range deployments {
+ if err := e.module().reconcilePipeline(octx, e.pid, []*PipelineDeployment{&deployments[i]}, instances); err != nil {
+ e.t.Fatalf("reconcile: %v", err)
+ }
+ }
+}
+
+func (e *deployTestEnv) module() *PipeModule { return &PipeModule{} }
+
+func TestResolveTargetsModes(t *testing.T) {
+ instances := []model.Instance{
+ {ORMObjectBase: orm.ORMObjectBase{ID: "a"}, Labels: map[string]string{"env": "prod"}},
+ {ORMObjectBase: orm.ORMObjectBase{ID: "b"}, Labels: map[string]string{"env": "staging"}},
+ }
+ // no selector + no instances → broadcast
+ targets, broadcast := resolveTargets(&PipelineDeployment{}, instances)
+ if !broadcast || targets != nil {
+ t.Fatalf("expected broadcast, got %v %v", targets, broadcast)
+ }
+ // label match
+ targets, broadcast = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod"}}}, instances)
+ if broadcast || len(targets) != 1 || !targets["a"] {
+ t.Fatalf("expected {a}, got %v broadcast=%v", targets, broadcast)
+ }
+ // group membership: the reserved `group` selector key matches
+ // console-assigned groups, comma-separated values act as OR
+ grouped := []model.Instance{
+ {ORMObjectBase: orm.ORMObjectBase{ID: "g1"}, Groups: []string{"edge", "cn"}},
+ {ORMObjectBase: orm.ORMObjectBase{ID: "g2"}, Labels: map[string]string{"group": "edge"}},
+ {ORMObjectBase: orm.ORMObjectBase{ID: "g3"}},
+ }
+ targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge"}}}, grouped)
+ if len(targets) != 1 || !targets["g1"] {
+ t.Fatalf("group selector should match membership only, got %v", targets)
+ }
+ targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge,cn , missing"}}}, grouped)
+ if len(targets) != 1 || !targets["g1"] {
+ t.Fatalf("multi-group OR should still match only g1, got %v", targets)
+ }
+ // labels and groups are separate namespaces: a label named group
+ // cannot satisfy the group selector, and vice versa
+ targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge", "env": "prod"}}}, grouped)
+ if len(targets) != 0 {
+ t.Fatalf("group + unrelated label must AND to nothing here, got %v", targets)
+ }
+
+ // multi-label AND
+ targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod", "tier": "edge"}}}, instances)
+ if len(targets) != 0 {
+ t.Fatalf("expected no match for AND miss, got %v", targets)
+ }
+ // explicit IDs, unknown dropped
+ targets, broadcast = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Instances: []string{"b", "ghost"}}}, instances)
+ if broadcast || len(targets) != 1 || !targets["b"] {
+ t.Fatalf("expected {b}, got %v broadcast=%v", targets, broadcast)
+ }
+ // explicit + labels union
+ targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod"}, Instances: []string{"b"}}}, instances)
+ if len(targets) != 2 || !targets["a"] || !targets["b"] {
+ t.Fatalf("expected {a,b}, got %v", targets)
+ }
+}
+
+func TestDeploymentLabelScopeConvergesExactly(t *testing.T) {
+ env := newDeployTestEnv(t)
+ env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "prod-only", PipelineID: env.pid, Enabled: true,
+ Placement: PlacementSpec{Strategy: "all", Selector: map[string]string{"env": env.prod}}})
+
+ env.reconcile()
+ rows := env.rows()
+ // exactly the two env=prod instances, no broadcast row, no staging
+ if len(rows) != 2 || rows[env.instA].ID == "" || rows[env.instC].ID == "" || rows[env.instB].ID != "" || rows["*"].ID != "" {
+ t.Fatalf("expected exactly {gw-a,gw-c}, got %v", rowIDs(rows))
+ }
+ v1 := rows[env.instA].Version
+
+ // converge again → no version churn, still exactly two rows
+ env.reconcile()
+ rows = env.rows()
+ if len(rows) != 2 || rows[env.instA].Version != v1 {
+ t.Fatalf("expected stable convergence, got %s v%d", rowIDs(rows), rows["gw-a"].Version)
+ }
+}
+
+func TestDeploymentScopeChangeRecallsOldInstances(t *testing.T) {
+ env := newDeployTestEnv(t)
+ env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "mover", PipelineID: env.pid, Enabled: true,
+ Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA}}})
+
+ env.reconcile()
+ if rows := env.rows(); len(rows) != 1 || rows[env.instA].ID == "" {
+ t.Fatalf("expected {gw-a}, got %v", rowIDs(rows))
+ }
+ oldVersion := env.rows()[env.instA].Version
+
+ // scope A → B: switch to gw-b; gw-a must be recalled (row deleted),
+ // gw-b's row must land ABOVE gw-a's old version or gw-a's client
+ // would refuse the "update" it never sees... (version monotonicity
+ // per name is what keeps instances from sticking to stale content)
+ dep := PipelineDeployment{}
+ {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ dep.ID = "d1"
+ if found, err := orm.GetV2(octx, &dep); err != nil || !found {
+ t.Fatal("deployment vanished")
+ }
+ }
+ dep.Placement.Instances = []string{env.instB}
+ env.saveDeployment(&dep)
+
+ env.reconcile()
+ rows := env.rows()
+ if len(rows) != 1 || rows[env.instB].ID == "" {
+ t.Fatalf("expected scope change to leave exactly {gw-b}, got %v", rowIDs(rows))
+ }
+ if rows[env.instB].Version <= oldVersion {
+ t.Fatalf("new row version %d must exceed old %d (per-name monotonicity)", rows["gw-b"].Version, oldVersion)
+ }
+}
+
+func TestDeploymentBroadcastMigratesToTargeted(t *testing.T) {
+ env := newDeployTestEnv(t)
+ env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "legacy", PipelineID: env.pid, Enabled: true,
+ Placement: PlacementSpec{Strategy: "all"}})
+
+ env.reconcile()
+ rows := env.rows()
+ if len(rows) != 1 || rows["*"].ID == "" {
+ t.Fatalf("expected broadcast row, got %v", rowIDs(rows))
+ }
+ broadcastVersion := rows["*"].Version
+
+ // edit the deployment to target one instance — the legacy "*" row
+ // must be replaced by the per-instance row, seeded above its version
+ dep := PipelineDeployment{}
+ {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ dep.ID = "d1"
+ if found, err := orm.GetV2(octx, &dep); err != nil || !found {
+ t.Fatal("deployment vanished")
+ }
+ }
+ dep.Placement.Selector = map[string]string{"tier": env.edge}
+ env.saveDeployment(&dep)
+
+ env.reconcile()
+ rows = env.rows()
+ if len(rows) != 1 || rows[env.instC].ID == "" || rows["*"].ID != "" {
+ t.Fatalf("expected {gw-c} only, got %v", rowIDs(rows))
+ }
+ if rows[env.instC].Version <= broadcastVersion {
+ t.Fatalf("migrated row version %d must exceed broadcast %d", rows["gw-c"].Version, broadcastVersion)
+ }
+}
+
+func rowIDs(rows map[string]server.ManagedConfig) []string {
+ out := []string{}
+ for id := range rows {
+ out = append(out, id)
+ }
+ return out
+}
diff --git a/modules/pipeline/dryrun/dryrun.go b/modules/pipeline/dryrun/dryrun.go
new file mode 100644
index 000000000..e7a6074ce
--- /dev/null
+++ b/modules/pipeline/dryrun/dryrun.go
@@ -0,0 +1,240 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+// Package dryrun is the unified dry-run engine: replay sample payloads
+// through a chain with per-step before/after snapshots, field diffs and
+// side-effect capture (queue/lane writes are captured, never delivered).
+package dryrun
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+// Sample is one replay input: raw payload bytes plus the codec that
+// decodes them ("" = default).
+type Sample struct {
+ Raw []byte `json:"raw"`
+ Codec string `json:"codec"`
+}
+
+// Options controls a run.
+type Options struct {
+ // ScopeCheck compiles the chain with contract validation (mismatches
+ // surface in Report.Compiled instead of running).
+ ScopeCheck bool `json:"scope_check"`
+ // SideEffects: "capture" (default — writes collected in the report) |
+ // "passthrough" (real writes; explicit opt-in).
+ SideEffects string `json:"side_effects"`
+ // LLMPolicy: "real" (default) | "mock" (recorded replay; host-app
+ // supplied via the mock registry).
+ LLMPolicy string `json:"llm_policy"`
+}
+
+// FieldDiff is the added/removed/changed path set between two snapshots.
+type FieldDiff struct {
+ Added util.MapStr `json:"added,omitempty"`
+ Removed util.MapStr `json:"removed,omitempty"`
+ Changed util.MapStr `json:"changed,omitempty"`
+}
+
+// StepResult is one processor's effect on one sample.
+type StepResult struct {
+ Step int `json:"step"`
+ Processor string `json:"processor"`
+ Before util.MapStr `json:"before,omitempty"`
+ After util.MapStr `json:"after,omitempty"`
+ Diff FieldDiff `json:"diff"`
+ DurationMs int64 `json:"duration_ms"`
+ Outcome string `json:"outcome"` // ok | error | dropped
+ Err string `json:"err,omitempty"`
+}
+
+// StepDesc is the compile-phase outcome per entry.
+type StepDesc struct {
+ Step int `json:"step"`
+ Processor string `json:"processor"`
+ OK bool `json:"ok"`
+ Note string `json:"note,omitempty"`
+}
+
+// CapturedWrite is a side effect intercepted during the run.
+type CapturedWrite = pipeline.DryRunWrite
+
+// Report is the full dry-run result.
+type Report struct {
+ Compiled []StepDesc `json:"compiled"`
+ Steps [][]StepResult `json:"steps"` // per sample × per step
+ SideEffects []CapturedWrite `json:"side_effects"`
+}
+
+// Run replays samples through the chain. The chain runs in record scope
+// per sample (the record convention); batch processors receive the
+// single-record slice.
+func Run(chain []map[string]interface{}, samples []Sample, opts Options) (*Report, error) {
+ if opts.SideEffects == "" {
+ opts.SideEffects = "capture"
+ }
+ cfgs, err := toConfigs(chain)
+ if err != nil {
+ return nil, err
+ }
+
+ report := &Report{}
+
+ // compile phase
+ scope := pipeline.ScopeInfo{Kind: pipeline.ScopeRecord, Strict: opts.ScopeCheck}
+ procs, compileErr := pipeline.NewPipelineWithScope(cfgs, scope, true)
+ for i, m := range chain {
+ desc := StepDesc{Step: i, Processor: firstKey(m), OK: true}
+ report.Compiled = append(report.Compiled, desc)
+ }
+ if compileErr != nil {
+ // surface the compile error on the failing entry; when scope-check
+ // is on the error is the deliverable (not a run failure)
+ if n := len(report.Compiled); n > 0 {
+ report.Compiled[n-1].OK = false
+ report.Compiled[n-1].Note = compileErr.Error()
+ } else {
+ report.Compiled = append(report.Compiled, StepDesc{
+ Step: 0, Processor: "", OK: false, Note: compileErr.Error(),
+ })
+ }
+ if opts.ScopeCheck {
+ return report, nil
+ }
+ return report, compileErr
+ }
+
+ // replay phase
+ sink := &[]pipeline.DryRunWrite{}
+ for _, s := range samples {
+ results := replaySample(procs, s, opts, sink)
+ report.Steps = append(report.Steps, results)
+ }
+ report.SideEffects = *sink
+ return report, nil
+}
+
+func replaySample(procs *pipeline.Processors, s Sample, opts Options, sink *[]CapturedWrite) []StepResult {
+ codec := pipeline.GetRecordCodec(s.Codec)
+ rec, err := codec.Decode(s.Raw)
+ if err != nil {
+ return []StepResult{{Step: 0, Processor: "codec:" + s.Codec, Outcome: "error", Err: err.Error()}}
+ }
+
+ ctxCfg := pipeline.PipelineConfigV2{Name: "dry-run"}
+ ctx := pipeline.AcquireContext(ctxCfg)
+ ctx.Context = context.Background()
+ ctx.Started()
+ defer pipeline.ReleaseContext(ctx)
+ ctx.Set(pipeline.RecordContextKey, rec)
+ pipeline.EnableDryRun(ctx, sink, opts.SideEffects == "capture")
+
+ var results []StepResult
+ for i, p := range procs.List {
+ before := snapshotFields(rec)
+ start := time.Now()
+ outcome, errMsg := "ok", ""
+ var runErr error
+ if bp, isBatch := p.(pipeline.BatchProcessor); isBatch {
+ runErr = bp.ProcessBatch(ctx, []*event.Event{rec})
+ } else {
+ runErr = p.Process(ctx)
+ }
+ if runErr != nil {
+ outcome, errMsg = "error", runErr.Error()
+ } else if pipeline.IsDropped(rec) {
+ outcome = "dropped"
+ }
+ after := snapshotFields(rec)
+ results = append(results, StepResult{
+ Step: i, Processor: p.Name(),
+ Before: before, After: after,
+ Diff: diffMaps(before, after),
+ DurationMs: time.Since(start).Milliseconds(),
+ Outcome: outcome, Err: errMsg,
+ })
+ if runErr != nil {
+ break
+ }
+ }
+ return results
+}
+
+func snapshotFields(rec *event.Event) util.MapStr {
+ if rec == nil || rec.Fields == nil {
+ return util.MapStr{}
+ }
+ b, err := util.ToJSONBytes(rec.Fields)
+ if err != nil {
+ return util.MapStr{}
+ }
+ m := util.MapStr{}
+ _ = util.FromJSONBytes(b, &m)
+ return m
+}
+
+// diffMaps walks two flat snapshots; nested maps recurse with dotted paths.
+func diffMaps(before, after util.MapStr) FieldDiff {
+ d := FieldDiff{Added: util.MapStr{}, Removed: util.MapStr{}, Changed: util.MapStr{}}
+ seenAfter := map[string]bool{}
+ for k, av := range after {
+ seenAfter[k] = true
+ bv, ok := before[k]
+ if !ok {
+ d.Added[k] = av
+ continue
+ }
+ aSub, aIsMap := av.(map[string]interface{})
+ bSub, bIsMap := bv.(map[string]interface{})
+ if aIsMap && bIsMap {
+ sub := diffMaps(util.MapStr(bSub), util.MapStr(aSub))
+ for sk, sv := range sub.Added {
+ d.Added[k+"."+sk] = sv
+ }
+ for sk, sv := range sub.Removed {
+ d.Removed[k+"."+sk] = sv
+ }
+ for sk, sv := range sub.Changed {
+ d.Changed[k+"."+sk] = sv
+ }
+ continue
+ }
+ if fmt.Sprint(av) != fmt.Sprint(bv) {
+ d.Changed[k] = util.MapStr{"from": bv, "to": av}
+ }
+ }
+ for k, bv := range before {
+ if !seenAfter[k] {
+ d.Removed[k] = bv
+ }
+ }
+ return d
+}
+
+func toConfigs(chain []map[string]interface{}) ([]*config.Config, error) {
+ c, err := config.NewConfigFrom(chain)
+ if err != nil {
+ return nil, err
+ }
+ var list []*config.Config
+ if err := c.Unpack(&list); err != nil {
+ return nil, err
+ }
+ return list, nil
+}
+
+func firstKey(m map[string]interface{}) string {
+ for k := range m {
+ return k
+ }
+ return "?"
+}
diff --git a/modules/pipeline/dryrun/dryrun_test.go b/modules/pipeline/dryrun/dryrun_test.go
new file mode 100644
index 000000000..f6bda6caf
--- /dev/null
+++ b/modules/pipeline/dryrun/dryrun_test.go
@@ -0,0 +1,87 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package dryrun
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/pipeline"
+)
+
+// testDouble is a local processor registration for replay tests (the
+// host package cannot be imported here — import cycle).
+type testDouble struct{ name string }
+
+func (p *testDouble) Name() string { return p.name }
+func (p *testDouble) Process(ctx *pipeline.Context) error { return nil }
+
+func init() {
+ pipeline.RegisterProcessorPlugin("dryrun_noop", func(c *config.Config) (pipeline.Processor, error) {
+ return &testDouble{"dryrun_noop"}, nil
+ })
+}
+
+func upperChain(t *testing.T) []map[string]interface{} {
+ t.Helper()
+ // for_each hosts a test_upper record processor (registered by the
+ // modules/pipeline test suite init is NOT exported — use for_each with
+ // the otel codec and mutate-style echo)
+ return []map[string]interface{}{
+ {"for_each": map[string]interface{}{
+ "codec": "raw",
+ "processor": []interface{}{map[string]interface{}{"dryrun_noop": map[string]interface{}{}}},
+ }},
+ }
+}
+
+func TestRun_ReportShapeAndSideEffectCapture(t *testing.T) {
+ report, err := Run(
+ []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}},
+ []Sample{{Raw: []byte(`{"payload":{"message":"hello"}}`), Codec: "raw"}},
+ Options{},
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(report.Steps) != 1 || len(report.Steps[0]) == 0 {
+ t.Fatalf("steps shape: %+v", report.Steps)
+ }
+ if report.Steps[0][0].Outcome != "ok" {
+ t.Fatalf("outcome: %+v", report.Steps[0][0])
+ }
+}
+
+func TestRun_FieldDiffDetectsMutations(t *testing.T) {
+ // raw codec echoes bytes back; before/after must be equal → empty diff
+ report, err := Run(
+ []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}},
+ []Sample{{Raw: []byte(`{"payload":{"message":"m"}}`), Codec: "raw"}},
+ Options{},
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ d := report.Steps[0][0].Diff
+ if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 {
+ t.Fatalf("echo chain must not diff: %+v", d)
+ }
+}
+
+func TestRun_ScopeCheckSurfacesContractMismatches(t *testing.T) {
+ // batch-only test processor inside a nested if branch → compile error
+ // surfaced in Compiled, not a panic
+ report, err := Run(
+ []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}},
+ nil,
+ Options{ScopeCheck: true},
+ )
+ if err != nil {
+ t.Fatalf("scope-check run errored: %v", err)
+ }
+ if len(report.Compiled) == 0 {
+ t.Fatal("compiled phase empty")
+ }
+}
diff --git a/modules/pipeline/empty_config_audit_test.go b/modules/pipeline/empty_config_audit_test.go
new file mode 100644
index 000000000..319063b1a
--- /dev/null
+++ b/modules/pipeline/empty_config_audit_test.go
@@ -0,0 +1,102 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline_test
+
+import (
+ "sort"
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/pipeline"
+ // register the OSS processor catalog this repo ships (direct imports:
+ // the plugins umbrella pulls optional deps like clashx). Enterprise
+ // processors live in the separate enterprise-plugins overlay repo and
+ // are audited by the twin test in that repo — importing them here would
+ // break CI checkouts where the overlay is absent.
+ _ "infini.sh/framework/plugins/elastic/bulk_indexing"
+ _ "infini.sh/framework/plugins/elastic/indexing_merge"
+ _ "infini.sh/framework/plugins/elastic/json_indexing"
+ _ "infini.sh/framework/plugins/elastic/lane_bulk"
+ _ "infini.sh/framework/plugins/elastic/merge_to_bulk"
+ _ "infini.sh/framework/plugins/http"
+ _ "infini.sh/framework/plugins/queue/consumer"
+ _ "infini.sh/framework/plugins/queue/queue_output"
+ _ "infini.sh/framework/plugins/replay"
+ _ "infini.sh/framework/plugins/smtp"
+)
+
+// TestEmptyConfigAudit reports which processors accept an EMPTY config.
+// Empty-accept is legitimate for processors whose every parameter has a
+// sane default (drop_event, clone...), but a gap for processors that
+// cannot work without a required input (dissect needs pattern...).
+// The allowlist below documents the audited verdict per processor; a new
+// processor that lands here must either gain required-field validation
+// or be consciously added to the allowlist.
+func TestEmptyConfigAudit(t *testing.T) {
+ // audited: every parameter has a working default (OSS catalog only —
+ // enterprise verdicts live in the twin test in the enterprise repo)
+ allowlist := map[string]bool{
+ "bulk_indexing": true, // detect_active_queue auto-discovery mode
+ "for_each": true, // host — inner chain required, checked separately
+ "merge_to_bulk": true,
+ "output": true, // queue defaults to event routing
+ "replay": true, // localhost:9200 test-utility default
+ "tap": true, // taps everything by default
+ }
+
+ ctors := pipeline.ProcessorConstructors()
+ names := make([]string, 0, len(ctors))
+ for name := range ctors {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+
+ accepted := []string{}
+ panicked := []string{}
+ for _, name := range names {
+ ctor := ctors[name]
+ cfg, err := config.NewConfigFrom(map[string]interface{}{})
+ if err != nil {
+ t.Fatalf("build empty config: %v", err)
+ }
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ panicked = append(panicked, name)
+ }
+ }()
+ if _, err := ctor(cfg); err == nil {
+ accepted = append(accepted, name)
+ }
+ }()
+ }
+ if len(panicked) > 0 {
+ t.Fatalf("processors PANIC on empty config (must return an error instead): %v", panicked)
+ }
+
+ // namespace aliases (parsing:csv == csv) share constructors with the
+ // bare names — normalize before the allowlist lookup; test doubles
+ // registered by sibling test files are out of catalog scope
+ bare := func(name string) string {
+ if i := strings.Index(name, ":"); i >= 0 {
+ return name[i+1:]
+ }
+ return name
+ }
+ gaps := []string{}
+ for _, name := range accepted {
+ if strings.HasPrefix(name, "test_") {
+ continue
+ }
+ if !allowlist[bare(name)] {
+ gaps = append(gaps, name)
+ }
+ }
+ if len(gaps) > 0 {
+ t.Fatalf("processors accepting empty config without an audited default (add required-field validation or allowlist): %v", gaps)
+ }
+ t.Logf("empty-accept: %d processors, all audited", len(accepted))
+}
diff --git a/modules/pipeline/fleet_control.go b/modules/pipeline/fleet_control.go
new file mode 100644
index 000000000..6db5cf7c7
--- /dev/null
+++ b/modules/pipeline/fleet_control.go
@@ -0,0 +1,177 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "strings"
+
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/model"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+ configs_server "infini.sh/framework/modules/configs/server"
+)
+
+// Fleet control (#125): transient commands never ride the config channel
+// (SPEC §8.5) — the control plane forwards them to each target instance
+// THROUGH the agent-dialed reverse channel. Managed instances sit behind
+// NAT/firewalls and are not directly reachable; the reverse websocket is
+// the only way in.
+
+// seam for tests: the reverse-channel client is a process-global that
+// needs a live manager websocket — handler tests swap these out.
+var (
+ reverseProxyRequest = configs_server.ReverseProxyRequest
+ reverseIsConnected = configs_server.ReverseIsConnected
+)
+
+// broadcastDeploymentActionHandler — POST /pipeline/deployments/:id/_pause|_resume|_stop
+// Sends the control command to every instance in the deployment's scope,
+// addressed by the pipeline's task name (instances key tasks by config
+// name). The optional ?generation=N rides along verbatim: instances still
+// on another config generation refuse the command with 409 (#120) — the
+// split-brain guard the broadcast transport was missing.
+func (h *PipeModule) broadcastDeploymentActionHandler(action string) httprouter.Handle {
+ return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+
+ dep := PipelineDeployment{}
+ dep.ID = id
+ exists, err := orm.GetV2(octx, &dep)
+ if err != nil || !exists {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ // resolve the pipeline NAME the instances run it under
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = dep.PipelineID
+ if found, gerr := orm.GetV2(octx, &cfg); gerr != nil || !found {
+ h.WriteError(w, fmt.Sprintf("pipeline %s not found", dep.PipelineID), http.StatusBadRequest)
+ return
+ }
+
+ instances := h.loadRegisteredInstances(octx)
+ targets := deploymentTargetIDs(&dep, instances)
+ if len(targets) == 0 {
+ h.WriteJSON(w, util.MapStr{"total": 0, "delivered": 0, "results": map[string]interface{}{}}, http.StatusOK)
+ return
+ }
+
+ gen := req.URL.Query().Get("generation")
+ results := map[string]util.MapStr{}
+ delivered := 0
+ for _, instID := range targets {
+ results[instID] = h.controlOne(instID, cfg.Name, action, gen)
+ if ok, _ := results[instID]["ok"].(bool); ok {
+ delivered++
+ }
+ }
+ h.WriteJSON(w, util.MapStr{
+ "total": len(targets),
+ "delivered": delivered,
+ "results": results,
+ }, http.StatusOK)
+ }
+}
+
+// controlOne issues one instance control command over the reverse channel
+// and shapes the per-instance outcome for the broadcast aggregate.
+func (h *PipeModule) controlOne(instID, taskName, action, generation string) util.MapStr {
+ if reverseIsConnected != nil && !reverseIsConnected(instID) {
+ return util.MapStr{"ok": false, "error": "reverse channel not connected (instance offline or pre-reverse-channel build)"}
+ }
+ path := fmt.Sprintf("/pipeline/task/%s/_%s", url.PathEscape(taskName), action)
+ if generation != "" {
+ path += "?generation=" + url.QueryEscape(generation)
+ }
+ res, err := reverseProxyRequest(instID, &util.Request{Method: http.MethodPost, Path: path})
+ if err != nil {
+ return util.MapStr{"ok": false, "error": err.Error()}
+ }
+ out := util.MapStr{"ok": res.StatusCode >= 200 && res.StatusCode < 300, "status": res.StatusCode}
+ if len(res.Body) > 0 {
+ out["body"] = string(res.Body)
+ }
+ return out
+}
+
+// instanceActionHandler — POST /pipeline/instances/:id/_drain|_cordon|_uncordon
+// Forwards the instance-level maintenance operations (#121) to one managed
+// instance through the reverse channel. Query strings pass through (drain
+// accepts ?timeout_seconds=).
+func (h *PipeModule) instanceActionHandler(action string) httprouter.Handle {
+ return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ if !h.instanceExists(req, id) {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ path := "/pipeline/_" + action
+ if qs := req.URL.RawQuery; qs != "" {
+ path += "?" + qs
+ }
+ res, err := h.forwardToInstance(id, &util.Request{Method: http.MethodPost, Path: path})
+ h.proxyReply(w, res, err)
+ }
+}
+
+// instanceProxyHandler — GET /pipeline/instances/:id/proxy?path=/pipeline/...
+// Read-only passthrough to an instance's own pipeline API (task detail,
+// stages, flight — single-instance debugging without shell access).
+// Methods stay GET: anything mutating has a dedicated action route above.
+func (h *PipeModule) instanceProxyHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ target := req.URL.Query().Get("path")
+ // allowlist: pipeline-scope reads only — the proxy must not become an
+ // open relay onto the instance's management plane
+ if !strings.HasPrefix(target, "/pipeline/") || strings.Contains(target, "..") {
+ h.Error400(w, "path must target a /pipeline/ endpoint")
+ return
+ }
+ if !h.instanceExists(req, id) {
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ res, err := h.forwardToInstance(id, &util.Request{Method: http.MethodGet, Path: target})
+ h.proxyReply(w, res, err)
+}
+
+// forwardToInstance runs one logical request over the reverse channel.
+func (h *PipeModule) forwardToInstance(instID string, r *util.Request) (*util.Result, error) {
+ if reverseIsConnected != nil && !reverseIsConnected(instID) {
+ return nil, fmt.Errorf("instance %s is not connected through the reverse channel", instID)
+ }
+ return reverseProxyRequest(instID, r)
+}
+
+// proxyReply maps a forwarded result onto the HTTP response, passing the
+// instance's status code and body through verbatim (a 409 from a
+// generation-guarded instance must stay a 409 for the caller).
+func (h *PipeModule) proxyReply(w http.ResponseWriter, res *util.Result, err error) {
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ status := res.StatusCode
+ if status == 0 {
+ status = http.StatusBadGateway
+ }
+ w.Header().Set("Content-Type", util.ContentTypeJson)
+ w.WriteHeader(status)
+ _, _ = w.Write(res.Body)
+}
+
+func (h *PipeModule) instanceExists(req *http.Request, id string) bool {
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ inst := model.Instance{}
+ inst.ID = id
+ found, err := orm.GetV2(octx, &inst)
+ return err == nil && found
+}
diff --git a/modules/pipeline/fleet_control_test.go b/modules/pipeline/fleet_control_test.go
new file mode 100644
index 000000000..eb9ddba5d
--- /dev/null
+++ b/modules/pipeline/fleet_control_test.go
@@ -0,0 +1,182 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/util"
+)
+
+// The reverse channel is stubbed at the seam: tests capture what would be
+// sent down the websocket and shape the response per instance.
+type fakeReverse struct {
+ mu sync.Mutex
+ calls []fakeCall
+ statusOf map[string]int // instance → response status (default 200)
+}
+
+type fakeCall struct {
+ Instance string
+ Method string
+ Path string
+}
+
+func (f *fakeReverse) install(t *testing.T) {
+ origProxy, origConn := reverseProxyRequest, reverseIsConnected
+ reverseProxyRequest = func(peerID string, req *util.Request) (*util.Result, error) {
+ f.mu.Lock()
+ f.calls = append(f.calls, fakeCall{Instance: peerID, Method: req.Method, Path: req.Path})
+ f.mu.Unlock()
+ status := http.StatusOK
+ if s, ok := f.statusOf[peerID]; ok {
+ status = s
+ }
+ return &util.Result{StatusCode: status, Body: []byte(`{"ok":true}`)}, nil
+ }
+ reverseIsConnected = func(string) bool { return true }
+ t.Cleanup(func() { reverseProxyRequest, reverseIsConnected = origProxy, origConn })
+}
+
+func (f *fakeReverse) recorded() []fakeCall {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]fakeCall(nil), f.calls...)
+}
+
+func callHandler(t *testing.T, h httprouter.Handle, method, target string, params httprouter.Params) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(method, target, nil)
+ w := httptest.NewRecorder()
+ h(w, req, params)
+ return w
+}
+
+func TestBroadcastDeploymentAction(t *testing.T) {
+ env := newDeployTestEnv(t)
+ dep := &PipelineDeployment{
+ PipelineID: env.pid,
+ Enabled: true,
+ Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA, env.instB}},
+ }
+ dep.ID = "d1" // shared upsert id (deployment_targeting_test.go convention)
+ dep.Name = "broadcast-test"
+ env.saveDeployment(dep)
+
+ fr := &fakeReverse{statusOf: map[string]int{env.instB: http.StatusConflict}} // B is on another generation
+ fr.install(t)
+
+ h := (&PipeModule{}).broadcastDeploymentActionHandler("pause")
+ w := callHandler(t, h, http.MethodPost, "/pipeline/deployments/"+dep.ID+"/_pause?generation=7",
+ httprouter.Params{{Key: "id", Value: dep.ID}})
+ if w.Code != http.StatusOK {
+ t.Fatalf("aggregate must be 200 even with per-instance failures: %d %s", w.Code, w.Body.String())
+ }
+ out := util.MapStr{}
+ if err := util.FromJSONBytes(w.Body.Bytes(), &out); err != nil {
+ t.Fatal(err)
+ }
+ if out["total"].(float64) != 2 || out["delivered"].(float64) != 1 {
+ t.Fatalf("total=2 delivered=1 expected: %v", out)
+ }
+
+ calls := fr.recorded()
+ if len(calls) != 2 {
+ t.Fatalf("both targets must receive the command: %+v", calls)
+ }
+ for _, c := range calls {
+ if c.Method != http.MethodPost {
+ t.Fatalf("control commands are POSTs: %+v", c)
+ }
+ if c.Path != "/pipeline/task/"+env.pipeName+"/_pause?generation=7" {
+ t.Fatalf("command must target the task by NAME with the generation guard: %+v", c)
+ }
+ }
+}
+
+func TestBroadcastUnreachableInstance(t *testing.T) {
+ env := newDeployTestEnv(t)
+ dep := &PipelineDeployment{
+ PipelineID: env.pid,
+ Enabled: true,
+ Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA}},
+ }
+ dep.ID = "d1" // shared upsert id
+ dep.Name = "broadcast-offline"
+ env.saveDeployment(dep)
+
+ origProxy, origConn := reverseProxyRequest, reverseIsConnected
+ reverseIsConnected = func(string) bool { return false } // nobody dialed in
+ t.Cleanup(func() { reverseProxyRequest, reverseIsConnected = origProxy, origConn })
+
+ h := (&PipeModule{}).broadcastDeploymentActionHandler("stop")
+ w := callHandler(t, h, http.MethodPost, "/pipeline/deployments/"+dep.ID+"/_stop",
+ httprouter.Params{{Key: "id", Value: dep.ID}})
+ out := util.MapStr{}
+ _ = util.FromJSONBytes(w.Body.Bytes(), &out)
+ if out["delivered"].(float64) != 0 {
+ t.Fatalf("offline instance must not count as delivered: %v", out)
+ }
+}
+
+func TestInstanceActionForwarding(t *testing.T) {
+ env := newDeployTestEnv(t)
+ fr := &fakeReverse{}
+ fr.install(t)
+
+ h := (&PipeModule{}).instanceActionHandler("drain")
+ w := callHandler(t, h, http.MethodPost, "/pipeline/instances/"+env.instA+"/_drain?timeout_seconds=5",
+ httprouter.Params{{Key: "id", Value: env.instA}})
+ if w.Code != http.StatusOK {
+ t.Fatalf("drain forward: %d %s", w.Code, w.Body.String())
+ }
+ calls := fr.recorded()
+ if len(calls) != 1 || calls[0].Path != "/pipeline/_drain?timeout_seconds=5" {
+ t.Fatalf("query string must ride along: %+v", calls)
+ }
+
+ // unknown instance → 404 before any reverse traffic
+ before := len(fr.recorded())
+ w = callHandler(t, h, http.MethodPost, "/pipeline/instances/ghost/_drain",
+ httprouter.Params{{Key: "id", Value: "ghost"}})
+ if w.Code != http.StatusNotFound || len(fr.recorded()) != before {
+ t.Fatalf("ghost instance must 404 without proxying: %d", w.Code)
+ }
+}
+
+func TestInstanceProxyAllowlist(t *testing.T) {
+ env := newDeployTestEnv(t)
+ fr := &fakeReverse{}
+ fr.install(t)
+ h := (&PipeModule{}).instanceProxyHandler
+
+ // off-scope path refused
+ w := callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/admin/",
+ httprouter.Params{{Key: "id", Value: env.instA}})
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("non-pipeline path must be refused: %d", w.Code)
+ }
+ // traversal refused
+ w = callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/pipeline/../../x",
+ httprouter.Params{{Key: "id", Value: env.instA}})
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("traversal must be refused: %d", w.Code)
+ }
+ // in-scope read passes through, status code preserved
+ fr.statusOf = map[string]int{env.instA: http.StatusConflict}
+ w = callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/pipeline/task/x/flight",
+ httprouter.Params{{Key: "id", Value: env.instA}})
+ if w.Code != http.StatusConflict {
+ t.Fatalf("instance status code must pass through verbatim: %d", w.Code)
+ }
+ calls := fr.recorded()
+ if len(calls) != 1 || calls[0].Method != http.MethodGet || calls[0].Path != "/pipeline/task/x/flight" {
+ t.Fatalf("proxy must forward the read as-is: %+v", calls)
+ }
+}
diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go
new file mode 100644
index 000000000..8f6698b85
--- /dev/null
+++ b/modules/pipeline/flight.go
@@ -0,0 +1,108 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "sync"
+ "sync/atomic"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/pipeline"
+)
+
+// flightRing caps the retained steps per pipeline.
+const flightRing = 1024
+
+// flightSample keeps 1/N of the OK steps; error steps are ALWAYS kept.
+const flightSample = 100
+
+// flightRecorder is the sampled execution observer: a bounded ring of
+// StepEvents per pipeline, errors always captured, OK steps sampled 1/N.
+// It installs globally at PipeModule setup — the empty-registry fast
+// path applies before installation, and the per-step cost after is one
+// counter increment plus the sampled append.
+type flightRecorder struct {
+ mu sync.Mutex
+ rings map[string][]pipeline.StepEvent
+ seq int64
+}
+
+func newFlightRecorder() *flightRecorder {
+ return &flightRecorder{rings: map[string][]pipeline.StepEvent{}}
+}
+
+func (f *flightRecorder) OnStep(e *pipeline.StepEvent) {
+ keep := e.Outcome != pipeline.OutcomeOK
+ if !keep {
+ n := atomic.AddInt64(&f.seq, 1)
+ if n%flightSample != 0 {
+ return
+ }
+ }
+ cp := *e
+ f.mu.Lock()
+ ring := append(f.rings[e.Pipeline], cp)
+ if len(ring) > flightRing {
+ ring = ring[len(ring)-flightRing:]
+ }
+ f.rings[e.Pipeline] = ring
+ f.mu.Unlock()
+}
+
+// flightQuery filters a pipeline's ring: by trace id, errors-only, newest
+// first, capped at limit.
+func flightQuery(pipelineName, traceID string, errorsOnly bool, limit int) []pipeline.StepEvent {
+ if limit <= 0 || limit > flightRing {
+ limit = 50
+ }
+ f := globalFlight.Load()
+ if f == nil {
+ return nil
+ }
+ rec := f.(*flightRecorder)
+ rec.mu.Lock()
+ ring := rec.rings[pipelineName]
+ out := make([]pipeline.StepEvent, 0, limit)
+ for i := len(ring) - 1; i >= 0 && len(out) < limit; i-- {
+ e := ring[i]
+ if traceID != "" && e.TraceID != traceID {
+ continue
+ }
+ if errorsOnly && e.Outcome == pipeline.OutcomeOK {
+ continue
+ }
+ out = append(out, e)
+ }
+ rec.mu.Unlock()
+ return out
+}
+
+var globalFlight atomic.Value
+
+// installFlight wires the recorder once (idempotent).
+func installFlight() {
+ if globalFlight.Load() != nil {
+ return
+ }
+ pipeline.RegisterObserver(newFlightRecorder())
+ globalFlight.Store(newFlightRecorder())
+}
+
+// recordPipelineEvent files a control-plane lifecycle event (singleton
+// lock takeover...) into the flight ring — the alert surface operators
+// watch. No-op when flight is not installed; non-ok outcomes are always
+// retained by the recorder.
+func recordPipelineEvent(pipelineName string, outcome pipeline.Outcome, msg string) {
+ f := globalFlight.Load()
+ if f == nil {
+ return
+ }
+ f.(*flightRecorder).OnStep(&pipeline.StepEvent{
+ InstanceID: global.Env().SystemConfig.NodeConfig.ID,
+ Pipeline: pipelineName,
+ Outcome: outcome,
+ Err: msg,
+ })
+}
diff --git a/modules/pipeline/flight_test.go b/modules/pipeline/flight_test.go
new file mode 100644
index 000000000..09da912c7
--- /dev/null
+++ b/modules/pipeline/flight_test.go
@@ -0,0 +1,49 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/pipeline"
+)
+
+func TestFlight_RecordsErrorsAlwaysAndSamplesOK(t *testing.T) {
+ installFlight()
+ f := globalFlight.Load().(*flightRecorder)
+
+ // errors always land
+ f.OnStep(&pipeline.StepEvent{Pipeline: "fp", Outcome: pipeline.OutcomeError, Err: "boom", TraceID: "tr-1"})
+ // OK steps: 2x flightSample land at most 2
+ for i := 0; i < 2*flightSample; i++ {
+ f.OnStep(&pipeline.StepEvent{Pipeline: "fp", Outcome: pipeline.OutcomeOK})
+ }
+ errs := flightQuery("fp", "", true, 10)
+ if len(errs) != 1 || errs[0].Err != "boom" {
+ t.Fatalf("error query: %+v", errs)
+ }
+ all := flightQuery("fp", "", false, flightRing)
+ if len(all) > 2+3 { // 2 sampled + error + slack
+ t.Fatalf("OK sampling not applied: %d events", len(all))
+ }
+ // trace filter
+ if got := flightQuery("fp", "tr-1", false, 10); len(got) != 1 || got[0].TraceID != "tr-1" {
+ t.Fatalf("trace query: %+v", got)
+ }
+}
+
+func TestFlight_RingBounded(t *testing.T) {
+ installFlight()
+ f := globalFlight.Load().(*flightRecorder)
+ for i := 0; i < flightRing*2; i++ {
+ f.OnStep(&pipeline.StepEvent{Pipeline: "ring", Outcome: pipeline.OutcomeError})
+ }
+ f.mu.Lock()
+ n := len(f.rings["ring"])
+ f.mu.Unlock()
+ if n != flightRing {
+ t.Fatalf("ring = %d, want %d", n, flightRing)
+ }
+}
diff --git a/modules/pipeline/for_each.go b/modules/pipeline/for_each.go
index b972b890f..0f194b16a 100644
--- a/modules/pipeline/for_each.go
+++ b/modules/pipeline/for_each.go
@@ -28,6 +28,8 @@ package pipeline
import (
"fmt"
+ "sync"
+ "sync/atomic"
log "github.com/cihub/seelog"
"infini.sh/framework/core/config"
@@ -76,11 +78,26 @@ import (
// regardless of their position in the sub-chain; plain Processors keep
// the per-record semantics unchanged.
type ForEachConfig struct {
- MessageField string `config:"message_field"`
- Codec string `config:"codec"`
- OnFailure string `config:"on_failure"`
- FailureTag string `config:"failure_tag"`
- Processors []*config.Config `config:"processor"`
+ MessageField string `config:"message_field"`
+ Codec string `config:"codec"`
+ // Parallel runs each record segment with N workers (default 1 =
+ // serial). Processors must be safe for concurrent Process calls on
+ // DISTINCT records (they mutate the record, never shared state) —
+ // every in-repo record processor satisfies this.
+ Parallel int `config:"parallel"`
+ OnFailure string `config:"on_failure"`
+ FailureTag string `config:"failure_tag"`
+ Processors []*config.Config `config:"processor"`
+}
+
+// execSegment is one stage of the stage-wise (stage) executor: a run of
+// consecutive per-record processors, or a single batch processor. Segments
+// execute in WRITTEN order; a batch processor therefore sees the edits of
+// the record processors declared before it (and vice versa).
+type execSegment struct {
+ batch bool
+ proc pipeline.Processor // batch segments: the single processor
+ chain *pipeline.Processors // record segments: per-record sub-chain
}
type ForEachProcessor struct {
@@ -89,6 +106,21 @@ type ForEachProcessor struct {
sub *pipeline.Processors
}
+// buildSegments splits a compiled sub-chain into alternating segments.
+func buildSegments(list []pipeline.Processor) []execSegment {
+ var segs []execSegment
+ for _, p := range list {
+ if _, isBatch := p.(pipeline.BatchProcessor); isBatch {
+ segs = append(segs, execSegment{batch: true, proc: p})
+ } else if n := len(segs); n > 0 && !segs[n-1].batch {
+ segs[n-1].chain.List = append(segs[n-1].chain.List, p)
+ } else {
+ segs = append(segs, execSegment{chain: &pipeline.Processors{List: []pipeline.Processor{p}}})
+ }
+ }
+ return segs
+}
+
func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) {
cfg := ForEachConfig{MessageField: "messages"}
if err := c.Unpack(&cfg); err != nil {
@@ -97,14 +129,17 @@ func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) {
if len(cfg.Processors) == 0 {
return nil, fmt.Errorf("for_each processor requires a non-empty processor sub-chain")
}
- sub, err := pipeline.NewPipeline(cfg.Processors)
- if err != nil {
- return nil, fmt.Errorf("failed to compile for_each sub-chain: %s", err)
- }
codec := GetRecordCodec(cfg.Codec)
if cfg.Codec != "" && codec.Name() != cfg.Codec {
return nil, fmt.Errorf("unknown for_each codec %q (registered: %v)", cfg.Codec, recordCodecNames())
}
+ sub, err := pipeline.NewPipelineWithScope(cfg.Processors, pipeline.ScopeInfo{
+ Kind: pipeline.ScopeRecord,
+ Namespace: codec.Namespace(),
+ }, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to compile for_each sub-chain: %s", err)
+ }
switch cfg.OnFailure {
case "", "ignore", "tag", "fail":
default:
@@ -113,29 +148,44 @@ func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) {
if cfg.FailureTag == "" {
cfg.FailureTag = "_processing_failed"
}
+ if cfg.Parallel <= 0 {
+ cfg.Parallel = 1
+ }
return &ForEachProcessor{cfg: cfg, codec: codec, sub: sub}, nil
}
func recordCodecNames() []string {
- codecsMu.RLock()
- defer codecsMu.RUnlock()
- names := make([]string, 0, len(codecs))
- for n := range codecs {
- names = append(names, n)
+ infos := ListRecordCodecs()
+ names := make([]string, 0, len(infos))
+ for _, i := range infos {
+ names = append(names, i.Name)
}
return names
}
func (p *ForEachProcessor) Name() string { return "for_each" }
-func (p *ForEachProcessor) Process(c *pipeline.Context) error {
- v := c.Get(param.ParaKey(p.cfg.MessageField))
- msgs, ok := v.([]queue.Message)
- if !ok {
- log.Debugf("for_each: no batch found under context key [%s], skip", p.cfg.MessageField)
+// ApplyScope recompiles the sub-chain under the enclosing compile scope:
+// the registry constructor takes no scope, so the initial compile is
+// warn-only — a strict parent demands the strict view of the sub-chain
+// (pipeline.ScopeAwareProcessor; idempotent by replacement).
+func (p *ForEachProcessor) ApplyScope(scope pipeline.ScopeInfo) error {
+ if !scope.Strict {
return nil
}
+ sub, err := pipeline.NewPipelineWithScope(p.cfg.Processors, pipeline.ScopeInfo{
+ Kind: pipeline.ScopeRecord,
+ Namespace: p.codec.Namespace(),
+ Strict: true,
+ }, true)
+ if err != nil {
+ return fmt.Errorf("failed to compile for_each sub-chain: %s", err)
+ }
+ p.sub = sub
+ return nil
+}
+func (p *ForEachProcessor) Process(c *pipeline.Context) error {
// The record convention is only valid inside the sub-chain; save and
// restore the previous values so the batch's last record (and its
// failure tags) does not leak to processors downstream of for_each,
@@ -147,6 +197,24 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error {
c.Set(pipeline.FailureTagsKey, previousTags)
}()
+ v := c.Get(param.ParaKey(p.cfg.MessageField))
+ msgs, ok := v.([]queue.Message)
+ if !ok {
+ // Dry-run replay: the harness binds one decoded record instead of
+ // a message batch. Adopt it as the single input record — the same
+ // zero-decode typed entry lane_consumer uses at runtime — so the
+ // sub-chain really executes and step diffs show its effect. There
+ // is no message to encode back into; the harness owns the record
+ // (runOnRecords skips release and encode for adopted records).
+ if pipeline.InDryRun(c) {
+ if rec, has := pipeline.CurrentRecord(c); has {
+ return p.runOnRecords(c, []*event.Event{rec}, nil, nil)
+ }
+ }
+ log.Debugf("for_each: no batch found under context key [%s], skip", p.cfg.MessageField)
+ return nil
+ }
+
// Decode pass: build the record slice; undecodable payloads keep their
// original bytes and are passed through untouched (payload transparency
// for mixed batches).
@@ -164,112 +232,162 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error {
records = append(records, rec)
decodedIdx = append(decodedIdx, i)
}
+ return p.runOnRecords(c, records, msgs, decodedIdx)
+}
+
+// runOnRecords executes the sub-chain stage-wise over records. With msgs
+// non-nil (normal runtime) originals encode back into their message slot,
+// clones append messages, and pooled records are released. With msgs nil
+// (dry-run adoption) the caller owns the records: serial execution only
+// (deterministic diffs), no encode pass, no release.
+func (p *ForEachProcessor) runOnRecords(c *pipeline.Context, records []*event.Event, msgs []queue.Message, decodedIdx []int) error {
+ segments := buildSegments(p.sub.List)
- // Batch pass: sub-processors implementing BatchProcessor run once per
- // batch; the rest run per record below. Drop marks from the batch pass
- // are honored through the same per-record drop handling.
- for _, proc := range p.sub.List {
+ // Stage-wise execution: segments run in WRITTEN order (record runs,
+ // then a batch call, then more record runs...). A batch processor sees
+ // the edits of record processors declared before it; drops propagate
+ // to every later segment; clones join the active set for later
+ // segments.
+ active := make([]forEachActive, 0, len(records))
+ for j, rec := range records {
+ idx := -1
+ if msgs != nil {
+ idx = decodedIdx[j]
+ }
+ active = append(active, forEachActive{rec: rec, msgIdx: idx})
+ }
+ // offsetOf is for logging only; adopted records carry no message.
+ offsetOf := func(ar *forEachActive) interface{} {
+ if msgs == nil || ar.msgIdx < 0 {
+ return -1
+ }
+ return msgs[ar.msgIdx].Offset
+ }
+ // release pooled records on every exit path — by now each record has
+ // been encoded (or dropped) and clones have joined active; non-pooled
+ // records (codec- or clone-allocated) make ReleaseEvent a no-op.
+ // Adopted dry-run records belong to the harness — never released here.
+ if msgs != nil {
+ defer func() {
+ for i := range active {
+ event.ReleaseEvent(active[i].rec)
+ }
+ }()
+ }
+
+ for _, seg := range segments {
if c.IsCanceled() || !c.ShouldContinue() {
return nil
}
- if bp, ok := proc.(pipeline.BatchProcessor); ok {
- if err := bp.ProcessBatch(c, records); err != nil {
- if err2 := p.handleFailure(c, err); err2 != nil {
- return err2
+ if seg.batch {
+ live := make([]*event.Event, 0, len(active))
+ for i := range active {
+ if !pipeline.IsDropped(active[i].rec) {
+ live = append(live, active[i].rec)
}
}
+ if len(live) == 0 {
+ continue
+ }
+ bp := seg.proc.(pipeline.BatchProcessor)
+ if err := bp.ProcessBatch(c, live); err != nil {
+ if p.cfg.OnFailure == "fail" {
+ return p.handleFailure(c, err)
+ }
+ if p.cfg.OnFailure == "tag" {
+ // a batch-segment failure tags every live record of
+ // the batch (there is no per-record scope here)
+ for _, rec := range live {
+ mergeFailureTags(rec, []string{p.cfg.FailureTag})
+ }
+ }
+ log.Warnf("for_each: batch segment [%s] error (%s): %v", seg.proc.Name(), p.cfg.OnFailure, err)
+ }
+ continue
}
- }
- // Record pass: plain processors, per record. Batch-aware processors
- // already ran in the batch pass and are skipped here.
- perRecord := make([]pipeline.Processor, 0, len(p.sub.List))
- for _, proc := range p.sub.List {
- if _, isBatch := proc.(pipeline.BatchProcessor); !isBatch {
- perRecord = append(perRecord, proc)
- }
- }
- // ignore/fail keep the historical whole-chain execution per record
- // (one Processors pass: panic recovery, flow tracing, and an error
- // skips the rest of the chain for that record); tag needs the
- // per-processor loop to continue the chain after a tagged failure.
- plain := &pipeline.Processors{List: perRecord, SkipCatchError: p.sub.SkipCatchError}
- for j, rec := range records {
- if c.IsCanceled() || !c.ShouldContinue() {
- break
- }
- if pipeline.IsDropped(rec) {
- // dropped by the batch pass
- i := decodedIdx[j]
- msgs[i].Data = nil
- msgs[i].Size = 0
+ // record segment: per active record, run the segment's sub-chain
+ // (dry-run adoption stays serial for deterministic diffs)
+ if p.cfg.Parallel > 1 && msgs != nil {
+ runRecordSegmentParallel(c, seg.chain, active, p.cfg, func(cl *event.Event, idx int) {
+ active = append(active, forEachActive{rec: cl, msgIdx: idx, isClone: true})
+ })
continue
}
- c.Set(pipeline.RecordContextKey, rec)
- clones := &[]*event.Event{}
- c.Set(pipeline.CloneContextKey, clones)
- if p.cfg.OnFailure == "tag" {
- var tags []string
- c.Set(pipeline.FailureTagsKey, &tags)
- tagged := false
- for _, proc := range perRecord {
- if err := proc.Process(c); err != nil {
- if !tagged {
- // one failure_tag per record is enough to route on
- pipeline.AppendFailureTag(c, p.cfg.FailureTag)
- tagged = true
+ for i := range active {
+ if c.IsCanceled() || !c.ShouldContinue() {
+ break
+ }
+ ar := &active[i]
+ if pipeline.IsDropped(ar.rec) {
+ continue
+ }
+ c.Set(pipeline.RecordContextKey, ar.rec)
+ clones := &[]*event.Event{}
+ c.Set(pipeline.CloneContextKey, clones)
+ if p.cfg.OnFailure == "tag" {
+ var tags []string
+ c.Set(pipeline.FailureTagsKey, &tags)
+ tagged := false
+ for _, proc := range seg.chain.List {
+ if err := proc.Process(c); err != nil {
+ if !tagged {
+ pipeline.AppendFailureTag(c, p.cfg.FailureTag)
+ tagged = true
+ }
+ log.Warnf("for_each: sub-chain error tagged %q on record at offset %v: %v",
+ p.cfg.FailureTag, offsetOf(ar), err)
}
- log.Warnf("for_each: sub-chain error tagged %q on record at offset %v: %v",
- p.cfg.FailureTag, msgs[decodedIdx[j]].Offset, err)
}
- }
- // persist accumulated tags into the record, so they survive
- // re-encoding and downstream stages can route on them
- mergeFailureTags(rec, tags)
- } else {
- if err := plain.Process(c); err != nil {
- if err2 := p.handleFailure(c, err); err2 != nil {
- return err2
+ mergeFailureTags(ar.rec, tags)
+ } else {
+ if err := seg.chain.Process(c); err != nil {
+ if err2 := p.handleFailure(c, err); err2 != nil {
+ return err2
+ }
}
}
+ // clones join the active set and flow through later segments
+ for _, cl := range pipeline.TakeClones(c) {
+ active = append(active, forEachActive{rec: cl, msgIdx: ar.msgIdx, isClone: true})
+ }
}
+ }
- // drop_event marks: clear the payload so downstream stages
- // skip this record entirely.
- if pipeline.IsDropped(rec) {
- i := decodedIdx[j]
- msgs[i].Data = nil
- msgs[i].Size = 0
+ // Adopted dry-run records have no message to encode back into — the
+ // harness snapshots the record directly.
+ if msgs == nil {
+ return nil
+ }
+
+ // Encode pass: originals write their message slot, clones append new
+ // messages sharing the parent's offset semantics.
+ for i := range active {
+ ar := active[i]
+ if pipeline.IsDropped(ar.rec) {
+ if !ar.isClone {
+ msgs[ar.msgIdx].Data = nil
+ msgs[ar.msgIdx].Size = 0
+ }
continue
}
-
- encoded, err := p.codec.Encode(rec)
+ encoded, err := p.codec.Encode(ar.rec)
if err != nil {
- log.Warnf("for_each: failed to encode record at offset %v: %v", msgs[decodedIdx[j]].Offset, err)
+ log.Warnf("for_each: failed to encode record at offset %v: %v", msgs[ar.msgIdx].Offset, err)
continue
}
- i := decodedIdx[j]
- msgs[i].Data = encoded
- msgs[i].Size = len(encoded)
-
- // Clones (clone processor): materialize each as an extra batch
- // member sharing the original's offset semantics.
- for _, cl := range pipeline.TakeClones(c) {
- if pipeline.IsDropped(cl) {
- continue
- }
- cloned, cerr := p.codec.Encode(cl)
- if cerr != nil {
- log.Warnf("for_each: failed to encode clone at offset %v: %v", msgs[i].Offset, cerr)
- continue
- }
+ if ar.isClone {
msgs = append(msgs, queue.Message{
- Offset: msgs[i].Offset,
- Data: cloned,
- Size: len(cloned),
+ Offset: msgs[ar.msgIdx].Offset,
+ Data: encoded,
+ Size: len(encoded),
})
+ } else {
+ msgs[ar.msgIdx].Data = encoded
+ msgs[ar.msgIdx].Size = len(encoded)
}
}
+
// publish the (possibly extended) batch back to the context
c.Set(param.ParaKey(p.cfg.MessageField), msgs)
return nil
@@ -314,3 +432,74 @@ func mergeFailureTags(rec *event.Event, tags []string) {
rec.Fields["tags"] = existing
}
}
+
+// forEachActive is one in-flight record of a batch (record + origin
+// message slot + clone marker).
+type forEachActive struct {
+ rec *event.Event
+ msgIdx int
+ isClone bool
+}
+
+// runRecordSegmentParallel runs one record segment with N workers. Each
+// worker derives its own context (per-record state like RecordContextKey
+// is goroutine-local; cancellation still propagates from the parent), so
+// processors only ever touch their own record — the concurrency contract
+// for parallel record processing.
+func runRecordSegmentParallel(c *pipeline.Context, chain *pipeline.Processors, active []forEachActive, cfg ForEachConfig, addClone func(cl *event.Event, idx int)) {
+ if len(active) == 0 {
+ return
+ }
+ workers := cfg.Parallel
+ if workers > len(active) {
+ workers = len(active)
+ }
+ var next int64
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ // per-worker derived context: own record/clone/tag state
+ wc := &pipeline.Context{ParentContext: c, Context: c.Context, Config: c.Config}
+ for {
+ if c.IsCanceled() || !c.ShouldContinue() {
+ return
+ }
+ i := int(atomic.AddInt64(&next, 1) - 1)
+ if i >= len(active) {
+ return
+ }
+ ar := active[i]
+ if pipeline.IsDropped(ar.rec) {
+ continue
+ }
+ wc.Set(pipeline.RecordContextKey, ar.rec)
+ clones := &[]*event.Event{}
+ wc.Set(pipeline.CloneContextKey, clones)
+ if cfg.OnFailure == "tag" {
+ var tags []string
+ wc.Set(pipeline.FailureTagsKey, &tags)
+ for _, proc := range chain.List {
+ if err := proc.Process(wc); err != nil {
+ pipeline.AppendFailureTag(wc, cfg.FailureTag)
+ log.Warnf("for_each: sub-chain error tagged %q (parallel): %v", cfg.FailureTag, err)
+ }
+ }
+ mergeFailureTags(ar.rec, tags)
+ } else {
+ if err := chain.Process(wc); err != nil {
+ log.Warnf("for_each: record error (parallel, on_failure=%s): %v", cfg.OnFailure, err)
+ }
+ }
+ for _, cl := range pipeline.TakeClones(wc) {
+ mu.Lock()
+ addClone(cl, ar.msgIdx)
+ mu.Unlock()
+ }
+ }
+ }()
+ }
+ wg.Wait()
+}
diff --git a/modules/pipeline/for_each_bench_test.go b/modules/pipeline/for_each_bench_test.go
new file mode 100644
index 000000000..99d4f0a02
--- /dev/null
+++ b/modules/pipeline/for_each_bench_test.go
@@ -0,0 +1,192 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package pipeline
+
+import (
+ "encoding/json"
+ "hash/fnv"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/param"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// P4 benchmark suite (#75): record-mode data movement through for_each.
+//
+// BenchmarkForEach_1MB_K4_* — 1MB document tree, K=4 record processors:
+// pooled vs fresh allocation (the pool gate)
+// BenchmarkForEach_1KB_Parall — 1KB events, parallel 1 vs 4 (the ≥3×
+// throughput gate on CPU-shaped chains)
+//
+// The codec decodes the whole JSON tree into the record Fields — the
+// document-codec shape — so the numbers track the real data-movement cost,
+// not framework scaffolding.
+
+// benchTreeCodec decodes a JSON envelope into the record fields tree.
+type benchTreeCodec struct{ pooled bool }
+
+func (benchTreeCodec) Name() string { return "bench_tree" }
+func (benchTreeCodec) Namespace() string { return "" }
+func (c benchTreeCodec) Decode(data []byte) (*event.Event, error) {
+ var tree util.MapStr
+ if err := json.Unmarshal(data, &tree); err != nil {
+ return nil, err
+ }
+ var e *event.Event
+ if c.pooled {
+ e = event.AcquireEvent()
+ } else {
+ e = &event.Event{}
+ }
+ e.Fields = tree
+ return e, nil
+}
+func (benchTreeCodec) Encode(rec *event.Event) ([]byte, error) {
+ return json.Marshal(rec.Fields)
+}
+
+// benchHasher burns CPU proportional to the content — the CPU-shaped
+// chain the parallel gate measures (a no-op chain scales nothing).
+type benchHasher struct{}
+
+func (benchHasher) Name() string { return "test_bench_hasher" }
+func (benchHasher) Process(c *pipeline.Context) error {
+ rec, _ := pipeline.CurrentRecord(c)
+ if rec == nil {
+ return nil
+ }
+ doc, _ := rec.Fields["document"].(map[string]interface{})
+ if doc == nil {
+ return nil
+ }
+ content, _ := doc["content"].(string)
+ h := fnv.New64a()
+ for i := 0; i < 256; i++ { // burn enough CPU that the chain, not the codec, dominates
+ h.Reset()
+ h.Write([]byte(content))
+ }
+ doc["hash"] = h.Sum64()
+ return nil
+}
+
+// benchPayload builds a document-shaped payload of roughly size bytes.
+func benchPayload(size int) []byte {
+ content := make([]byte, 0, size)
+ unit := []byte("The quick brown fox jumps over the lazy dog. ")
+ for len(content) < size {
+ content = append(content, unit...)
+ }
+ body := map[string]interface{}{
+ "document": map[string]interface{}{
+ "id": "bench-doc",
+ "title": "Benchmark Document",
+ "content": string(content[:size]),
+ "source": map[string]interface{}{"id": "bench-src", "type": "test"},
+ },
+ }
+ b, _ := json.Marshal(body)
+ return b
+}
+
+func init() {
+ pipeline.RegisterProcessorPlugin("test_bench_hasher", func(*config.Config) (pipeline.Processor, error) { return benchHasher{}, nil })
+}
+
+func buildForBench(b *testing.B, extra map[string]interface{}) *ForEachProcessor {
+ b.Helper()
+ cfgMap := util.MapStr{
+ "codec": "bench_tree",
+ "processor": []interface{}{
+ util.MapStr{benchProc: util.MapStr{}},
+ util.MapStr{benchProc: util.MapStr{}},
+ util.MapStr{benchProc: util.MapStr{}},
+ util.MapStr{benchProc: util.MapStr{}},
+ },
+ }
+ for k, v := range extra {
+ cfgMap[k] = v
+ }
+ c, err := config.NewConfigFrom(cfgMap)
+ if err != nil {
+ b.Fatal(err)
+ }
+ p, err := NewForEachProcessor(c)
+ if err != nil {
+ b.Fatalf("NewForEachProcessor: %v", err)
+ }
+ return p.(*ForEachProcessor)
+}
+
+func runForEachBench(b *testing.B, p *ForEachProcessor, payload []byte, parallel int) {
+ b.Helper()
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ msgs := []queue.Message{{Data: payload, Size: len(payload), Offset: queue.AcquireOffset(0, int64(i))}}
+ ctx := &pipeline.Context{Context: b.Context()}
+ ctx.Set(param.ParaKey("messages"), msgs)
+ if err := p.Process(ctx); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func BenchmarkForEach_1MB_K4_Pooled(b *testing.B) {
+ if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil {
+ b.Fatal(err)
+ }
+ runForEachBench(b, buildForBench(b, nil), benchPayload(1<<20), 1)
+}
+
+func BenchmarkForEach_1MB_K4_Fresh(b *testing.B) {
+ if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: false}); err != nil {
+ b.Fatal(err)
+ }
+ runForEachBench(b, buildForBench(b, nil), benchPayload(1<<20), 1)
+}
+
+// benchProc selects the K=4 chain shape: the data-movement benches use the
+// cheap mutator, the parallel gates use the CPU burner.
+var benchProc = "test_upper"
+
+// runForEachBatchBench drives a 32-record batch — parallel workers need
+// records to steal; a single-record batch cannot scale.
+func runForEachBatchBench(b *testing.B, p *ForEachProcessor, payload []byte) {
+ b.Helper()
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ msgs := make([]queue.Message, 0, 32)
+ for j := 0; j < 32; j++ {
+ msgs = append(msgs, queue.Message{Data: payload, Size: len(payload), Offset: queue.AcquireOffset(0, int64(i*32+j))})
+ }
+ ctx := &pipeline.Context{Context: b.Context()}
+ ctx.Set(param.ParaKey("messages"), msgs)
+ if err := p.Process(ctx); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func BenchmarkForEach_1KB_CPUChain_Parallel1(b *testing.B) {
+ if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil {
+ b.Fatal(err)
+ }
+ benchProc = "test_bench_hasher"
+ defer func() { benchProc = "test_upper" }()
+ runForEachBatchBench(b, buildForBench(b, map[string]interface{}{"parallel": 1}), benchPayload(1<<10))
+}
+
+func BenchmarkForEach_1KB_CPUChain_Parallel4(b *testing.B) {
+ if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil {
+ b.Fatal(err)
+ }
+ benchProc = "test_bench_hasher"
+ defer func() { benchProc = "test_upper" }()
+ runForEachBatchBench(b, buildForBench(b, map[string]interface{}{"parallel": 4}), benchPayload(1<<10))
+}
diff --git a/modules/pipeline/for_each_codec.go b/modules/pipeline/for_each_codec.go
index af8ceb075..cf2b25656 100644
--- a/modules/pipeline/for_each_codec.go
+++ b/modules/pipeline/for_each_codec.go
@@ -5,62 +5,23 @@
package pipeline
import (
- "fmt"
- "sync"
-
- "infini.sh/framework/core/event"
- "infini.sh/framework/core/otel"
+ "infini.sh/framework/core/pipeline"
)
-// RecordCodec converts between a queue message payload and the *event.Event
-// the per-record processor convention operates on. for_each is generic over
-// payloads: the default "otel" codec decodes the otel envelope (byte-
-// compatible with the agent LogEvent JSON); other payload formats register
-// their own codec and select it via the codec config field.
-type RecordCodec interface {
- Name() string
- Decode(data []byte) (*event.Event, error)
- Encode(rec *event.Event) ([]byte, error)
-}
+// The codec registry lives in core/pipeline (engine level) so lightweight
+// consumers — domain codec modules with a strict dependency surface — can
+// register without importing the modules tree. This file re-exports the
+// API for existing importers of this package.
-var (
- codecsMu sync.RWMutex
- codecs = map[string]RecordCodec{}
+type (
+ RecordCodec = pipeline.RecordCodec
+ CodecInfo = pipeline.CodecInfo
)
-// RegisterRecordCodec adds a payload codec under its name. Intended for
-// package init(); a duplicate name panics.
-func RegisterRecordCodec(c RecordCodec) {
- codecsMu.Lock()
- defer codecsMu.Unlock()
- if _, exists := codecs[c.Name()]; exists {
- panic(fmt.Sprintf("record codec with same name already exists: %v", c.Name()))
- }
- codecs[c.Name()] = c
-}
-
-// GetRecordCodec returns the named codec, falling back to the default.
-func GetRecordCodec(name string) RecordCodec {
- codecsMu.RLock()
- defer codecsMu.RUnlock()
- if c, ok := codecs[name]; ok {
- return c
- }
- return defaultCodec
-}
-
-// otelCodec is the default: the otel envelope JSON shared with the agent
-// LogEvent format and the OTLP transport boundary.
-type otelCodec struct{}
-
-func (otelCodec) Name() string { return "otel" }
-func (otelCodec) Decode(data []byte) (*event.Event, error) {
- return otel.DecodeEnvelope(data)
-}
-func (otelCodec) Encode(rec *event.Event) ([]byte, error) {
- return otel.EncodeEnvelope(rec)
-}
-
-var defaultCodec RecordCodec = otelCodec{}
-
-func init() { RegisterRecordCodec(otelCodec{}) }
+var (
+ RegisterRecordCodec = pipeline.RegisterRecordCodec
+ MustRegisterRecordCodec = pipeline.MustRegisterRecordCodec
+ RegisterRecordCodecOrReplace = pipeline.RegisterRecordCodecOrReplace
+ GetRecordCodec = pipeline.GetRecordCodec
+ ListRecordCodecs = pipeline.ListRecordCodecs
+)
diff --git a/modules/pipeline/for_each_dryrun_test.go b/modules/pipeline/for_each_dryrun_test.go
new file mode 100644
index 000000000..cccb4eb39
--- /dev/null
+++ b/modules/pipeline/for_each_dryrun_test.go
@@ -0,0 +1,59 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+/* for_each under a dry-run replay: the dryrun harness binds one decoded
+ * record and no message batch. The host must adopt the bound record (the
+ * zero-decode typed entry) and actually run the sub-chain on it — before
+ * this path existed, for_each silently no-opped in every dry-run, so
+ * migrated legacy chains replayed as "no effect". */
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/pipeline"
+)
+
+func TestForEach_DryRunAdoptsBoundRecord(t *testing.T) {
+ p := buildForEach(t, nil)
+
+ rec := event.AcquireEvent()
+ rec.Fields = map[string]interface{}{"message": "hello"}
+ ctx := &pipeline.Context{Context: t.Context()}
+ ctx.Set(pipeline.RecordContextKey, rec)
+ sink := &[]pipeline.DryRunWrite{}
+ pipeline.EnableDryRun(ctx, sink, true)
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ if got := rec.Fields["message"]; got != "PROCESSED:hello" {
+ t.Fatalf("sub-chain did not run on the adopted record: message=%v", got)
+ }
+ // the harness's record must still be the bound record afterwards
+ bound, ok := pipeline.CurrentRecord(ctx)
+ if !ok || bound != rec {
+ t.Fatal("bound record not restored after dry-run adoption")
+ }
+}
+
+// outside dry-run the historical behavior stands: no batch, no-op
+func TestForEach_NoBatchOutsideDryRunSkips(t *testing.T) {
+ p := buildForEach(t, nil)
+
+ rec := event.AcquireEvent()
+ rec.Fields = map[string]interface{}{"message": "hello"}
+ ctx := &pipeline.Context{Context: t.Context()}
+ ctx.Set(pipeline.RecordContextKey, rec)
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if got := rec.Fields["message"]; got != "hello" {
+ t.Fatalf("non-dry-run run must not touch the bound record: message=%v", got)
+ }
+}
diff --git a/modules/pipeline/for_each_pool_test.go b/modules/pipeline/for_each_pool_test.go
new file mode 100644
index 000000000..866368488
--- /dev/null
+++ b/modules/pipeline/for_each_pool_test.go
@@ -0,0 +1,109 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infini.ltd */
+
+package pipeline
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/param"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// pooledRawCodec is rawCodec with the record coming from the event pool —
+// the production shape of coco's document/attachment codecs.
+type pooledRawCodec struct{}
+
+func (pooledRawCodec) Name() string { return "raw_pooled" }
+func (pooledRawCodec) Namespace() string { return "" }
+func (pooledRawCodec) Decode(data []byte) (*event.Event, error) {
+ e := event.AcquireEvent()
+ e.Fields = util.MapStr{"message": string(data)}
+ return e, nil
+}
+func (pooledRawCodec) Encode(rec *event.Event) ([]byte, error) {
+ msg, _ := rec.Fields["message"].(string)
+ return []byte(msg), nil
+}
+
+// dropper marks every record dropped so the wiring test covers the drop
+// path — dropped records must be released too.
+type dropper struct{}
+
+func (dropper) Name() string { return "test_dropper" }
+func (dropper) Process(c *pipeline.Context) error {
+ if rec, ok := pipeline.CurrentRecord(c); ok {
+ pipeline.MarkDropped(rec)
+ }
+ return nil
+}
+
+func init() {
+ pipeline.RegisterProcessorPlugin("test_dropper", func(*config.Config) (pipeline.Processor, error) { return dropper{}, nil })
+}
+
+// TestForEach_PooledRecordsReleased: every pooled record must be back in
+// the pool by the time Process returns — originals, transformed records
+// and dropped records alike (leak detection gate for the pool wiring).
+func TestForEach_PooledRecordsReleased(t *testing.T) {
+ if err := RegisterRecordCodecOrReplace(pooledRawCodec{}); err != nil {
+ t.Fatal(err)
+ }
+ p := buildForEach(t, map[string]interface{}{"codec": "raw_pooled"})
+
+ _, r0 := event.PoolStats()
+ for run := 0; run < 3; run++ {
+ msgs := []queue.Message{
+ {Data: []byte("keep-1"), Size: 6, Offset: queue.AcquireOffset(0, 1)},
+ {Data: []byte("keep-2"), Size: 6, Offset: queue.AcquireOffset(0, 2)},
+ }
+ ctx := &pipeline.Context{Context: t.Context()}
+ ctx.Set(param.ParaKey("messages"), msgs)
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if string(msgs[0].Data) != "PROCESSED:keep-1" {
+ t.Fatalf("record not transformed: %s", msgs[0].Data)
+ }
+ }
+ _, r1 := event.PoolStats()
+ if got := r1 - r0; got != 6 {
+ t.Fatalf("all 6 pooled records must be released, got %d", got)
+ }
+}
+
+func TestForEach_PooledDroppedRecordsReleased(t *testing.T) {
+ if err := RegisterRecordCodecOrReplace(pooledRawCodec{}); err != nil {
+ t.Fatal(err)
+ }
+ c, err := config.NewConfigFrom(util.MapStr{
+ "codec": "raw_pooled",
+ "processor": []interface{}{util.MapStr{"test_dropper": util.MapStr{}}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ p, err := NewForEachProcessor(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, r0 := event.PoolStats()
+ msgs := []queue.Message{{Data: []byte("gone"), Size: 4, Offset: queue.AcquireOffset(0, 1)}}
+ ctx := &pipeline.Context{Context: t.Context()}
+ ctx.Set(param.ParaKey("messages"), msgs)
+ if err := p.(*ForEachProcessor).Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if msgs[0].Data != nil || msgs[0].Size != 0 {
+ t.Fatalf("dropped record must leave an empty slot: %q", msgs[0].Data)
+ }
+ _, r1 := event.PoolStats()
+ if got := r1 - r0; got != 1 {
+ t.Fatalf("dropped pooled record must be released, got %d", got)
+ }
+}
diff --git a/modules/pipeline/for_each_stage_test.go b/modules/pipeline/for_each_stage_test.go
new file mode 100644
index 000000000..30189804f
--- /dev/null
+++ b/modules/pipeline/for_each_stage_test.go
@@ -0,0 +1,226 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/param"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// ---- stage-wise test doubles ------------------------------------------------
+
+// orderRecorder logs execution order across processors.
+type orderRecorder struct {
+ name string
+ order *[]string
+}
+
+func (p orderRecorder) Name() string { return p.name }
+func (p orderRecorder) Process(ctx *pipeline.Context) error {
+ *p.order = append(*p.order, p.name)
+ return nil
+}
+
+// stageRecorder implements BatchProcessor: records the batch size and
+// whether the field written by a preceding record processor is visible.
+type stageBatchProc struct {
+ name string
+ order *[]string
+ batchLen *[]int
+ sawUpper *[]string
+}
+
+func (p *stageBatchProc) Name() string { return p.name }
+func (p *stageBatchProc) Process(ctx *pipeline.Context) error {
+ *p.order = append(*p.order, p.name)
+ rec, _ := pipeline.CurrentRecord(ctx)
+ if rec != nil {
+ v, _ := rec.Fields.GetValue("upper")
+ *p.sawUpper = append(*p.sawUpper, util.ToString(v))
+ }
+ return nil
+}
+func (p *stageBatchProc) ProcessBatch(ctx *pipeline.Context, records []*event.Event) error {
+ *p.order = append(*p.order, p.name)
+ for _, rec := range records {
+ v, _ := rec.Fields.GetValue("upper")
+ *p.sawUpper = append(*p.sawUpper, util.ToString(v))
+ }
+ *p.batchLen = append(*p.batchLen, len(records))
+ return nil
+}
+
+// upperProc upper-cases the message field (a record-processor edit the
+// later batch segment must observe).
+type upperProc struct {
+ name string
+ order *[]string
+}
+
+func (p upperProc) Name() string { return p.name }
+func (p upperProc) Process(ctx *pipeline.Context) error {
+ *p.order = append(*p.order, p.name)
+ rec, ok := pipeline.CurrentRecord(ctx)
+ if !ok {
+ return nil
+ }
+ rec.Fields["upper"] = "yes"
+ return nil
+}
+
+func TestForEach_StageWiseOrderIsWrittenOrder(t *testing.T) {
+ var order []string
+ var batchLen []int
+ var sawUpper []string
+
+ upper := upperProc{"upper", &order}
+ batch := &stageBatchProc{"batch", &order, &batchLen, &sawUpper}
+ lower := orderRecorder{"lower", &order}
+
+ p := buildForEachCfg(t, util.MapStr{"processor": []interface{}{
+ util.MapStr{"test_upper": util.MapStr{}},
+ }})
+ p.sub.List = []pipeline.Processor{upper, batch, lower}
+
+ ctx := &pipeline.Context{Context: t.Context()}
+ msgs := []queue.Message{otelMsg(`{"payload":{"message":"a"}}`), otelMsg(`{"payload":{"message":"b"}}`)}
+ ctx.Set(param.ParaKey("messages"), msgs)
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ // execution order: upper(per record a,b) -> batch(once) -> lower(a,b)
+ want := []string{"upper", "upper", "batch", "lower", "lower"}
+ if strings.Join(order, ",") != strings.Join(want, ",") {
+ t.Fatalf("execution order = %v, want %v", order, want)
+ }
+ if len(batchLen) != 1 || batchLen[0] != 2 {
+ t.Fatalf("batch called %v with sizes %v, want once with 2", batchLen, batchLen)
+ }
+ // the batch segment must SEE the record processor's edits (the core
+ // semantic the two-phase behavior broke)
+ for _, saw := range sawUpper {
+ if saw != "yes" {
+ t.Fatalf("batch segment saw stale record state: %q", saw)
+ }
+ }
+}
+
+type dropFirstProc struct{ dropped *int }
+
+func (p dropFirstProc) Name() string { return "test_drop_first" }
+func (p dropFirstProc) Process(ctx *pipeline.Context) error {
+ rec, _ := pipeline.CurrentRecord(ctx)
+ if msg, _ := rec.Fields.GetValue("message"); util.ToString(msg) == "drop-me" {
+ pipeline.MarkDropped(rec)
+ *p.dropped++
+ }
+ return nil
+}
+
+func TestForEach_DropExcludesFromLaterSegments(t *testing.T) {
+ var batchLen []int
+ var sawUpper []string
+ var order []string
+ dropped := 0
+
+ p := buildForEachCfg(t, util.MapStr{"processor": []interface{}{
+ util.MapStr{"test_upper": util.MapStr{}},
+ }})
+ p.sub.List = []pipeline.Processor{
+ dropFirstProc{&dropped},
+ &stageBatchProc{"batch", &order, &batchLen, &sawUpper},
+ orderRecorder{"lower", &order},
+ }
+
+ ctx := &pipeline.Context{Context: t.Context()}
+ msgs := []queue.Message{
+ otelMsg(`{"payload":{"message":"drop-me"}}`),
+ otelMsg(`{"payload":{"message":"keep-me"}}`),
+ }
+ ctx.Set(param.ParaKey("messages"), msgs)
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if dropped != 1 {
+ t.Fatalf("dropped %d, want 1", dropped)
+ }
+ if batchLen[0] != 1 {
+ t.Fatalf("batch saw %d records, want 1 (dropped must be excluded)", batchLen[0])
+ }
+ if len(msgs[0].Data) != 0 {
+ t.Fatal("dropped record still encoded into message")
+ }
+ if len(msgs[1].Data) == 0 {
+ t.Fatal("kept record lost")
+ }
+}
+
+// cpuProc simulates CPU-bound per-record work for the parallel benchmark.
+type cpuProc struct{ n int }
+
+func (p *cpuProc) Name() string { return "bench_cpu" }
+func (p *cpuProc) Process(ctx *pipeline.Context) error {
+ rec, _ := pipeline.CurrentRecord(ctx)
+ if rec == nil {
+ return nil
+ }
+ h := sha256.New()
+ for i := 0; i < p.n; i++ {
+ h.Write([]byte("x"))
+ }
+ rec.Fields["digest"] = hex.EncodeToString(h.Sum(nil))[:8]
+ return nil
+}
+
+func parallelForEach(b *testing.B, parallel int) {
+ p := buildForEachCfgB(b, util.MapStr{
+ "processor": []interface{}{util.MapStr{"test_upper": util.MapStr{}}},
+ "parallel": parallel,
+ })
+ p.sub.List = []pipeline.Processor{&cpuProc{2048}}
+ msgs := make([]queue.Message, 64)
+ for i := range msgs {
+ msgs[i] = otelMsg(`{"payload":{"message":"m"}}`)
+ }
+ ctx := &pipeline.Context{Context: context.Background()}
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ ctx.Set(param.ParaKey("messages"), msgs)
+ if err := p.Process(ctx); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func BenchmarkForEach_Parallel1(b *testing.B) { parallelForEach(b, 1) }
+func BenchmarkForEach_Parallel4(b *testing.B) { parallelForEach(b, 4) }
+func BenchmarkForEach_Parallel8(b *testing.B) { parallelForEach(b, 8) }
+
+func buildForEachCfgB(b *testing.B, cfgMap util.MapStr) *ForEachProcessor {
+ b.Helper()
+ c, err := config.NewConfigFrom(cfgMap)
+ if err != nil {
+ b.Fatal(err)
+ }
+ p, err := NewForEachProcessor(c)
+ if err != nil {
+ b.Fatalf("NewForEachProcessor: %v", err)
+ }
+ return p.(*ForEachProcessor)
+}
diff --git a/modules/pipeline/for_each_test.go b/modules/pipeline/for_each_test.go
index cbfdd059a..4162b8b63 100644
--- a/modules/pipeline/for_each_test.go
+++ b/modules/pipeline/for_each_test.go
@@ -105,7 +105,9 @@ func TestForEach_UnknownCodecRejected(t *testing.T) {
}
func TestForEach_CustomCodec(t *testing.T) {
- RegisterRecordCodec(rawCodec{})
+ if err := RegisterRecordCodecOrReplace(rawCodec{}); err != nil {
+ t.Fatal(err)
+ }
p := buildForEach(t, map[string]interface{}{"codec": "raw"})
ctx := &pipeline.Context{Context: t.Context()}
msgs := []queue.Message{{Data: []byte("raw-bytes"), Size: 9, Offset: queue.AcquireOffset(0, 7)}}
@@ -130,7 +132,8 @@ func TestForEach_EmptyBatchNoop(t *testing.T) {
// rawCodec decodes arbitrary bytes as message-only events (test fixture).
type rawCodec struct{}
-func (rawCodec) Name() string { return "raw" }
+func (rawCodec) Name() string { return "raw" }
+func (rawCodec) Namespace() string { return "" }
func (rawCodec) Decode(data []byte) (*event.Event, error) {
e := &event.Event{Fields: util.MapStr{"message": string(data)}}
return e, nil
diff --git a/modules/pipeline/hotreload_test.go b/modules/pipeline/hotreload_test.go
new file mode 100644
index 000000000..cd76e495a
--- /dev/null
+++ b/modules/pipeline/hotreload_test.go
@@ -0,0 +1,111 @@
+/* Copyright © INFINI LTD. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "sync"
+ "testing"
+
+ "infini.sh/framework/core/pipeline"
+)
+
+// A processor name unique to this file — the plugin registry is
+// process-global and Setup() (which registers "echo") never runs in tests,
+// so a plain RegisterProcessorPlugin here cannot collide.
+var hotReloadOnce sync.Once
+
+func hotReloadTestProcessor() string {
+ const name = "test_hotreload_echo"
+ hotReloadOnce.Do(func() {
+ pipeline.RegisterProcessorPlugin(name, NewEchoProcessor)
+ })
+ return name
+}
+
+func hotCfg(id, procName, message string) pipeline.PipelineConfigV2 {
+ cfg := pipeline.PipelineConfigV2{
+ Name: id,
+ AutoStart: true,
+ Processors: []map[string]interface{}{
+ {procName: map[string]interface{}{"message": message}},
+ },
+ }
+ cfg.ID = id
+ return cfg
+}
+
+// TestHotReload_SwapsRunningTask covers the ORM CRUD reload path (#131):
+// a semantically identical edit is a no-op, a changed config drains and
+// rebuilds the task with a generation bump, delete removes the task, and
+// unknown IDs / transient tasks are left untouched.
+func TestHotReload_SwapsRunningTask(t *testing.T) {
+ proc := hotReloadTestProcessor()
+ m := &PipeModule{}
+
+ cfg1 := hotCfg("hr-1", proc, "v1")
+ if err := m.createPipelineTask(cfg1, false); err != nil {
+ t.Fatalf("create v1: %v", err)
+ }
+ if _, ok := m.contexts.Load("hr-1"); !ok {
+ t.Fatal("task context missing after create")
+ }
+
+ // semantically identical edit: no rebuild, no generation bump
+ same := hotCfg("hr-1", proc, "v1")
+ m.hotReloadPipeline("hr-1", &same)
+ if got := m.generationOf("hr-1"); got != 1 {
+ t.Fatalf("no-op edit bumped generation to %d", got)
+ }
+
+ // changed config: drain, rebuild, generation increments
+ cfg2 := hotCfg("hr-1", proc, "v2")
+ m.hotReloadPipeline("hr-1", &cfg2)
+ stored, ok := m.configs.Load("hr-1")
+ if !ok {
+ t.Fatal("task config missing after hot reload")
+ }
+ if !stored.(pipeline.PipelineConfigV2).Equals(cfg2) {
+ t.Fatalf("task not swapped to the new config: %+v", stored)
+ }
+ if _, ok := m.contexts.Load("hr-1"); !ok {
+ t.Fatal("task context missing after hot reload")
+ }
+ if got := m.generationOf("hr-1"); got != 2 {
+ t.Fatalf("generation = %d, want 2", got)
+ }
+
+ // the runtime report carries the generation so controllers can see
+ // the reload landed
+ snap := m.RuntimeSnapshot()
+ entry, ok := snap["hr-1"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("runtime snapshot has no entry for hr-1: %+v", snap)
+ }
+ if entry["generation"] != 2 {
+ t.Fatalf("runtime snapshot generation = %v, want 2", entry["generation"])
+ }
+
+ // delete: stop and remove, no recreate
+ m.hotReloadPipeline("hr-1", nil)
+ if _, ok := m.configs.Load("hr-1"); ok {
+ t.Fatal("config survives a delete hot reload")
+ }
+ if _, ok := m.contexts.Load("hr-1"); ok {
+ t.Fatal("context survives a delete hot reload")
+ }
+
+ // unknown ID: no-op, no panic
+ m.hotReloadPipeline("no-such-pipeline", nil)
+
+ // transient tasks are never reloaded
+ cfgT := hotCfg("hr-transient", proc, "t")
+ if err := m.createPipelineTask(cfgT, true); err != nil {
+ t.Fatalf("create transient: %v", err)
+ }
+ m.hotReloadPipeline("hr-transient", nil)
+ if _, ok := m.configs.Load("hr-transient"); !ok {
+ t.Fatal("transient task was reloaded")
+ }
+}
diff --git a/modules/pipeline/instance_id_test.go b/modules/pipeline/instance_id_test.go
new file mode 100644
index 000000000..356745ac5
--- /dev/null
+++ b/modules/pipeline/instance_id_test.go
@@ -0,0 +1,46 @@
+/* Copyright © INFINI LTD. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "testing"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/pipeline"
+)
+
+// TestStageStatsCarryInstanceID — every stage row self-describes its
+// emitting node (#127), so rows shipped to cluster views or persisted
+// cross-instance can be grouped by origin.
+func TestStageStatsCarryInstanceID(t *testing.T) {
+ installStats()
+ rec := globalStats.Load().(*statsRecorder)
+ rec.OnStep(&pipeline.StepEvent{Pipeline: "iid-1", Processor: "echo", Outcome: pipeline.OutcomeOK})
+
+ rows := stageStats("iid-1")
+ if len(rows) == 0 {
+ t.Fatal("no stage rows after feeding the recorder")
+ }
+ self := global.Env().SystemConfig.NodeConfig.ID
+ for _, r := range rows {
+ if r.InstanceID != self {
+ t.Fatalf("stage %q row instance_id = %q, want %q", r.Stage, r.InstanceID, self)
+ }
+ }
+}
+
+// TestRecordPipelineEventStampsInstance — control-plane events (lock
+// takeover) land in the flight ring attributed to this instance.
+func TestRecordPipelineEventStampsInstance(t *testing.T) {
+ installFlight()
+ recordPipelineEvent("iid-evt-1", pipeline.OutcomeTakeover, "took from node-X")
+ evs := flightQuery("iid-evt-1", "", true, 10)
+ if len(evs) != 1 {
+ t.Fatalf("event not retained: %+v", evs)
+ }
+ if evs[0].InstanceID != global.Env().SystemConfig.NodeConfig.ID {
+ t.Fatalf("event instance_id = %q", evs[0].InstanceID)
+ }
+}
diff --git a/modules/pipeline/lane_consumer.go b/modules/pipeline/lane_consumer.go
new file mode 100644
index 000000000..d9d8f4e5d
--- /dev/null
+++ b/modules/pipeline/lane_consumer.go
@@ -0,0 +1,232 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ log "github.com/cihub/seelog"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+)
+
+// LaneConsumerConfig declares the lane_consumer chain host: the record-
+// convention counterpart of the queue consumer. Records arrive as live
+// pointers (no codec decode), the sub-chain executes stage-wise in
+// written order, and surviving records optionally encode once into the
+// output_queue durable bridge.
+type LaneConsumerConfig struct {
+ Lane string `config:"lane"`
+
+ BatchMax int `config:"batch_max"` // flush on size
+ BatchFlushMs int `config:"batch_flush_ms"` // flush on idle
+
+ OnFailure string `config:"on_failure"` // tag (default) | dead_letter: | ignore
+ FailureTag string `config:"failure_tag"`
+
+ OutputQueue string `config:"output_queue"` // durable bridge (encode once)
+ OutputCodec string `config:"output_codec"` // override the 3-level resolution
+
+ Processors []*config.Config `config:"processor"`
+}
+
+type LaneConsumerProcessor struct {
+ cfg LaneConsumerConfig
+ lane lane.Lane
+ sub *pipeline.Processors
+ deadQ *queue.QueueConfig
+ outQ *queue.QueueConfig
+}
+
+const laneConsumerName = "lane_consumer"
+
+func init() {
+ pipeline.RegisterProcessorPluginWithConfigMetadata(laneConsumerName, NewLaneConsumerProcessor, LaneConsumerConfig{})
+ pipeline.RegisterProcessorContract(laneConsumerName, pipeline.RecordContract{Scope: pipeline.ScopeBatch})
+}
+
+func NewLaneConsumerProcessor(c *config.Config) (pipeline.Processor, error) {
+ cfg := LaneConsumerConfig{BatchMax: 64, BatchFlushMs: 100, OnFailure: "tag"}
+ if err := c.Unpack(&cfg); err != nil {
+ return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", laneConsumerName, err)
+ }
+ if cfg.Lane == "" {
+ return nil, fmt.Errorf("%s requires a lane", laneConsumerName)
+ }
+ if len(cfg.Processors) == 0 {
+ return nil, fmt.Errorf("%s requires a non-empty processor sub-chain", laneConsumerName)
+ }
+ if cfg.BatchMax <= 0 {
+ cfg.BatchMax = 64
+ }
+ if cfg.FailureTag == "" {
+ cfg.FailureTag = "_processing_failed"
+ }
+ // GetOrCreate (not Get+error): a lane_consumer task restart races its
+ // producers (output{lane} / consumer{output_lane}) re-creating the lane
+ // after this task's previous instance deleted it on exit — a hard "not
+ // found" here would leave the pipeline dead until the next producer
+ // emit. An explicit `lane:` stanza still wins when present; only the
+ // absent/closed case falls back to a default-config lane.
+ l := lane.GetOrCreate(lane.LaneConfig{Name: cfg.Lane})
+ sub, err := pipeline.NewPipelineWithScope(cfg.Processors, pipeline.ScopeInfo{
+ Kind: pipeline.ScopeRecord,
+ Namespace: l.Config().Namespace,
+ }, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to compile %s sub-chain: %s", laneConsumerName, err)
+ }
+ p := &LaneConsumerProcessor{cfg: cfg, lane: l, sub: sub}
+ if cfg.OutputQueue != "" {
+ p.outQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.OutputQueue})
+ }
+ if len(cfg.OnFailure) > len("dead_letter:") && cfg.OnFailure[:len("dead_letter:")] == "dead_letter:" {
+ p.deadQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.OnFailure[len("dead_letter:"):], Labels: nil})
+ }
+ return p, nil
+}
+
+func (p *LaneConsumerProcessor) Name() string { return laneConsumerName }
+
+// ApplyScope recompiles the sub-chain under the enclosing compile scope:
+// the registry constructor takes no scope, so the initial compile is
+// warn-only — a strict parent demands the strict view of the sub-chain
+// (pipeline.ScopeAwareProcessor; idempotent by replacement).
+func (p *LaneConsumerProcessor) ApplyScope(scope pipeline.ScopeInfo) error {
+ if !scope.Strict {
+ return nil
+ }
+ sub, err := pipeline.NewPipelineWithScope(p.cfg.Processors, pipeline.ScopeInfo{
+ Kind: pipeline.ScopeRecord,
+ Namespace: p.lane.Config().Namespace,
+ Strict: true,
+ }, true)
+ if err != nil {
+ return fmt.Errorf("failed to compile %s sub-chain: %s", laneConsumerName, err)
+ }
+ p.sub = sub
+ return nil
+}
+
+// Process subscribes (once) and blocks until the pipeline task cancels —
+// the host drives the lane for its whole lifetime.
+func (p *LaneConsumerProcessor) Process(ctx *pipeline.Context) error {
+ segments := buildSegments(p.sub.List)
+ batchFn := func(recs []*event.Event) {
+ p.runBatch(ctx, segments, recs)
+ }
+ _ = p.lane.SubscribeBatch(p.cfg.BatchMax, time.Duration(p.cfg.BatchFlushMs)*time.Millisecond, batchFn)
+ <-ctx.Done()
+ // Delete (close + unregister), not bare Close: a closed-but-registered
+ // lane accepts nothing forever and wedges every producer, and this
+ // task's own restart would re-bind the dead instance. Delete keeps the
+ // Close drain semantics (buffered batch flushes, receipts settle) and
+ // lets GetOrCreate re-create the lane on the next cycle.
+ _ = lane.Delete(p.cfg.Lane, true)
+ return nil
+}
+
+// runBatch executes the stage-wise segments over one micro-batch, then
+// encodes survivors into the durable bridge. Receipts of the batch settle
+// when this returns (SubscribeBatch contract) — processing completion is
+// the settlement point.
+func (p *LaneConsumerProcessor) runBatch(ctx *pipeline.Context, segments []execSegment, recs []*event.Event) {
+ active := recs
+ for _, seg := range segments {
+ if ctx.IsCanceled() || !ctx.ShouldContinue() {
+ return
+ }
+ if seg.batch {
+ live := make([]*event.Event, 0, len(active))
+ for _, rec := range active {
+ if !pipeline.IsDropped(rec) {
+ live = append(live, rec)
+ }
+ }
+ if len(live) == 0 {
+ continue
+ }
+ if err := seg.proc.(pipeline.BatchProcessor).ProcessBatch(ctx, live); err != nil {
+ if p.cfg.OnFailure == "ignore" {
+ log.Warnf("lane_consumer: batch segment error ignored: %v", err)
+ } else {
+ for _, rec := range live {
+ mergeFailureTags(rec, []string{p.cfg.FailureTag})
+ }
+ log.Warnf("lane_consumer: batch segment error tagged %q: %v", p.cfg.FailureTag, err)
+ }
+ }
+ continue
+ }
+ next := active[:0]
+ for _, rec := range active {
+ if pipeline.IsDropped(rec) {
+ continue
+ }
+ c := &pipeline.Context{ParentContext: ctx, Context: ctx.Context, Config: ctx.Config}
+ c.Set(pipeline.RecordContextKey, rec)
+ if p.cfg.OnFailure == "tag" {
+ var tags []string
+ c.Set(pipeline.FailureTagsKey, &tags)
+ for _, proc := range seg.chain.List {
+ if err := proc.Process(c); err != nil {
+ pipeline.AppendFailureTag(c, p.cfg.FailureTag)
+ log.Warnf("lane_consumer: sub-chain error tagged %q: %v", p.cfg.FailureTag, err)
+ }
+ }
+ mergeFailureTags(rec, tags)
+ } else if err := seg.chain.Process(c); err != nil {
+ log.Warnf("lane_consumer: record error (on_failure=%s): %v", p.cfg.OnFailure, err)
+ if p.deadQ != nil {
+ p.pushEncoded(p.deadQ, rec)
+ continue // diverted: dead_letter is a settlement
+ }
+ }
+ if !pipeline.IsDropped(rec) {
+ next = append(next, rec)
+ }
+ }
+ active = next
+ }
+ if p.outQ != nil {
+ for _, rec := range active {
+ if !pipeline.IsDropped(rec) {
+ p.pushEncoded(p.outQ, rec)
+ }
+ }
+ }
+}
+
+// pushEncoded applies the 3-level codec resolution and pushes to a queue.
+func (p *LaneConsumerProcessor) pushEncoded(q *queue.QueueConfig, rec *event.Event) {
+ codecName := p.cfg.OutputCodec
+ if codecName == "" {
+ codecName = pipeline.CodecHint(rec)
+ }
+ codec := GetRecordCodec(codecName)
+ if codecName == "" && p.lane.Config().Namespace != "" {
+ for _, ci := range pipeline.ListRecordCodecs() {
+ if ci.Namespace == p.lane.Config().Namespace {
+ codec = GetRecordCodec(ci.Name)
+ break
+ }
+ }
+ }
+ b, err := codec.Encode(rec)
+ if err != nil {
+ log.Errorf("lane_consumer: encode for output failed: %v", err)
+ return
+ }
+ if err := queue.Push(q, b); err != nil {
+ log.Errorf("lane_consumer: push to queue %q failed: %v", q.Name, err)
+ }
+}
+
+var _ = context.Background // keep context import for future cancellation plumbing
diff --git a/modules/pipeline/metrics.go b/modules/pipeline/metrics.go
new file mode 100644
index 000000000..fd95ff5f2
--- /dev/null
+++ b/modules/pipeline/metrics.go
@@ -0,0 +1,136 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+)
+
+// StageStats is the standard per-stage metric entry: one row per
+// processor (in-process stages) plus one row per lane carrier — the
+// runtime projection of the StepEvent stream.
+type StageStats struct {
+ // InstanceID identifies the emitting node: rows shipped to cluster
+ // views (or persisted cross-instance) self-describe their origin.
+ InstanceID string `json:"instance_id"`
+ Stage string `json:"stage"` // processor name or lane name
+ Carrier string `json:"carrier"` // "processor" | "lane:"
+ Count int64 `json:"count"` // invocations
+ Errors int64 `json:"errors"`
+ AvgLatency int64 `json:"avg_latency_ns"` // mean per invocation
+ LastOutcom string `json:"last_outcome,omitempty"`
+}
+
+type procStat struct {
+ count int64
+ errors int64
+ duration int64 // total ns
+ last pipeline.Outcome
+}
+
+// statsRecorder aggregates StepEvents into per-pipeline processor stats.
+type statsRecorder struct {
+ mu sync.Mutex
+ per map[string]map[string]*procStat
+ seq int64
+}
+
+func newStatsRecorder() *statsRecorder { return &statsRecorder{per: map[string]map[string]*procStat{}} }
+
+func (s *statsRecorder) OnStep(e *pipeline.StepEvent) {
+ // every event counts (aggregation is cheap: map ops); sampling is a
+ // flight-recorder concern, not a counter concern
+ s.mu.Lock()
+ m := s.per[e.Pipeline]
+ if m == nil {
+ m = map[string]*procStat{}
+ s.per[e.Pipeline] = m
+ }
+ st := m[e.Processor]
+ if st == nil {
+ st = &procStat{}
+ m[e.Processor] = st
+ }
+ st.count++
+ st.duration += int64(e.Duration)
+ st.last = e.Outcome
+ if e.Outcome == pipeline.OutcomeError {
+ st.errors++
+ }
+ s.mu.Unlock()
+}
+
+// stageStats snapshots one pipeline's processor stages plus every lane
+// carrier (depth from the live registry).
+func stageStats(pipelineName string) []StageStats {
+ out := processorStageStats(pipelineName)
+ out = append(out, laneStageStats()...)
+ return out
+}
+
+// processorStageStats snapshots one pipeline's processor rows only —
+// the per-pipeline slice of stageStats. Persistence and the sync report
+// call this per pipeline and add lane rows once globally (stageStats'
+// lane section is per-CALL, so folding it into a per-pipeline loop would
+// duplicate every lane under every pipeline).
+func processorStageStats(pipelineName string) []StageStats {
+ self := global.Env().SystemConfig.NodeConfig.ID
+ out := []StageStats{}
+ if rec := globalStats.Load(); rec != nil {
+ s := rec.(*statsRecorder)
+ s.mu.Lock()
+ if m := s.per[pipelineName]; m != nil {
+ for name, st := range m {
+ entry := StageStats{InstanceID: self, Stage: name, Carrier: "processor", Count: st.count, Errors: st.errors, LastOutcom: string(st.last)}
+ if st.count > 0 {
+ entry.AvgLatency = st.duration / st.count
+ }
+ out = append(out, entry)
+ }
+ }
+ s.mu.Unlock()
+ }
+ return out
+}
+
+// laneStageStats snapshots the cross-pipeline lane carrier rows — lanes
+// live in a global registry, they belong to no single pipeline.
+func laneStageStats() []StageStats {
+ self := global.Env().SystemConfig.NodeConfig.ID
+ out := make([]StageStats, 0, len(lane.List()))
+ for _, l := range lane.List() {
+ out = append(out, StageStats{
+ InstanceID: self,
+ Stage: l.Name,
+ Carrier: "lane:" + l.Name,
+ Count: l.Emitted,
+ Errors: l.Drops + l.Spills, // degraded deliveries surface as stage errors
+ })
+ }
+ return out
+}
+
+var globalStats atomic.Value
+
+func installStats() {
+ if globalStats.Load() != nil {
+ return
+ }
+ // one recorder for BOTH roles: registered observer (fed by the
+ // execution loop) and the globalStats snapshot (read by stageStats).
+ // Two separate instances here would count into one and read from the
+ // other — stage views would silently stay empty.
+ rec := newStatsRecorder()
+ pipeline.RegisterObserver(rec)
+ globalStats.Store(rec)
+}
+
+var _ = time.Second
diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go
new file mode 100644
index 000000000..8107c156e
--- /dev/null
+++ b/modules/pipeline/metrics_persist.go
@@ -0,0 +1,302 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "net/http"
+ "sort"
+ "time"
+
+ log "github.com/cihub/seelog"
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/env"
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+ "infini.sh/framework/modules/configs/common"
+)
+
+// PipelineStageStat is the persisted snapshot row of one stage at one
+// sampling tick (D-7: framework built-in persistence — one exporter,
+// every product inherits).
+type PipelineStageStat struct {
+ orm.ORMObjectBase `json:",inline" elastic_mapping:"-"`
+
+ // InstanceID identifies the emitting node — cross-instance queries
+ // group on it; the #130 acceptance (cluster aggregate = sum of the
+ // per-instance rows) keys off this column.
+ InstanceID string `json:"instance_id" elastic_mapping:"instance_id:{type:keyword}"`
+ Pipeline string `json:"pipeline" elastic_mapping:"pipeline:{type:keyword}"`
+ Stage string `json:"stage" elastic_mapping:"stage:{type:keyword}"`
+ Carrier string `json:"carrier" elastic_mapping:"carrier:{type:keyword}"`
+ Count int64 `json:"count" elastic_mapping:"count:{type:long}"`
+ Errors int64 `json:"errors" elastic_mapping:"errors:{type:long}"`
+ AvgLatency int64 `json:"avg_latency_ns" elastic_mapping:"avg_latency_ns:{type:long}"`
+ Ts time.Time `json:"ts" elastic_mapping:"ts:{type:date}"`
+}
+
+// metricsPersistConfig reads pipeline_metrics.persistence.
+type metricsPersistConfig struct {
+ Persistence struct {
+ Enabled bool `config:"enabled"`
+ Interval int `config:"interval_seconds"`
+ // RetentionSeconds bounds the snapshot table: rows older than the
+ // cutoff are pruned on every persist pass. The cluster fold only
+ // reads each instance's LATEST tick, so history beyond a window is
+ // dead weight (and would eventually crowd the aggregation query).
+ // Default 3600; 0 also means the default; negative disables.
+ Retention int `config:"retention_seconds"`
+ } `config:"persistence"`
+}
+
+// stageStatsRetention is the resolved retention window (shared by the
+// local exporter and the sync-report sink — one table, one policy).
+var stageStatsRetention = time.Hour
+
+// startMetricsPersist runs the snapshot exporter when enabled
+// (default interval 10s). Snapshots are cumulative counters; consumers
+// derive rates by differencing consecutive ticks.
+func (module *PipeModule) startMetricsPersist() {
+ cfg := metricsPersistConfig{}
+ exist, err := env.ParseConfig("pipeline_metrics", &cfg)
+ if err != nil || !exist || !cfg.Persistence.Enabled {
+ return
+ }
+ interval := time.Duration(cfg.Persistence.Interval) * time.Second
+ if interval <= 0 {
+ interval = 10 * time.Second
+ }
+ if cfg.Persistence.Retention > 0 {
+ stageStatsRetention = time.Duration(cfg.Persistence.Retention) * time.Second
+ } else if cfg.Persistence.Retention < 0 {
+ stageStatsRetention = 0 // disabled
+ }
+ go func() {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ if module.closed.Load() {
+ return
+ }
+ module.persistStageSnapshot()
+ }
+ }
+ }()
+ log.Debugf("pipeline metrics persistence enabled (interval %v)", interval)
+}
+
+// reportedStageRows collects every running pipeline's processor counters
+// plus the lane carriers once, in the shared wire shape (common.StageStat):
+// rows are non-zero-collectable, tagged with the pipeline they belong to
+// (Config.Name, falling back to the context key for id-less configs —
+// same resolution rule as RuntimeSnapshot). Lane rows are cross-pipeline
+// and carry an empty Pipeline tag.
+func (module *PipeModule) reportedStageRows() []common.StageStat {
+ self := global.Env().SystemConfig.NodeConfig.ID
+ toRow := func(pipelineName string, st StageStats) common.StageStat {
+ return common.StageStat{
+ Pipeline: pipelineName, InstanceID: st.InstanceID, Stage: st.Stage, Carrier: st.Carrier,
+ Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency,
+ }
+ }
+ out := []common.StageStat{}
+ seen := map[string]bool{}
+ module.contexts.Range(func(k, v interface{}) bool {
+ ctx, ok := v.(*pipeline.Context)
+ if !ok {
+ return true
+ }
+ name := ctx.Config.Name
+ if name == "" {
+ if s, ok := k.(string); ok {
+ name = s
+ }
+ }
+ if name == "" || seen[name] {
+ return true
+ }
+ seen[name] = true
+ for _, st := range processorStageStats(name) {
+ out = append(out, toRow(name, st))
+ }
+ return true
+ })
+ for _, st := range laneStageStats() {
+ out = append(out, toRow("", st))
+ }
+ // self-tag rows whose recorder ran before node config resolved
+ for i := range out {
+ if out[i].InstanceID == "" {
+ out[i].InstanceID = self
+ }
+ }
+ return out
+}
+
+// saveStageStatRows writes one snapshot tick of counters — shared by the
+// local exporter and the configs _sync sink (same table, same row shape).
+func saveStageStatRows(octx *orm.Context, rows []common.StageStat, now time.Time, src string) {
+ for _, st := range rows {
+ if st.Count == 0 && st.Errors == 0 {
+ continue
+ }
+ row := PipelineStageStat{
+ InstanceID: st.InstanceID, Pipeline: st.Pipeline, Stage: st.Stage, Carrier: st.Carrier,
+ Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency,
+ Ts: now,
+ }
+ row.ID = util.GetUUID()
+ if err := orm.Save(octx, &row); err != nil {
+ log.Warnf("%s: save %s/%s failed: %v", src, st.Pipeline, st.Stage, err)
+ }
+ }
+}
+
+// PersistReportedStageStats is the configs-server sink: an instance's
+// _sync-reported counters snapshot into the manager's store so cluster
+// aggregation covers instances that do not share its datastore. Same
+// row shape as the local exporter (cumulative; the fold takes each
+// instance's latest tick) and the same retention policy.
+func PersistReportedStageStats(octx *orm.Context, rows []common.StageStat) {
+ if len(rows) == 0 {
+ return
+ }
+ now := time.Now().UTC()
+ saveStageStatRows(octx, rows, now, "stage stats report")
+ pruneStageStats(now)
+}
+
+func (module *PipeModule) persistStageSnapshot() {
+ // DirectAccess: this is a background system writer (no request user);
+ // the security orm hooks require either a session user or the direct
+ // read/write flags — a bare context panics on the first save under
+ // security-enabled apps (coco) and takes the process down
+ octx := orm.NewContext().DirectAccess()
+ now := time.Now().UTC()
+ saveStageStatRows(octx, module.reportedStageRows(), now, "metrics persist")
+ pruneStageStats(now)
+}
+
+// pruneStageStats drops snapshot rows older than the retention window.
+// No-op when retention is disabled.
+func pruneStageStats(now time.Time) {
+ if stageStatsRetention <= 0 {
+ return
+ }
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineStageStat{})
+ qb := orm.NewQuery().Filter(orm.Range("ts").Lt(now.Add(-stageStatsRetention)))
+ if _, err := orm.DeleteByQuery(octx, qb); err != nil {
+ log.Debugf("stage stats prune: %v", err)
+ }
+}
+
+// ClusterStageStat is one stage aggregated across every instance that
+// reported it: counters sum over each instance's LATEST snapshot tick,
+// latency is the invocation-weighted mean, and PerInstance carries the
+// detail rows so the cluster math stays auditable (aggregate must equal
+// the sum of the parts).
+type ClusterStageStat struct {
+ Pipeline string `json:"pipeline"`
+ Stage string `json:"stage"`
+ Carrier string `json:"carrier"`
+ Count int64 `json:"count"`
+ Errors int64 `json:"errors"`
+ AvgLatency int64 `json:"avg_latency_ns"`
+ PerInstance map[string]StageStats `json:"per_instance"`
+}
+
+// aggregateStageRows folds persisted snapshot rows (cumulative counters
+// per tick) into per-stage cluster totals. Rows are sorted newest-first
+// internally; the first row of each (instance, stage) pair is that
+// instance's latest tick — older ticks are superseded, not summed.
+func aggregateStageRows(rows []PipelineStageStat) []ClusterStageStat {
+ sort.SliceStable(rows, func(i, j int) bool { return rows[i].Ts.After(rows[j].Ts) })
+
+ type stageKey struct{ pipeline, stage, carrier string }
+ latest := map[string]map[stageKey]StageStats{} // instance -> stage -> row
+ for _, r := range rows {
+ inst := r.InstanceID
+ if inst == "" {
+ inst = "?" // pre-#127 rows: attribute to a synthetic bucket
+ }
+ key := stageKey{r.Pipeline, r.Stage, r.Carrier}
+ m := latest[inst]
+ if m == nil {
+ m = map[stageKey]StageStats{}
+ latest[inst] = m
+ }
+ if _, seen := m[key]; seen {
+ continue
+ }
+ m[key] = StageStats{
+ InstanceID: r.InstanceID, Stage: r.Stage, Carrier: r.Carrier,
+ Count: r.Count, Errors: r.Errors, AvgLatency: r.AvgLatency,
+ }
+ }
+
+ totals := map[stageKey]*ClusterStageStat{}
+ for _, m := range latest {
+ for key, st := range m {
+ agg := totals[key]
+ if agg == nil {
+ agg = &ClusterStageStat{
+ Pipeline: key.pipeline, Stage: key.stage, Carrier: key.carrier,
+ PerInstance: map[string]StageStats{},
+ }
+ totals[key] = agg
+ }
+ agg.Count += st.Count
+ agg.Errors += st.Errors
+ agg.PerInstance[st.InstanceID] = st
+ }
+ }
+ out := make([]ClusterStageStat, 0, len(totals))
+ for _, agg := range totals {
+ var totalInvocations int64
+ for _, st := range agg.PerInstance {
+ totalInvocations += st.Count
+ }
+ if totalInvocations > 0 {
+ var weighted int64
+ for _, st := range agg.PerInstance {
+ weighted += st.AvgLatency * st.Count
+ }
+ agg.AvgLatency = weighted / totalInvocations
+ }
+ out = append(out, *agg)
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if out[i].Pipeline != out[j].Pipeline {
+ return out[i].Pipeline < out[j].Pipeline
+ }
+ return out[i].Stage < out[j].Stage
+ })
+ return out
+}
+
+// clusterStagesHandler — GET /pipeline/cluster/stages[?pipeline=]
+// Cluster-wide stage totals from the persisted snapshots: every
+// instance's latest tick summed per stage, with the per-instance detail
+// alongside (the aggregate must equal the sum of the parts).
+func (h *PipeModule) clusterStagesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineStageStat{})
+ q := orm.NewQuery().Size(4000)
+ if p := req.URL.Query().Get("pipeline"); p != "" {
+ q = q.Filter(orm.TermQuery("pipeline", p))
+ }
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ rows, _, _ := elastic.DecodeHits[PipelineStageStat](res)
+ h.WriteJSON(w, util.MapStr{"stages": aggregateStageRows(rows)}, http.StatusOK)
+}
diff --git a/modules/pipeline/metrics_report_test.go b/modules/pipeline/metrics_report_test.go
new file mode 100644
index 000000000..1d481c48f
--- /dev/null
+++ b/modules/pipeline/metrics_report_test.go
@@ -0,0 +1,105 @@
+/* Copyright © INFINI Ltd. All rights reserved. */
+
+package pipeline
+
+import (
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/modules/configs/common"
+)
+
+// the package's single sqlite handler (see ensureDeployORMHandler) already
+// carries the pipeline-stage-stats-test schema — reuse it, no second
+// orm.Register (last-wins would strand the deploy tests' schemas)
+
+func TestReportedStageRows_TagByPipelineAndDedupNames(t *testing.T) {
+ installStats()
+ rec := globalStats.Load().(*statsRecorder)
+ rec.OnStep(&pipeline.StepEvent{Pipeline: "pA", Processor: "a1", Outcome: pipeline.OutcomeOK})
+ rec.OnStep(&pipeline.StepEvent{Pipeline: "pA", Processor: "a2", Outcome: pipeline.OutcomeOK})
+ rec.OnStep(&pipeline.StepEvent{Pipeline: "pB", Processor: "b1", Outcome: pipeline.OutcomeOK})
+
+ module := &PipeModule{}
+ // two context keys resolving to the SAME Config.Name — the seen-map
+ // must collect pA once (id-less configs key by name; duplicate keys
+ // under one name must not double the rows)
+ module.contexts.Store("pA", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pA"}))
+ module.contexts.Store("pA-id-alias", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pA"}))
+ module.contexts.Store("pB", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pB"}))
+ // a context whose Config.Name is empty falls back to the key
+ module.contexts.Store("pC", pipeline.AcquireContext(pipeline.PipelineConfigV2{}))
+
+ rows := module.reportedStageRows()
+ type key struct{ p, s string }
+ got := map[key]int64{}
+ for _, r := range rows {
+ if r.Carrier != "processor" {
+ t.Fatalf("unexpected carrier in report: %+v", r)
+ }
+ got[key{r.Pipeline, r.Stage}]++
+ }
+ want := map[key]int64{
+ {"pA", "a1"}: 1, {"pA", "a2"}: 1,
+ {"pB", "b1"}: 1,
+ {"pC", "pC"}: 0, // placeholder replaced below; pC has no recorded steps
+ }
+ delete(want, key{"pC", "pC"})
+ if len(got) != len(want) {
+ t.Fatalf("rows = %+v, want exactly %+v", rows, want)
+ }
+ for k, n := range want {
+ if got[k] != n {
+ t.Fatalf("row %v appears %d times, want %d (rows: %+v)", k, got[k], n, rows)
+ }
+ }
+}
+
+func TestPersistReportedStageStats_SavesAndPrunes(t *testing.T) {
+ ensureDeployORMHandler(t)
+ prevRetention := stageStatsRetention
+ defer func() { stageStatsRetention = prevRetention }()
+
+ octx := orm.NewContext().DirectAccess()
+ PersistReportedStageStats(octx, []common.StageStat{
+ {Pipeline: "sp", InstanceID: "inst-1", Stage: "echo", Carrier: "processor", Count: 10, Errors: 0, AvgLatency: 1000},
+ {Pipeline: "sp", InstanceID: "inst-1", Stage: "idle", Carrier: "processor", Count: 0, Errors: 0}, // zero row skipped
+ })
+
+ // a stale row beyond the retention window — direct save, Ts backdated
+ old := PipelineStageStat{
+ InstanceID: "inst-0", Pipeline: "sp", Stage: "echo", Carrier: "processor",
+ Count: 1, Ts: time.Now().UTC().Add(-2 * time.Hour),
+ }
+ old.ID = "stats-old-tick"
+ if err := orm.Save(octx, &old); err != nil {
+ t.Fatal(err)
+ }
+
+ countRows := func() int {
+ qctx := orm.NewContext().DirectAccess()
+ orm.WithModel(qctx, &PipelineStageStat{})
+ res, err := orm.SearchV2(qctx, orm.NewQuery().Size(100))
+ if err != nil {
+ t.Fatal(err)
+ }
+ rows, _, _ := elastic.DecodeHits[PipelineStageStat](res)
+ return len(rows)
+ }
+
+ if n := countRows(); n != 2 {
+ t.Fatalf("after sink: %d rows, want 2 (fresh + backdated)", n)
+ }
+
+ // 1h retention prunes the 2h-old row on the next persist pass
+ stageStatsRetention = time.Hour
+ PersistReportedStageStats(octx, []common.StageStat{
+ {Pipeline: "sp", InstanceID: "inst-1", Stage: "echo", Carrier: "processor", Count: 12, Errors: 0, AvgLatency: 1000},
+ })
+ if n := countRows(); n != 2 {
+ t.Fatalf("after prune: %d rows, want 2 (two fresh ticks, backdated gone)", n)
+ }
+}
diff --git a/modules/pipeline/metrics_test.go b/modules/pipeline/metrics_test.go
new file mode 100644
index 000000000..58ea7dbc4
--- /dev/null
+++ b/modules/pipeline/metrics_test.go
@@ -0,0 +1,67 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/pipeline"
+)
+
+func TestStats_AggregatesPerProcessor(t *testing.T) {
+ installStats()
+ s := globalStats.Load().(*statsRecorder)
+ s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeOK, Duration: 100 * time.Millisecond})
+ s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeOK, Duration: 300 * time.Millisecond})
+ s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeError, Duration: 50 * time.Millisecond})
+
+ stages := stageStats("sp")
+ var found *StageStats
+ for i := range stages {
+ if stages[i].Stage == "a" && stages[i].Carrier == "processor" {
+ found = &stages[i]
+ }
+ }
+ if found == nil {
+ t.Fatalf("stage a missing: %+v", stages)
+ }
+ if found.Count != 3 || found.Errors != 1 {
+ t.Fatalf("counters: %+v", found)
+ }
+ if want := int64(150 * time.Millisecond); found.AvgLatency != want {
+ t.Fatalf("avg latency = %d, want %d", found.AvgLatency, want)
+ }
+}
+
+// obsWiringProc pairs with TestStats_ObserverWiring: a no-op processor so
+// the shared Processors.Process loop runs it and emits real StepEvents.
+type obsWiringProc struct{}
+
+func (obsWiringProc) Name() string { return "wiring_probe" }
+func (obsWiringProc) Process(_ *pipeline.Context) error { return nil }
+
+// TestStats_ObserverWiring guards the installStats identity contract: the
+// recorder registered as observer must be the one stageStats reads. The
+// historical bug registered one instance and stored a second — every
+// stage view stayed empty while counters accumulated into a dead object.
+func TestStats_ObserverWiring(t *testing.T) {
+ installStats()
+
+ ctx := pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "stats-wiring"})
+ ctx.Started()
+ chain := &pipeline.Processors{List: []pipeline.Processor{obsWiringProc{}}}
+ if err := chain.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ stages := stageStats("stats-wiring")
+ for _, st := range stages {
+ if st.Stage == "wiring_probe" && st.Carrier == "processor" && st.Count >= 1 {
+ return
+ }
+ }
+ t.Fatalf("observer-fed counters missing from stageStats: %+v", stages)
+}
diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go
index e318b73fd..ecbd00b3f 100755
--- a/modules/pipeline/module.go
+++ b/modules/pipeline/module.go
@@ -42,9 +42,13 @@ import (
"infini.sh/framework/core/env"
"infini.sh/framework/core/errors"
"infini.sh/framework/core/global"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/orm"
"infini.sh/framework/core/pipeline"
"infini.sh/framework/core/rate"
"infini.sh/framework/core/util"
+ configs_client "infini.sh/framework/modules/configs/client"
+ configs_server "infini.sh/framework/modules/configs/server"
)
type PipeModule struct {
@@ -57,6 +61,112 @@ type PipeModule struct {
pipelines sync.Map
configs sync.Map
contexts sync.Map
+ // failures records pipelines whose config failed to compile on THIS
+ // instance (e.g. a processor missing from the local catalog) — the
+ // deployment status needs to see WHY, not just that it is absent
+ failures sync.Map // name -> string (compile/load error)
+ // generations counts the config revisions applied per task (1 = the
+ // startup version); each successful hot reload increments it and the
+ // runtime report exposes it so controllers can see a reload landed
+ generations sync.Map // taskID -> int
+ // cordoned marks the instance under maintenance: running tasks drain
+ // to completion, but no task starts (auto_start/_start/_resume, and
+ // tasks recreated by hot reload park STOPPED) until uncordoned
+ cordoned atomic.Bool
+ // rolloutMu serializes staged-rollout state transitions between the
+ // reconciler tick and the _rollout endpoint (both read-modify-write
+ // the deployment's RolloutState)
+ rolloutMu sync.Mutex
+}
+
+// recordFailure stores a pipeline compile/load failure for the runtime
+// report; recordSuccess clears a stale entry after the pipeline runs.
+func (module *PipeModule) recordFailure(name string, err error) {
+ if name == "" || err == nil {
+ return
+ }
+ module.failures.Store(name, err.Error())
+}
+
+func (module *PipeModule) recordSuccess(name string) {
+ module.failures.Delete(name)
+}
+
+// RuntimeSnapshot returns the current pipeline task states for the
+// configs _sync runtime report (consumed by the deployment reconciler).
+func (module *PipeModule) RuntimeSnapshot() map[string]interface{} {
+ out := map[string]interface{}{}
+ module.contexts.Range(func(k, v interface{}) bool {
+ ctx, ok := v.(*pipeline.Context)
+ if !ok {
+ return true
+ }
+ // report under the pipeline NAME — consumers (deployment status)
+ // match by name; the map key is the config ID which only equals
+ // the name for id-less YAML configs
+ name := ctx.Config.Name
+ if name == "" {
+ if s, ok := k.(string); ok {
+ name = s
+ }
+ }
+ state := "UNKNOWN"
+ switch st := ctx.GetRunningState(); st {
+ case pipeline.STARTED, pipeline.STARTING:
+ state = "STARTED"
+ case pipeline.FAILED:
+ state = "FAILED"
+ case pipeline.STOPPED, pipeline.STOPPING:
+ state = "STOPPED"
+ case pipeline.FINISHED:
+ // keep_running tasks idle between passes (retry delay) with
+ // state FINISHED — the loop goroutine is alive and will run
+ // again. Reporting STOPPED here flaps a healthy fleet to
+ // "down" once per inter-pass sleep and permanently stalls
+ // rollout waves (readiness requires STARTED). A FAILED pass
+ // still reports FAILED: crash-loops must hold rollouts.
+ if ctx.Config.KeepRunning && !ctx.IsExit() {
+ state = "STARTED"
+ } else {
+ state = "STOPPED"
+ }
+ }
+ entry := map[string]interface{}{
+ "state": state,
+ }
+ // the running config's content digest — same canonical shape the
+ // deployment renderer produces, so the control plane can verify a
+ // staged rollout wave actually runs the pushed content (#125)
+ entry["config_digest"] = deployedContentDigest(deployedPipelineMap(&ctx.Config, renderSpec{
+ enabled: isPipelineEnabled(ctx.Config.Enabled),
+ singleton: ctx.Config.Singleton,
+ }))
+ if taskID, ok := k.(string); ok {
+ entry["generation"] = module.generationOf(taskID)
+ }
+ out[name] = entry
+ return true
+ })
+ // pipelines that never made it into contexts (config rejected or
+ // processor missing locally) are reported as FAILED with the reason —
+ // absent entries hide breakage from the deployment status
+ module.failures.Range(func(k, v interface{}) bool {
+ name, ok := k.(string)
+ if !ok {
+ return true
+ }
+ if _, running := out[name]; running {
+ return true
+ }
+ if msg, ok := v.(string); ok {
+ out[name] = map[string]interface{}{
+ "state": "FAILED",
+ "reason": msg,
+ }
+ }
+ return true
+ })
+ return out
}
func (module *PipeModule) Name() string {
@@ -67,7 +177,48 @@ var moduleCfg = struct {
PipelineEnabledByDefault bool `config:"pipeline_enabled_by_default"`
}{PipelineEnabledByDefault: true}
+// loadLaneConfigs reads the top-level `lane:` stanzas (the static plane;
+// dynamic lanes come via lane.Create at runtime).
+func (module *PipeModule) loadLaneConfigs() {
+ cfgs := []lane.LaneConfig{}
+ if exist, err := env.ParseConfig("lane", &cfgs); !exist || err != nil {
+ log.Debugf("no lane config section (exist=%v err=%v)", exist, err)
+ }
+ for _, c := range cfgs {
+ if c.Name == "" {
+ continue
+ }
+ if _, err := lane.Create(c); err != nil {
+ log.Warnf("lane %s: %v", c.Name, err)
+ }
+ }
+}
+
func (module *PipeModule) Setup() {
+ // provide the runtime snapshot for the configs _sync client
+ // (deployment reconciler reads it for health status)
+ configs_client.SetPipelineRuntimeSnapshotProvider(module.RuntimeSnapshot)
+ // instance-level maintenance flags ride the same _sync report (#128):
+ // the control plane mirrors the local cordon state onto the Instance
+ configs_client.SetInstanceFlagsProvider(func() map[string]interface{} {
+ return map[string]interface{}{"cordoned": module.cordoned.Load()}
+ })
+ // per-stage counters ride the same heartbeat: instances without a
+ // shared datastore still land in the manager's cluster aggregation.
+ // The sink half runs only on apps that embed the configs server
+ // (coco/logpilot managers); on agent-only processes it is inert.
+ configs_client.SetStageStatsProvider(module.reportedStageRows)
+ configs_server.SetStageStatsSink(PersistReportedStageStats)
+ // ORM schema registration (table/index auto-creation on startup)
+ orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs")
+ orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions")
+ orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments")
+ orm.MustRegisterSchemaWithIndexName(PipelineStageStat{}, "pipeline-stage-stats")
+
+ module.loadLaneConfigs()
+ installFlight()
+ installStats()
+ module.startDeploymentReconciler()
if global.Env().IsDebug {
log.Debug("pipeline framework config: ", moduleCfg)
}
@@ -104,10 +255,106 @@ func (module *PipeModule) Setup() {
api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopPipelineTaskHandler)
api.HandleUIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ // graceful stop (#121): finish the current batch, commit, park
+ api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_drain", module.drainPipelineTaskHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/task/:id/_drain", module.drainPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
+ // instance maintenance (#121): drain everything / cordon against new starts
+ api.HandleAPIMethod(api.POST, "/pipeline/_drain", module.drainInstanceHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/_drain", module.drainInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.POST, "/pipeline/_cordon", module.cordonInstanceHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/_cordon", module.cordonInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.POST, "/pipeline/_uncordon", module.uncordonInstanceHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/_uncordon", module.uncordonInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.GET, "/pipeline/_cordon", module.getCordonHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/_cordon", module.getCordonHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
// processor discovery: names + config schemas for pipeline designers
api.HandleAPIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler)
api.HandleUIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+ // config pre-flight: compile check without persisting (editor save hint)
+ api.HandleAPIMethod(api.POST, "/pipeline/_validate", module.validatePipelineHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/_validate", module.validatePipelineHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ api.HandleAPIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ api.HandleAPIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ // Fleet API (deployment management, D7)
+ api.HandleAPIMethod(api.GET, "/pipeline/deployments/", module.listDeploymentsHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/deployments/", module.listDeploymentsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+ api.HandleAPIMethod(api.POST, "/pipeline/deployments/", module.createDeploymentHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/deployments/", module.createDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineCreate))
+ api.HandleAPIMethod(api.GET, "/pipeline/deployments/:id", module.getDeploymentHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/deployments/:id", module.getDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+ api.HandleAPIMethod(api.PUT, "/pipeline/deployments/:id", module.updateDeploymentHandler)
+ api.HandleUIMethod(api.PUT, "/pipeline/deployments/:id", module.updateDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler)
+ api.HandleUIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineDelete))
+ api.HandleAPIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ // staged rollout control (#125): manual advance / content rollback
+ api.HandleAPIMethod(api.POST, "/pipeline/deployments/:id/_rollout", module.rolloutActionHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/deployments/:id/_rollout", module.rolloutActionHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
+ // cluster broadcast of transient commands (#125): forwarded per target
+ // over the reverse channel; ?generation=N carries the #120 split-brain
+ // guard — instances on another config generation answer 409
+ for _, action := range []string{"pause", "resume", "stop"} {
+ path := "/pipeline/deployments/:id/_" + action
+ api.HandleAPIMethod(api.POST, path, module.broadcastDeploymentActionHandler(action))
+ api.HandleUIMethod(api.POST, path, module.broadcastDeploymentActionHandler(action), api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ }
+
+ // per-instance maintenance over the reverse channel (#125): the
+ // instance-side primitives are #121's _drain/_cordon/_uncordon
+ for _, action := range []string{"drain", "cordon", "uncordon"} {
+ path := "/pipeline/instances/:id/_" + action
+ api.HandleAPIMethod(api.POST, path, module.instanceActionHandler(action))
+ api.HandleUIMethod(api.POST, path, module.instanceActionHandler(action), api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ }
+ // read-only passthrough onto an instance's own pipeline API
+ api.HandleAPIMethod(api.GET, "/pipeline/instances/:id/proxy", module.instanceProxyHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/instances/:id/proxy", module.instanceProxyHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ api.HandleAPIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ // cluster-wide stage totals from the persisted snapshots (#127/#130)
+ api.HandleAPIMethod(api.GET, "/pipeline/cluster/stages", module.clusterStagesHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/cluster/stages", module.clusterStagesHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ // pipeline version history + rollback
+ api.HandleAPIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler)
+ api.HandleUIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+ api.HandleAPIMethod(api.GET, "/pipelines/:id/versions/:version", module.getPipelineVersionHandler)
+ api.HandleUIMethod(api.GET, "/pipelines/:id/versions/:version", module.getPipelineVersionHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+ api.HandleAPIMethod(api.POST, "/pipelines/:id/versions/:version/rollback", module.rollbackPipelineHandler)
+ api.HandleUIMethod(api.POST, "/pipelines/:id/versions/:version/rollback", module.rollbackPipelineHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
+ api.HandleAPIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
+ api.HandleAPIMethod(api.POST, "/pipeline/ai-generate", module.postAIGenerateHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/ai-generate", module.postAIGenerateHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
+ api.HandleAPIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ api.HandleAPIMethod(api.GET, "/pipeline/task/:id/flight", module.flightPipelineTaskHandler)
+ api.HandleUIMethod(api.GET, "/pipeline/task/:id/flight", module.flightPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead))
+
+ api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_pause", module.pausePipelineTaskHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/task/:id/_pause", module.pausePipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_resume", module.resumePipelineTaskHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/task/:id/_resume", module.resumePipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+ api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_inject", module.injectPipelineTaskHandler)
+ api.HandleUIMethod(api.POST, "/pipeline/task/:id/_inject", module.injectPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate))
+
//use pipelines to avoid naming conflicts
api.HandleUIMethod(api.POST, "/pipelines/_search", module.searchPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Search)))
api.HandleUIMethod(api.POST, "/pipelines/", module.createPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Create)))
@@ -122,6 +369,23 @@ func (module *PipeModule) Setup() {
//api.HandleUIMethod(api.POST, "/pipeline/task/:id/metrics/_search", module.getPipelineMetricsHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin)))
}
+// pauseTask parks a running pipeline after its CURRENT batch: flow
+// stops between fetches, in-flight records finish (no mid-batch abort).
+func (module *PipeModule) pauseTask(taskID string) (exists bool) {
+ ctx, ok := module.contexts.Load(taskID)
+ if !ok {
+ return
+ }
+ if v1, ok := ctx.(*pipeline.Context); ok {
+ v1.Pause()
+ }
+ return true
+}
+
+func (module *PipeModule) resumeTask(taskID string) (exists bool) {
+ return module.startTask(taskID)
+}
+
func (module *PipeModule) startTask(taskID string) (exists bool) {
if module.closed.Load() {
return false
@@ -256,6 +520,7 @@ func getPipelineConfig() ([]pipeline.PipelineConfigV2, error) {
}
func (module *PipeModule) Start() error {
+ module.startMetricsPersist()
var (
pipelines []pipeline.PipelineConfigV2
err error
@@ -447,6 +712,186 @@ const pipelineSingleton = "pipeline_singleton"
var creatingLocker = sync.Mutex{}
+// hotReloadLocker serializes hot reloads — two rapid API edits to the same
+// pipeline would otherwise race their stop/delete/create sequences and
+// LoadOrStore could converge on the older config.
+var hotReloadLocker = sync.Mutex{}
+
+// generationOf returns the config revision counter for a task (1 = the
+// startup version; each successful hot reload increments it).
+func (module *PipeModule) generationOf(taskID string) int {
+ if v, ok := module.generations.Load(taskID); ok {
+ if n, ok := v.(int); ok && n > 0 {
+ return n
+ }
+ }
+ return 1
+}
+
+// hotReloadPipeline swaps the locally running task for pipelineID to newCfg
+// (nil = stop and remove only) — the ORM CRUD path. API edits used to only
+// persist: a running task kept its stale compiled chain until the instance
+// restarted. Same semantics as the fsnotify reload: drain the in-flight
+// batch (stopAndWaitForRelease), then rebuild from the new config.
+// Transient tasks are never reloaded.
+//
+// Matching is strict on task ID — deployment-rendered configs keep the ORM
+// pipeline ID, so an edit reaches every instance running the pipeline,
+// while unrelated local YAML tasks (their own IDs) stay untouched.
+func (module *PipeModule) hotReloadPipeline(pipelineID string, newCfg *pipeline.PipelineConfigV2) {
+ v, ok := module.configs.Load(pipelineID)
+ if !ok {
+ return
+ }
+ oldC, ok := v.(pipeline.PipelineConfigV2)
+ if !ok {
+ return
+ }
+ if oldC.Transient {
+ log.Debugf("pipeline [%s] is transient, skip hot reload", pipelineID)
+ return
+ }
+ if newCfg != nil && oldC.Equals(*newCfg) && isPipelineEnabled(newCfg.Enabled) {
+ return // semantically unchanged — nothing to rebuild
+ }
+
+ hotReloadLocker.Lock()
+ defer hotReloadLocker.Unlock()
+
+ defer func() {
+ if !global.Env().IsDebug {
+ if r := recover(); r != nil {
+ log.Error("error on hot reloading pipeline,", pipelineID, ",", r)
+ }
+ }
+ }()
+
+ log.Infof("pipeline [%s] config changed via API, reloading task", pipelineID)
+ module.stopAndWaitForRelease([]string{pipelineID}, time.Minute)
+ module.deleteTask(pipelineID)
+
+ if newCfg == nil {
+ return
+ }
+ if err := module.createPipelineTask(*newCfg, false); err != nil {
+ module.recordFailure(newCfg.Name, err)
+ log.Errorf("pipeline [%s] hot reload: recreate failed: %v", pipelineID, err)
+ return
+ }
+ module.generations.Store(pipelineID, module.generationOf(pipelineID)+1)
+}
+
+// reloadPipelineTaskAsync runs the hot reload off the request path —
+// draining the current batch can legitimately take seconds, and the API
+// response must not wait for it.
+func (module *PipeModule) reloadPipelineTaskAsync(pipelineID string, newCfg *pipeline.PipelineConfigV2) {
+ if _, ok := module.configs.Load(pipelineID); !ok {
+ return // no local task runs this ORM row
+ }
+ go module.hotReloadPipeline(pipelineID, newCfg)
+}
+
+// drainTask marks the task for a graceful stop (#121): the consumer
+// finishes the current batch and commits its offset, then the pipeline
+// loop parks STOPPED (resumable via _start). Returns false when the task
+// does not exist locally.
+func (module *PipeModule) drainTask(taskID string) bool {
+ v, ok := module.contexts.Load(taskID)
+ if !ok {
+ return false
+ }
+ ctx, ok := v.(*pipeline.Context)
+ if !ok {
+ return false
+ }
+ ctx.Drain()
+ ctx.Exit() // no keep_running restarts after the current pass
+ return true
+}
+
+// taskRunningState reports the task's current state ("" when absent).
+func (module *PipeModule) taskRunningState(taskID string) pipeline.RunningState {
+ v, ok := module.contexts.Load(taskID)
+ if !ok {
+ return ""
+ }
+ ctx, ok := v.(*pipeline.Context)
+ if !ok {
+ return ""
+ }
+ return ctx.GetRunningState()
+}
+
+// waitTaskSettled polls until the task leaves the running states or the
+// timeout elapses; returns the last observed state.
+func (module *PipeModule) waitTaskSettled(taskID string, timeout time.Duration) pipeline.RunningState {
+ deadline := time.Now().Add(timeout)
+ for {
+ state := module.taskRunningState(taskID)
+ if state != pipeline.STARTED && state != pipeline.STARTING && state != pipeline.STOPPING {
+ return state
+ }
+ if time.Now().After(deadline) {
+ return state
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}
+
+// drainAllTasks gracefully stops every running task on this instance
+// (maintenance path — pair with cordon so no new work starts while
+// draining). All drains are marked first, then one bounded wait; the
+// returned map carries each task's settled state.
+func (module *PipeModule) drainAllTasks(timeout time.Duration) map[string]string {
+ module.contexts.Range(func(k, v any) bool {
+ if id, ok := k.(string); ok {
+ module.drainTask(id)
+ }
+ return true
+ })
+ deadline := time.Now().Add(timeout)
+ out := map[string]string{}
+ for {
+ running := 0
+ module.contexts.Range(func(k, v any) bool {
+ id, ok := k.(string)
+ if !ok {
+ return true
+ }
+ state := module.taskRunningState(id)
+ out[id] = string(state)
+ if state == pipeline.STARTED || state == pipeline.STARTING || state == pipeline.STOPPING {
+ running++
+ }
+ return true
+ })
+ if running == 0 || time.Now().After(deadline) {
+ return out
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}
+
+// probeLockTakeover reports the previous owner when the singleton lock
+// for name sits EXPIRED under another instance — the Hold that follows
+// takes it over. kv-less environments (unit tests, lightweight agents)
+// report "" instead of panicking.
+func probeLockTakeover(bucket, name string, ttl time.Duration) string {
+ defer func() { _ = recover() }()
+ ok, info, err := locker.GetAllocateInfo(bucket, name)
+ if err != nil || !ok || info == nil {
+ return ""
+ }
+ self := global.Env().SystemConfig.NodeConfig.ID
+ if info.ClientID == "" || info.ClientID == self {
+ return ""
+ }
+ if time.Since(info.Timestamp) > ttl {
+ return info.ClientID
+ }
+ return ""
+}
+
// Helper function to run a pipeline, i.e., create a pipeline task.
func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transient bool) error {
if module.closed.Load() {
@@ -491,12 +936,14 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi
}
processorConfigs, err := cfg.GetProcessorsConfig()
if err != nil {
+ module.recordFailure(v.Name, err)
ready <- err
return errors.Errorf("failed to get processor config, %v", err)
}
processors, err := pipeline.NewPipeline(processorConfigs)
if err != nil {
+ module.recordFailure(v.Name, err)
ready <- err
return err
}
@@ -504,6 +951,7 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi
ctx := pipeline.AcquireContext(v)
module.pipelines.Store(v.ID, processors)
module.contexts.Store(v.ID, ctx)
+ module.recordSuccess(v.Name)
defer func() {
if !global.Env().IsDebug {
@@ -567,17 +1015,35 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi
switch state {
case pipeline.STARTING:
+ // cordoned instance: the task registers but does not run —
+ // it parks STOPPED and resumes via _start after _uncordon
+ if module.cordoned.Load() {
+ ctx.Stopped()
+ ctx.Pause()
+ continue
+ }
+
//check
if v.Singleton {
if v.MaxRunningInMs <= 0 {
v.MaxRunningInMs = 60000
}
- ok, err := locker.Hold(pipelineSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, time.Duration(v.MaxRunningInMs)*time.Millisecond, true)
+ ttl := time.Duration(v.MaxRunningInMs) * time.Millisecond
+ // takeover detection (#122): the lock's lease expired
+ // under another instance — the Hold below acquires it,
+ // ending that instance's run; surface the event
+ takeoverFrom := probeLockTakeover(pipelineSingleton, v.ID, ttl)
+ ok, err := locker.Hold(pipelineSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, ttl, true)
if !ok {
log.Debugf("pipeline [%v] is already running somewhere, %v", cfg.Name, err)
ctx.Finished()
continue
}
+ if takeoverFrom != "" {
+ msg := fmt.Sprintf("singleton lock taken over from instance %s (lease expired)", takeoverFrom)
+ log.Warnf("pipeline [%v]: %v", cfg.Name, msg)
+ recordPipelineEvent(cfg.Name, pipeline.OutcomeTakeover, msg)
+ }
}
// Pipeline needs to run
diff --git a/modules/pipeline/output.go b/modules/pipeline/output.go
new file mode 100644
index 000000000..1c2713e83
--- /dev/null
+++ b/modules/pipeline/output.go
@@ -0,0 +1,174 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+
+ log "github.com/cihub/seelog"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/param"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+)
+
+// OutputConfig declares the unified output step: exactly ONE target
+// field (queue or lane) — config unpacking rejects zero or multiple.
+type OutputConfig struct {
+ // Queue accepts either a bare queue name (string) or a full config
+ // object; the custom Unpack below handles both forms.
+ Queue interface{} `config:"queue"`
+ Lane string `config:"lane"`
+ // Ack (lane target only): emit with an acknowledgment receipt attached
+ // to the context's AckSet — the emitting consumer's commit gate waits
+ // for settlement (cross-lane at-least-once).
+ Ack *bool `config:"ack"`
+ Codec string `config:"codec"` // queue target: override codec resolution
+ MessageField param.ParaKey `config:"message_field"` // batch scope queue target
+}
+
+type OutputProcessor struct {
+ cfg OutputConfig
+ outQ *queue.QueueConfig
+ laneName string
+ ack bool
+}
+
+const outputName = "output"
+
+func init() {
+ pipeline.RegisterProcessorPluginWithConfigMetadata(outputName, NewOutputProcessor, OutputConfig{})
+ // batch+record conditional by target — declared record-agnostic; the
+ // batch-scope lane target errors at runtime with a clear pointer.
+ pipeline.RegisterProcessorContract(outputName, pipeline.RecordContract{})
+}
+
+func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) {
+ cfg := OutputConfig{MessageField: "messages"}
+ if err := c.Unpack(&cfg); err != nil {
+ return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", outputName, err)
+ }
+ targets := 0
+ queueName := ""
+ if cfg.Queue != nil {
+ targets++
+ // resolve: bare string or object with name
+ if s, ok := cfg.Queue.(string); ok {
+ queueName = s
+ } else if m, ok := cfg.Queue.(map[string]interface{}); ok {
+ if n, ok := m["name"].(string); ok {
+ queueName = n
+ }
+ }
+ }
+ if cfg.Lane != "" {
+ targets++
+ }
+ if targets != 1 {
+ return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", outputName, targets)
+ }
+ p := &OutputProcessor{cfg: cfg, laneName: cfg.Lane}
+ if queueName != "" {
+ qc := &queue.QueueConfig{Name: queueName}
+ p.outQ = queue.SmartGetOrInitConfig(qc)
+ }
+ if cfg.Ack != nil {
+ p.ack = *cfg.Ack
+ }
+ return p, nil
+}
+
+func (p *OutputProcessor) Name() string { return outputName }
+
+func (p *OutputProcessor) Process(ctx *pipeline.Context) error {
+ if p.laneName != "" {
+ return p.emitLane(ctx)
+ }
+ return p.pushQueue(ctx)
+}
+
+// capture records an intercepted write when the context runs a capturing
+// dry run; returns true when intercepted (the caller must NOT deliver).
+func (p *OutputProcessor) capture(ctx *pipeline.Context, kind, target string, preview []byte) bool {
+ sink := pipeline.DryRunCapture(ctx)
+ if sink == nil {
+ return false
+ }
+ *sink = append(*sink, pipeline.DryRunWrite{Kind: kind, Target: target, Records: 1, Preview: truncate(preview, 512)})
+ return true
+}
+
+func truncate(b []byte, n int) string {
+ if len(b) <= n {
+ return string(b)
+ }
+ return string(b[:n]) + "..."
+}
+
+// emitLane: record scope only — the batch scope has no live records.
+func (p *OutputProcessor) emitLane(ctx *pipeline.Context) error {
+ rec, ok := pipeline.CurrentRecord(ctx)
+ if !ok {
+ return fmt.Errorf("%s{lane:%s} requires record scope (inside for_each/lane_consumer); for disk→lane bridging use consumer{codec, output_lane}", outputName, p.laneName)
+ }
+ if p.capture(ctx, "lane", p.laneName, encodePreview(rec)) {
+ return nil
+ }
+ l := lane.Get(p.laneName)
+ if l == nil {
+ l = lane.GetOrCreate(lane.LaneConfig{Name: p.laneName})
+ }
+ if !p.ack {
+ return l.Emit(rec)
+ }
+ r := l.EmitAck(rec)
+ pipeline.AttachAckSet(ctx).AddReceipt(r)
+ return nil
+}
+
+// pushQueue: batch scope pushes message bytes as-is (queue_output
+// semantics); record scope encodes the current record once (durable
+// bridge, 3-level codec resolution).
+func (p *OutputProcessor) pushQueue(ctx *pipeline.Context) error {
+ if rec, ok := pipeline.CurrentRecord(ctx); ok {
+ codecName := p.cfg.Codec
+ if codecName == "" {
+ codecName = pipeline.CodecHint(rec)
+ }
+ b, err := GetRecordCodec(codecName).Encode(rec)
+ if err != nil {
+ return err
+ }
+ if p.capture(ctx, "queue", p.outQ.Name, b) {
+ return nil
+ }
+ return queue.Push(p.outQ, b)
+ }
+ obj := ctx.Get(p.cfg.MessageField)
+ msgs, ok := obj.([]queue.Message)
+ if !ok {
+ log.Warnf("processor [%s] no batch under [%s], skip", outputName, p.cfg.MessageField)
+ return nil
+ }
+ for i := range msgs {
+ if len(msgs[i].Data) == 0 {
+ continue
+ }
+ if err := queue.Push(p.outQ, msgs[i].Data); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func encodePreview(rec *event.Event) []byte {
+ b, err := GetRecordCodec(pipeline.CodecHint(rec)).Encode(rec)
+ if err != nil {
+ return nil
+ }
+ return b
+}
diff --git a/modules/pipeline/output_test.go b/modules/pipeline/output_test.go
new file mode 100644
index 000000000..86567281c
--- /dev/null
+++ b/modules/pipeline/output_test.go
@@ -0,0 +1,112 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+func buildOutput(t *testing.T, cfgMap util.MapStr) pipeline.Processor {
+ t.Helper()
+ c, err := config.NewConfigFrom(cfgMap)
+ if err != nil {
+ t.Fatal(err)
+ }
+ p, err := NewOutputProcessor(c)
+ if err != nil {
+ t.Fatalf("NewOutputProcessor: %v", err)
+ }
+ return p
+}
+
+func outCtx(t *testing.T, rec *event.Event) *pipeline.Context {
+ t.Helper()
+ cfg := pipeline.PipelineConfigV2{Name: "out-test"}
+ ctx := pipeline.AcquireContext(cfg)
+ ctx.Context = t.Context()
+ ctx.Started()
+ if rec != nil {
+ ctx.Set(pipeline.RecordContextKey, rec)
+ }
+ return ctx
+}
+
+// one-of target validation: zero or multiple targets reject at unpack
+func TestOutput_OneOfTargetValidation(t *testing.T) {
+ if _, err := NewOutputProcessor(mustCfg(t, util.MapStr{})); err == nil {
+ t.Fatal("zero targets must reject")
+ }
+ if _, err := NewOutputProcessor(mustCfg(t, util.MapStr{"queue": map[string]interface{}{"name": "q"}, "lane": "l"})); err == nil {
+ t.Fatal("two targets must reject")
+ }
+}
+
+func mustCfg(t *testing.T, m util.MapStr) *config.Config {
+ t.Helper()
+ c, err := config.NewConfigFrom(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+// lane target in record scope emits; ack attaches to the context AckSet
+func TestOutput_LaneTargetRecordScope(t *testing.T) {
+ ln, err := lane.Create(lane.LaneConfig{Name: "out-lane", Capacity: 4})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer lane.Delete("out-lane", true)
+
+ p := buildOutput(t, util.MapStr{"lane": "out-lane", "ack": true})
+ rec, _ := GetRecordCodec("").Decode([]byte(`{"payload":{"message":"m"}}`))
+ ctx := outCtx(t, rec)
+
+ if err := p.Process(ctx); err != nil {
+ t.Fatal(err)
+ }
+ set := pipeline.AckSetOf(ctx)
+ if set == nil || set.Pending() != 1 {
+ t.Fatalf("ack receipt not attached: %v", set)
+ }
+ // settle via a batch subscriber and confirm the gate drains
+ got := make(chan int, 1)
+ cancel := ln.SubscribeBatch(1, 10*time.Millisecond, func(recs []*event.Event) { got <- len(recs) })
+ defer cancel()
+ deadline := time.After(2 * time.Second)
+ for set.Pending() > 0 {
+ select {
+ case <-deadline:
+ t.Fatal("receipt never settled")
+ case <-got:
+ case <-time.After(time.Millisecond):
+ }
+ }
+}
+
+// lane target in batch scope errors with the bridge pointer
+func TestOutput_LaneTargetBatchScopeRejected(t *testing.T) {
+ _, err := lane.Create(lane.LaneConfig{Name: "out-lane2", Capacity: 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer lane.Delete("out-lane2", true)
+ p := buildOutput(t, util.MapStr{"lane": "out-lane2"})
+ ctx := outCtx(t, nil) // no current record = batch scope
+ if err := p.Process(ctx); err == nil {
+ t.Fatal("lane target in batch scope must error")
+ }
+}
+
+// queue-target construction requires the disk-queue subsystem (kv store);
+// the unpack-level one-of validation is covered above, byte-push semantics
+// ride the consumer integration tests.
diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go
index 7bedfeccc..573ea887f 100644
--- a/modules/pipeline/pipeline.go
+++ b/modules/pipeline/pipeline.go
@@ -8,6 +8,7 @@ import (
"net/http"
"time"
+ log "github.com/cihub/seelog"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/elastic"
"infini.sh/framework/core/orm"
@@ -27,6 +28,12 @@ func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Requ
return
}
+ // compile gate: reject configs that reference unknown processors or
+ // carry invalid parameters BEFORE they are persisted and distributed
+ if !h.validateAndReject(w, obj) {
+ return
+ }
+
ctx := orm.NewContextWithParent(req.Context())
ctx.Refresh = orm.WaitForRefresh
@@ -36,6 +43,10 @@ func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Requ
return
}
+ // a local task may already run under this ID (e.g. a deployment-
+ // rendered config) — swap it to the freshly created definition
+ h.reloadPipelineTaskAsync(obj.ID, obj)
+
h.WriteCreatedOKJSON(w, obj.ID)
}
@@ -103,6 +114,20 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ
// Ensure ID is set even if the client omitted it from the request body
newConfig.ID = id
+ // compile gate: an update that breaks the chain must not overwrite a
+ // working config (nor trigger a version snapshot of a rejected edit)
+ if !h.validateAndReject(w, &newConfig) {
+ return
+ }
+
+ // auto-snapshot the PREVIOUS config before overwriting (every edit is
+ // a version — no explicit publish step; transient pipelines skip)
+ if !oldConfig.Transient {
+ if err := h.saveVersion(ctx, &oldConfig, "auto-snapshot on update"); err != nil {
+ log.Warnf("pipeline %s: version snapshot failed: %v", id, err)
+ }
+ }
+
ctx.Refresh = orm.WaitForRefresh
err = orm.Save(ctx, &newConfig)
if err != nil {
@@ -110,12 +135,27 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ
return
}
+ // hot-reload any local task running this pipeline — the edit used to
+ // only persist, leaving a running chain stale until restart
+ h.reloadPipelineTaskAsync(newConfig.ID, &newConfig)
+
h.WriteUpdatedOKJSON(w, newConfig.ID)
}
func (h *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("id")
+ // archive before delete — the version history survives pipeline removal
+ // (transient pipelines skip archiving)
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ old := pipeline.PipelineConfigV2{}
+ old.ID = id
+ if exists, _ := orm.GetWithSystemFields(octx, &old); exists && !old.Transient {
+ if err := h.saveVersion(octx, &old, "archived on delete"); err != nil {
+ log.Warnf("pipeline %s: delete archive failed: %v", id, err)
+ }
+ }
+
obj := pipeline.PipelineConfigV2{}
obj.ID = id
@@ -138,6 +178,9 @@ func (h *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Requ
return
}
+ // stop and remove any local task running this pipeline
+ h.reloadPipelineTaskAsync(id, nil)
+
h.WriteDeletedOKJSON(w, id)
}
diff --git a/modules/pipeline/processors_api.go b/modules/pipeline/processors_api.go
index 2d7f8fdea..ab43758a0 100644
--- a/modules/pipeline/processors_api.go
+++ b/modules/pipeline/processors_api.go
@@ -31,7 +31,10 @@ import (
httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/lane"
"infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+ "infini.sh/framework/modules/pipeline/dryrun"
)
// getProcessorsHandler serves GET /pipeline/processors: the registry of
@@ -47,3 +50,55 @@ func (module *PipeModule) getProcessorsHandler(w http.ResponseWriter, req *http.
}
module.WriteJSON(w, pipeline.GetProcessorMetadata(), 200)
}
+
+// postDryRunHandler — POST /pipeline/dry-run
+// The unified dry-run (single definition): chain + inline samples +
+// options → Report (per-step snapshots, field diffs, captured writes).
+type dryRunRequest struct {
+ Processor []map[string]interface{} `json:"processor"`
+ Samples []struct {
+ Raw string `json:"raw"`
+ Codec string `json:"codec"`
+ } `json:"samples"`
+ Options struct {
+ ScopeCheck bool `json:"scope_check"`
+ SideEffects string `json:"side_effects"`
+ LLMPolicy string `json:"llm_policy"`
+ } `json:"options"`
+}
+
+func (module *PipeModule) postDryRunHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ var body dryRunRequest
+ if err := module.DecodeJSON(req, &body); err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ samples := make([]dryrun.Sample, 0, len(body.Samples))
+ for _, s := range body.Samples {
+ samples = append(samples, dryrun.Sample{Raw: []byte(s.Raw), Codec: s.Codec})
+ }
+ report, err := dryrun.Run(body.Processor, samples, dryrun.Options{
+ ScopeCheck: body.Options.ScopeCheck,
+ SideEffects: body.Options.SideEffects,
+ LLMPolicy: body.Options.LLMPolicy,
+ })
+ if err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ module.WriteJSON(w, report, http.StatusOK)
+}
+
+// getLanesHandler — GET /pipeline/lanes
+// Live snapshot of every lane: depth/rates/drops/spills/blocked_waits —
+// the in-process counterpart of queue depth observability.
+func (module *PipeModule) getLanesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ module.WriteJSON(w, util.MapStr{"lanes": lane.List()}, 200)
+}
+
+// getCodecsHandler — GET /pipeline/codecs
+// Discovery view of the record codec registry (name / namespace / version),
+// feeding for_each's codec picker and the canvas.
+func (module *PipeModule) getCodecsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ module.WriteJSON(w, util.MapStr{"codecs": pipeline.ListRecordCodecs()}, 200)
+}
diff --git a/modules/pipeline/rollout.go b/modules/pipeline/rollout.go
new file mode 100644
index 000000000..e9e691286
--- /dev/null
+++ b/modules/pipeline/rollout.go
@@ -0,0 +1,304 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "fmt"
+ "net/http"
+ "sort"
+
+ log "github.com/cihub/seelog"
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/util"
+)
+
+// Staged rollout (#125, SPEC §8.2): a deployment with a rollout spec does
+// not push new content to every target at once — the reconciler writes the
+// canary wave first, waits until those instances REPORT running the staged
+// content (digest match, see DeployedContentDigest), then advances in
+// batches. The _rollout endpoint overrides the machine: force the next
+// batch now, or roll every target back to the last converged content.
+
+// RolloutState is the persisted progress of a staged delivery. Lives on
+// the deployment; the reconciler advances it, the endpoints override it.
+type RolloutState struct {
+ // Active marks a rollout in progress (content staged, waves pending).
+ Active bool `json:"active,omitempty" elastic_mapping:"-"`
+ // Digest is the content digest being staged (what the waves converge to).
+ Digest string `json:"digest,omitempty" elastic_mapping:"digest:{type:keyword}"`
+ // Rolled lists the target instance IDs already on the staged content.
+ Rolled []string `json:"rolled,omitempty" elastic_mapping:"rolled:{type:keyword}"`
+ // RollbackContent is the last converged rendered config (the fallback
+ // the rollback action rewrites every target to). Digest-only would be
+ // unactionable — the bytes are what gets written back.
+ RollbackContent string `json:"rollback_content,omitempty" elastic_mapping:"-"`
+}
+
+// rolloutSpecActive reports whether the spec asks for staged delivery at
+// all (canary or batch > 0).
+func rolloutSpecActive(spec RolloutSpec) bool {
+ return spec.Canary > 0 || spec.Batch > 0
+}
+
+// rolloutGateOwner picks the deployment whose rollout spec stages this
+// pipeline's content: exactly one rollout bearer AND one uniform render
+// spec across the merged targets. Two bearers, or a merge that produces
+// diverging per-instance renders, have no single digest to stage — they
+// converge ungated (logged once per pass by the caller's Debugf absence:
+// the fleet behaves exactly as before the rollout feature existed).
+func rolloutGateOwner(deps []*PipelineDeployment, desired map[string]renderSpec) *PipelineDeployment {
+ var owner *PipelineDeployment
+ for _, dep := range deps {
+ if rolloutSpecActive(dep.Placement.Rollout) {
+ if owner != nil {
+ return nil
+ }
+ owner = dep
+ }
+ }
+ if owner == nil {
+ return nil
+ }
+ var first renderSpec
+ for _, spec := range desired {
+ first = spec
+ break
+ }
+ for _, spec := range desired {
+ if spec != first {
+ return nil
+ }
+ }
+ return owner
+}
+
+// rolloutWaveSizes returns (first, step): the canary wave size and the
+// follow-up batch size. Only canary set → step = canary (keep the same
+// chunk); only batch set → the first wave IS a batch.
+func rolloutWaveSizes(spec RolloutSpec) (int, int) {
+ first, step := spec.Canary, spec.Batch
+ if first <= 0 {
+ first = spec.Batch
+ }
+ if step <= 0 {
+ step = first
+ }
+ return first, step
+}
+
+// rolloutPlan is the pure gate the reconciler consults: given the rollout
+// spec, the persisted state, the (deterministically ordered) target IDs,
+// the digest of the content being distributed and a readiness callback,
+// it returns which targets may receive the content NOW plus the next
+// persisted state.
+//
+// - spec inactive → gate open, state untouched
+// - content changed → new rollout: canary wave only, the
+// previous converged content becomes the
+// rollback target
+// - same content, active → advance one batch when EVERY rolled
+// target is ready; hold otherwise
+// - waves cover all targets → rollout completes (Active=false)
+//
+// ready is only consulted for instances in the current wave. force skips
+// the readiness check (the manual advance action).
+func rolloutPlan(spec RolloutSpec, st RolloutState, targets []string, digest, prevContent string, ready func(id string) bool, force bool) (allowed map[string]bool, next RolloutState, changed bool) {
+ allowed = map[string]bool{}
+ next = st
+
+ if !rolloutSpecActive(spec) {
+ for _, id := range targets {
+ allowed[id] = true
+ }
+ return allowed, next, false
+ }
+
+ sorted := append([]string(nil), targets...)
+ sort.Strings(sorted)
+
+ // drop rolled IDs that fell out of scope (deployment re-targeted)
+ rolled := make([]string, 0, len(st.Rolled))
+ inTargets := map[string]bool{}
+ for _, id := range sorted {
+ inTargets[id] = true
+ }
+ for _, id := range st.Rolled {
+ if inTargets[id] {
+ rolled = append(rolled, id)
+ }
+ }
+ pruned := len(rolled) != len(st.Rolled)
+
+ if !st.Active || st.Digest != digest {
+ // new staged rollout (or first ever): canary wave
+ first, _ := rolloutWaveSizes(spec)
+ if first > len(sorted) {
+ first = len(sorted)
+ }
+ rolled = append([]string(nil), sorted[:first]...)
+ next = RolloutState{
+ Active: true,
+ Digest: digest,
+ Rolled: rolled,
+ RollbackContent: st.RollbackContent,
+ }
+ // the previous converged content is only a meaningful fallback
+ // when it differs from what we are staging now
+ if prevContent != "" {
+ next.RollbackContent = prevContent
+ }
+ changed = true
+ } else if force || rolloutWaveReady(rolled, ready) {
+ _, step := rolloutWaveSizes(spec)
+ nextRolled := append([]string(nil), rolled...)
+ for _, id := range sorted {
+ if len(nextRolled)-len(rolled) >= step {
+ break
+ }
+ if !containsString(nextRolled, id) {
+ nextRolled = append(nextRolled, id)
+ }
+ }
+ if len(nextRolled) != len(rolled) {
+ rolled = nextRolled
+ changed = true
+ }
+ }
+
+ // completion: every target carries the staged content
+ if len(rolled) >= len(sorted) {
+ next = RolloutState{
+ Active: false,
+ Digest: digest,
+ Rolled: rolled,
+ RollbackContent: next.RollbackContent,
+ }
+ if st.Active {
+ changed = true
+ }
+ } else {
+ next.Active = true
+ next.Digest = digest
+ next.Rolled = rolled
+ }
+ if pruned && !changed {
+ changed = true
+ }
+
+ for _, id := range rolled {
+ allowed[id] = true
+ }
+ return allowed, next, changed
+}
+
+// rolloutWaveReady reports whether every instance of the current wave
+// confirms running the staged content. An instance that never reported
+// blocks the wave — conservative on purpose: a canary that went dark
+// must stop the rollout, not let it sweep the fleet (the manual advance
+// endpoint is the operator's override).
+func rolloutWaveReady(rolled []string, ready func(id string) bool) bool {
+ for _, id := range rolled {
+ if !ready(id) {
+ return false
+ }
+ }
+ return true
+}
+
+func containsString(list []string, s string) bool {
+ for _, v := range list {
+ if v == s {
+ return true
+ }
+ }
+ return false
+}
+
+// rolloutActionHandler — POST /pipeline/deployments/:id/_rollout
+// Body: {"action":"advance"|"rollback"}.
+// advance → force the next batch immediately (readiness bypass)
+// rollback → rewrite EVERY target to the last converged content at once
+func (h *PipeModule) rolloutActionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ var body struct {
+ Action string `json:"action"`
+ }
+ if err := h.DecodeJSON(req, &body); err != nil {
+ h.Error400(w, err.Error())
+ return
+ }
+
+ h.rolloutMu.Lock()
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ dep := PipelineDeployment{}
+ dep.ID = id
+ exists, err := orm.GetV2(octx, &dep)
+ if err != nil || !exists {
+ h.rolloutMu.Unlock()
+ h.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+
+ switch body.Action {
+ case "advance":
+ if !dep.State.Active {
+ h.rolloutMu.Unlock()
+ h.WriteError(w, "no rollout in progress (state inactive)", http.StatusConflict)
+ return
+ }
+ // force-advance one batch: readiness always true
+ _, planned, _ := rolloutPlan(dep.Placement.Rollout, dep.State,
+ deploymentTargetIDs(&dep, h.loadRegisteredInstances(octx)),
+ dep.State.Digest, "", func(string) bool { return true }, true)
+ dep.State = planned
+ if err := orm.Save(octx, &dep); err != nil {
+ h.rolloutMu.Unlock()
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ h.rolloutMu.Unlock()
+ h.reconcileOnce()
+ h.WriteJSON(w, util.MapStr{"result": "advanced", "rollout": dep.State}, http.StatusOK)
+
+ case "rollback":
+ if dep.State.RollbackContent == "" {
+ h.rolloutMu.Unlock()
+ h.WriteError(w, "no previous converged content recorded for this deployment", http.StatusConflict)
+ return
+ }
+ if err := h.rollbackDeploymentContent(octx, &dep); err != nil {
+ h.rolloutMu.Unlock()
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ // The rollback is a BRAKE, not a desired-state change: the pipeline
+ // object still carries the new content, so the next reconciler pass
+ // starts a fresh CANARY-staged rollout toward it — visible, gradual,
+ // stoppable. Making a rollback permanent means rolling the pipeline
+ // version back too (POST /pipelines/:id/versions/:v/rollback).
+ dep.State = RolloutState{
+ Active: false,
+ Digest: deployedContentDigestOf([]byte(dep.State.RollbackContent)),
+ RollbackContent: "", // one level of fallback; deeper = pipeline version history
+ }
+ if err := orm.Save(octx, &dep); err != nil {
+ h.rolloutMu.Unlock()
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ h.rolloutMu.Unlock()
+ log.Infof("deployment %s rolled back: every target rewritten to the last converged content", dep.Name)
+ h.WriteJSON(w, util.MapStr{
+ "result": "rolled-back",
+ "rollout": dep.State,
+ "note": "desired state still carries the new content — the next reconcile re-stages it from canary; roll the pipeline version back to make this permanent",
+ }, http.StatusOK)
+
+ default:
+ h.rolloutMu.Unlock()
+ h.Error400(w, fmt.Sprintf("unknown action %q (advance|rollback)", body.Action))
+ }
+}
diff --git a/modules/pipeline/rollout_test.go b/modules/pipeline/rollout_test.go
new file mode 100644
index 000000000..df1cba1ef
--- /dev/null
+++ b/modules/pipeline/rollout_test.go
@@ -0,0 +1,339 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "fmt"
+ "testing"
+
+ "infini.sh/framework/core/model"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+ "infini.sh/framework/modules/configs/server"
+)
+
+// ── rolloutPlan (the pure gate) ──────────────────────────────────────────
+
+func TestRolloutPlanCanaryThenBatches(t *testing.T) {
+ spec := RolloutSpec{Canary: 1, Batch: 2}
+ targets := []string{"i1", "i2", "i3", "i4", "i5"}
+ digest := "d-new"
+
+ // 1. first pass: canary only, rollback content captured
+ allowed, st, changed := rolloutPlan(spec, RolloutState{}, targets, digest, "old-content", nil, false)
+ if !changed || !st.Active || len(st.Rolled) != 1 || !allowed["i1"] || allowed["i2"] {
+ t.Fatalf("canary wave wrong: allowed=%v state=%+v changed=%v", allowed, st, changed)
+ }
+ if st.RollbackContent != "old-content" {
+ t.Fatalf("rollback target must capture prev content, got %q", st.RollbackContent)
+ }
+
+ // 2. canary not ready → hold exactly where we are
+ ready := func(id string) bool { return false }
+ allowed, st2, changed := rolloutPlan(spec, st, targets, digest, "", ready, false)
+ if changed || !st2.Active || len(st2.Rolled) != 1 || len(allowed) != 1 {
+ t.Fatalf("unready wave must hold: allowed=%v state=%+v changed=%v", allowed, st2, changed)
+ }
+
+ // 3. canary ready → one batch of 2 more
+ ready = func(id string) bool { return true }
+ _, st3, changed := rolloutPlan(spec, st, targets, digest, "", ready, false)
+ if !changed || len(st3.Rolled) != 3 || !st3.Active {
+ t.Fatalf("ready wave must advance by batch: %+v changed=%v", st3, changed)
+ }
+
+ // 4. last batch covers the rest → rollout completes
+ _, st4, _ := rolloutPlan(spec, st3, targets, digest, "", ready, false)
+ if st4.Active || len(st4.Rolled) != 5 {
+ t.Fatalf("final wave must complete the rollout: %+v", st4)
+ }
+ if st4.RollbackContent != "old-content" {
+ t.Fatalf("completed rollout must keep the rollback target, got %q", st4.RollbackContent)
+ }
+}
+
+func TestRolloutPlanForceAdvancesWithoutReadiness(t *testing.T) {
+ spec := RolloutSpec{Canary: 1, Batch: 1}
+ targets := []string{"a", "b"}
+ _, st, _ := rolloutPlan(spec, RolloutState{}, targets, "d", "", nil, false)
+ if len(st.Rolled) != 1 {
+ t.Fatalf("canary first: %+v", st)
+ }
+ // readiness says no, force says go — manual advance wins
+ _, forced, _ := rolloutPlan(spec, st, targets, "d", "", func(string) bool { return false }, true)
+ if len(forced.Rolled) != 2 || forced.Active {
+ t.Fatalf("forced advance must finish the rollout: %+v", forced)
+ }
+}
+
+func TestRolloutPlanUnreportedCanaryBlocks(t *testing.T) {
+ spec := RolloutSpec{Canary: 1, Batch: 1}
+ targets := []string{"a", "b"}
+ _, st, _ := rolloutPlan(spec, RolloutState{}, targets, "d", "", nil, false)
+ // readiness callback returning false = canary never confirmed
+ _, held, changed := rolloutPlan(spec, st, targets, "d", "", func(string) bool { return false }, false)
+ if changed || len(held.Rolled) != 1 {
+ t.Fatalf("a dark canary must hold the rollout: %+v changed=%v", held, changed)
+ }
+}
+
+func TestRolloutPlanPrunesOutOfScope(t *testing.T) {
+ spec := RolloutSpec{Canary: 2, Batch: 2}
+ st := RolloutState{Active: true, Digest: "d", Rolled: []string{"gone", "keep"}}
+ allowed, next, _ := rolloutPlan(spec, st, []string{"keep", "other"}, "d", "", func(string) bool { return true }, false)
+ if allowed["gone"] {
+ t.Fatal("out-of-scope instance must not stay in the wave")
+ }
+ if len(next.Rolled) != 2 || !containsString(next.Rolled, "keep") || !containsString(next.Rolled, "other") {
+ t.Fatalf("pruned wave must re-batch from the survivors: %+v", next)
+ }
+}
+
+func TestRolloutPlanInactiveSpecOpenGate(t *testing.T) {
+ allowed, _, changed := rolloutPlan(RolloutSpec{}, RolloutState{}, []string{"a", "b"}, "d", "", nil, false)
+ if len(allowed) != 2 {
+ t.Fatalf("no rollout spec = ungated: %v", allowed)
+ }
+ if changed {
+ t.Fatal("ungated pass must not report a state change")
+ }
+}
+
+// ── digest parity: control-plane render vs instance-side reconstruction ──
+
+func TestDeployedContentDigestParity(t *testing.T) {
+ cfg := &pipeline.PipelineConfigV2{}
+ cfg.Name = "parity-check"
+ cfg.Processors = []map[string]interface{}{
+ {"test_rollout_echo": map[string]interface{}{"message": "hi"}},
+ {"test_rollout_echo": map[string]interface{}{"message": "bye"}},
+ }
+
+ spec := renderSpec{enabled: true, singleton: false}
+ rendered := renderDeployedPipeline(cfg, spec)
+ controlSide := deployedContentDigestOf(rendered)
+
+ // instance side: the config an agent actually loads from the rendered
+ // file, re-rendered through the same canonical shape
+ var wrapper struct {
+ Pipeline []struct {
+ Name string `json:"name"`
+ Enabled *bool `json:"enabled"`
+ AutoStart bool `json:"auto_start"`
+ KeepRunning bool `json:"keep_running"`
+ Singleton bool `json:"singleton"`
+ Processors []map[string]interface{} `json:"processor"`
+ } `json:"pipeline"`
+ }
+ if err := util.FromJSONBytes(rendered, &wrapper); err != nil || len(wrapper.Pipeline) != 1 {
+ t.Fatalf("rendered config must parse: %v", err)
+ }
+ loaded := pipeline.PipelineConfigV2{
+ Name: wrapper.Pipeline[0].Name,
+ Enabled: wrapper.Pipeline[0].Enabled,
+ AutoStart: wrapper.Pipeline[0].AutoStart,
+ KeepRunning: wrapper.Pipeline[0].KeepRunning,
+ Singleton: wrapper.Pipeline[0].Singleton,
+ Processors: wrapper.Pipeline[0].Processors,
+ }
+ instanceSide := deployedContentDigest(deployedPipelineMap(&loaded, renderSpec{
+ enabled: isPipelineEnabled(loaded.Enabled),
+ singleton: loaded.Singleton,
+ }))
+
+ if controlSide != instanceSide || controlSide == "" {
+ t.Fatalf("digest mismatch: control=%s instance=%s", controlSide, instanceSide)
+ }
+}
+
+// ── staged reconcile through the real ORM harness ────────────────────────
+
+// reportInstance overwrites an instance's runtime report (what _sync
+// would upsert).
+func reportInstance(t *testing.T, id string, pipelines string) {
+ t.Helper()
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &model.Instance{})
+ inst := model.Instance{}
+ inst.ID = id
+ found, err := orm.GetV2(octx, &inst)
+ if err != nil || !found {
+ t.Fatalf("instance %s missing: %v", id, err)
+ }
+ inst.RuntimePipelines = pipelines
+ if err := orm.Save(octx, &inst); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func loadDeployment(t *testing.T, id string) PipelineDeployment {
+ t.Helper()
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineDeployment{})
+ dep := PipelineDeployment{}
+ dep.ID = id
+ found, err := orm.GetV2(octx, &dep)
+ if err != nil || !found {
+ t.Fatalf("deployment %s missing: %v", id, err)
+ }
+ return dep
+}
+
+func rowInstanceIDs(rows map[string]server.ManagedConfig) []string {
+ ids := make([]string, 0, len(rows))
+ for id := range rows {
+ ids = append(ids, id)
+ }
+ return ids
+}
+
+func TestStagedReconcileCanaryBatchComplete(t *testing.T) {
+ env := newDeployTestEnv(t)
+ dep := &PipelineDeployment{
+ PipelineID: env.pid,
+ Enabled: true,
+ Placement: PlacementSpec{
+ Strategy: "all",
+ Instances: []string{env.instA, env.instB, env.instC},
+ },
+ }
+ dep.ID = "d1" // shared upsert id: exactly ONE deployment row exists per test run
+ dep.Name = "rollout-test"
+ env.saveDeployment(dep)
+
+ // phase 0: converge WITHOUT a rollout spec — every target on content v1
+ env.reconcile()
+ rows := env.rows()
+ if len(rows) != 3 {
+ t.Fatalf("ungated reconcile must converge all 3 targets, got %d", len(rows))
+ }
+ var v1Content string
+ for _, row := range rows {
+ v1Content = row.Content
+ break
+ }
+
+ // change the pipeline (content v2) AND arm staged delivery
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &pipeline.PipelineConfigV2{})
+ cfg := pipeline.PipelineConfigV2{}
+ cfg.ID = env.pid
+ if found, err := orm.GetV2(octx, &cfg); err != nil || !found {
+ t.Fatalf("pipeline missing: %v", err)
+ }
+ cfg.Processors = append(cfg.Processors, map[string]interface{}{
+ "validate_test_echo": map[string]interface{}{"message": "v2-marker"},
+ })
+ if err := orm.Save(octx, &cfg); err != nil {
+ t.Fatal(err)
+ }
+ armed := loadDeployment(t, dep.ID)
+ armed.Placement.Rollout = RolloutSpec{Canary: 1, Batch: 1}
+ env.saveDeployment(&armed)
+
+ // pass 1: canary only — exactly one row on the new content
+ env.reconcile()
+ rows = env.rows()
+ if len(rows) != 3 {
+ t.Fatalf("staging never drops converged rows, got %d", len(rows))
+ }
+ var canaryID string
+ onV2 := 0
+ for id, row := range rows {
+ if row.Content != v1Content {
+ canaryID = id
+ onV2++
+ }
+ }
+ if onV2 != 1 {
+ t.Fatalf("canary pass must update exactly 1 row to v2, got %d", onV2)
+ }
+ digest := deployedContentDigestOf([]byte(rows[canaryID].Content))
+
+ // pass 2: canary has not reported → nothing else moves
+ env.reconcile()
+ onV2 = 0
+ for _, row := range env.rows() {
+ if row.Content != v1Content {
+ onV2++
+ }
+ }
+ if onV2 != 1 {
+ t.Fatalf("unreported canary must hold the rollout, onV2=%d", onV2)
+ }
+
+ // canary reports running the pushed content → next batch opens
+ reportInstance(t, canaryID, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":%q}}`, env.pipeName, digest))
+ env.reconcile()
+ onV2 = 0
+ for _, row := range env.rows() {
+ if row.Content != v1Content {
+ onV2++
+ }
+ }
+ if onV2 != 2 {
+ t.Fatalf("ready canary must open the next batch, onV2=%d", onV2)
+ }
+
+ // every v2 target reports → completion covers the fleet
+ for id, row := range env.rows() {
+ if row.Content != v1Content {
+ reportInstance(t, id, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":%q}}`, env.pipeName, digest))
+ }
+ }
+ env.reconcile()
+ onV2 = 0
+ for _, row := range env.rows() {
+ if row.Content != v1Content {
+ onV2++
+ }
+ }
+ if onV2 != 3 {
+ t.Fatalf("completed rollout must cover all targets, onV2=%d", onV2)
+ }
+
+ // terminal state persisted: inactive, v1 recorded as the rollback target
+ st := loadDeployment(t, dep.ID)
+ if st.State.Active {
+ t.Fatalf("rollout must be terminal: %+v", st.State)
+ }
+ if st.State.RollbackContent != v1Content {
+ t.Fatalf("rollback target must be the pre-rollout content (v1), got %d bytes", len(st.State.RollbackContent))
+ }
+}
+
+func TestStagedReconcileOldDigestDoesNotAdvance(t *testing.T) {
+ env := newDeployTestEnv(t)
+ dep := &PipelineDeployment{
+ PipelineID: env.pid,
+ Enabled: true,
+ Placement: PlacementSpec{
+ Strategy: "all",
+ Instances: []string{env.instA, env.instB},
+ Rollout: RolloutSpec{Canary: 1, Batch: 1},
+ },
+ }
+ dep.ID = "d1" // shared upsert id (see rollout test above)
+ dep.Name = "rollout-stale-digest"
+ env.saveDeployment(dep)
+
+ env.reconcile()
+ rows := env.rows()
+ if len(rows) != 1 {
+ t.Fatalf("canary pass must write exactly 1 row, got %d", len(rows))
+ }
+ var canaryID string
+ for id := range rows {
+ canaryID = id
+ }
+ // canary reports STARTED but with the OLD content's digest — the very
+ // lie the digest check exists to catch
+ reportInstance(t, canaryID, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":"stale"}}`, env.pipeName))
+ env.reconcile()
+ if got := len(env.rows()); got != 1 {
+ t.Fatalf("a STARTED-but-stale canary must hold the rollout, rows=%d", got)
+ }
+}
diff --git a/modules/pipeline/scope_strict_test.go b/modules/pipeline/scope_strict_test.go
new file mode 100644
index 000000000..65b50c31e
--- /dev/null
+++ b/modules/pipeline/scope_strict_test.go
@@ -0,0 +1,122 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infinilabs.com */
+
+package pipeline
+
+import (
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+// batchScopedFixture declares a ScopeBatch contract — legal at a chain
+// root, illegal inside a record sub-chain. It exists to prove strict
+// compile reaches through for_each into the nested chain.
+type batchScopedFixture struct{}
+
+func (batchScopedFixture) Name() string { return "test_batch_scoped" }
+
+func (batchScopedFixture) Process(*pipeline.Context) error { return nil }
+
+func (batchScopedFixture) RecordContract() pipeline.RecordContract {
+ return pipeline.RecordContract{Scope: pipeline.ScopeBatch}
+}
+
+func init() {
+ // for_each registers during module setup, which these unit tests skip
+ pipeline.RegisterProcessorPlugin("for_each", NewForEachProcessor)
+ pipeline.RegisterProcessorPlugin("test_batch_scoped",
+ func(*config.Config) (pipeline.Processor, error) { return batchScopedFixture{}, nil })
+ pipeline.RegisterProcessorPlugin("test_registry_scoped",
+ func(*config.Config) (pipeline.Processor, error) { return registryContractFixture{}, nil })
+ // registry-declared contract, instance implements nothing — the path
+ // every product processor takes (RegisterProcessorContract)
+ pipeline.RegisterProcessorContract("test_registry_scoped",
+ pipeline.RecordContract{Scope: pipeline.ScopeBatch})
+}
+
+// registryContractFixture carries NO ContractProvider method; its
+// contract lives only in the registry.
+type registryContractFixture struct{}
+
+func (registryContractFixture) Name() string { return "test_registry_scoped" }
+
+func (registryContractFixture) Process(*pipeline.Context) error { return nil }
+
+func forEachWithBatchScopedStep(t *testing.T) *config.Config {
+ t.Helper()
+ c, err := config.NewConfigFrom(util.MapStr{
+ "for_each": util.MapStr{
+ "processor": []interface{}{util.MapStr{"test_batch_scoped": util.MapStr{}}},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+func TestForEach_SubChainStrictnessThreaded(t *testing.T) {
+ step := forEachWithBatchScopedStep(t)
+
+ // non-strict: the nested placement mismatch only warns, the chain
+ // compiles — this is the legacy behavior the default flip keeps
+ if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil {
+ t.Fatalf("non-strict compile should accept the nested mismatch: %v", err)
+ }
+
+ // strict: the violation inside the sub-chain must surface as a
+ // compile error (threaded through ApplyScope)
+ _, err := pipeline.NewPipelineWithScope([]*config.Config{step},
+ pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false)
+ if err == nil || !strings.Contains(err.Error(), "requires batch scope") {
+ t.Fatalf("strict compile must reject the nested violation, got: %v", err)
+ }
+}
+
+func TestValidatePipelineConfig_NestedContractWarns(t *testing.T) {
+ // the validation gate feeds the audit tool: without scope threading
+ // a nested mismatch never reached its warnings list
+ cfg := &pipeline.PipelineConfigV2{
+ Name: "strict-threading-probe",
+ Processors: []map[string]interface{}{
+ {"for_each": util.MapStr{
+ "processor": []interface{}{util.MapStr{"test_batch_scoped": util.MapStr{}}},
+ }},
+ },
+ }
+ errs, warns := ValidatePipelineConfig(cfg)
+ if len(errs) > 0 {
+ t.Fatalf("unexpected errors: %v", errs)
+ }
+ if len(warns) == 0 || !strings.Contains(warns[0].Message, "requires batch scope") {
+ t.Fatalf("nested contract mismatch must warn, got warns=%v", warns)
+ }
+}
+
+func TestValidatePlacement_RegistryDeclaredContract(t *testing.T) {
+ // registry-declared contracts (the product-processor path) must drive
+ // placement too — the instance-only assertion silently passed them
+ c, err := config.NewConfigFrom(util.MapStr{
+ "for_each": util.MapStr{
+ "processor": []interface{}{util.MapStr{"test_registry_scoped": util.MapStr{}}},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ step := []*config.Config{c}
+
+ if _, err := pipeline.NewPipeline(step); err != nil {
+ t.Fatalf("non-strict compile should accept the mismatch: %v", err)
+ }
+ _, err = pipeline.NewPipelineWithScope(step,
+ pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false)
+ if err == nil || !strings.Contains(err.Error(), "requires batch scope") {
+ t.Fatalf("strict compile must honor registry-declared contracts, got: %v", err)
+ }
+}
diff --git a/modules/pipeline/tap.go b/modules/pipeline/tap.go
new file mode 100644
index 000000000..b870d7ae5
--- /dev/null
+++ b/modules/pipeline/tap.go
@@ -0,0 +1,131 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+ "math/rand"
+
+ log "github.com/cihub/seelog"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// TapConfig declares the tap step: copy a sampled fraction of records to
+// a side target; the main flow is never affected (in-flight copy, not a
+// divert). Target is one-of queue/lane — the sanctioned side-channel for
+// debug indices, shadow runs and observation sampling.
+type TapConfig struct {
+ SampleRatio float64 `config:"sample_ratio"` // 0~1, 1.0 = all
+ Queue *queue.QueueConfig `config:"queue"`
+ Lane string `config:"lane"`
+}
+
+type TapProcessor struct {
+ cfg TapConfig
+ outQ *queue.QueueConfig
+}
+
+const tapName = "tap"
+
+func init() {
+ pipeline.RegisterProcessorPluginWithConfigMetadata(tapName, NewTapProcessor, TapConfig{})
+ pipeline.RegisterProcessorContract(tapName, pipeline.RecordContract{Scope: pipeline.ScopeRecord})
+}
+
+func NewTapProcessor(c *config.Config) (pipeline.Processor, error) {
+ cfg := TapConfig{SampleRatio: 1.0}
+ if err := c.Unpack(&cfg); err != nil {
+ return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", tapName, err)
+ }
+ targets := 0
+ if cfg.Queue != nil {
+ targets++
+ }
+ if cfg.Lane != "" {
+ targets++
+ }
+ if targets != 1 {
+ return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", tapName, targets)
+ }
+ if cfg.SampleRatio <= 0 || cfg.SampleRatio > 1 {
+ return nil, fmt.Errorf("%s sample_ratio must be within (0, 1]", tapName)
+ }
+ p := &TapProcessor{cfg: cfg}
+ if cfg.Queue != nil {
+ p.outQ = queue.SmartGetOrInitConfig(cfg.Queue)
+ }
+ return p, nil
+}
+
+func (p *TapProcessor) Name() string { return tapName }
+
+// Process copies the current record to the side target by sample ratio;
+// the original keeps flowing regardless of the copy's fate.
+func (p *TapProcessor) Process(ctx *pipeline.Context) error {
+ rec, ok := pipeline.CurrentRecord(ctx)
+ if !ok {
+ return nil
+ }
+ if p.cfg.SampleRatio < 1 && rand.Float64() > p.cfg.SampleRatio {
+ return nil
+ }
+ // deep copy: later main-flow mutations must not leak into the tapped
+ // copy (and vice versa — Fields is shared authority in the original)
+ cp := copyRecord(rec)
+ if sink := pipeline.DryRunCapture(ctx); sink != nil {
+ if b, err := GetRecordCodec(pipeline.CodecHint(cp)).Encode(cp); err == nil {
+ kind, target := "lane", p.cfg.Lane
+ if p.outQ != nil {
+ kind, target = "queue", p.outQ.Name
+ }
+ *sink = append(*sink, pipeline.DryRunWrite{Kind: kind, Target: target, Records: 1, Preview: string(b)})
+ }
+ return nil
+ }
+ if p.outQ != nil {
+ b, err := GetRecordCodec(pipeline.CodecHint(cp)).Encode(cp)
+ if err != nil {
+ log.Warnf("tap: encode failed: %v", err)
+ return nil
+ }
+ if err := queue.Push(p.outQ, b); err != nil {
+ log.Warnf("tap: push to queue %q failed: %v", p.outQ.Name, err)
+ }
+ return nil
+ }
+ l := lane.Get(p.cfg.Lane)
+ if l == nil {
+ l = lane.GetOrCreate(lane.LaneConfig{Name: p.cfg.Lane})
+ }
+ if err := l.Emit(cp); err != nil {
+ log.Warnf("tap: emit to lane %q failed: %v", p.cfg.Lane, err)
+ }
+ return nil
+}
+
+func copyRecord(rec *event.Event) *event.Event {
+ b, err := util.ToJSONBytes(map[string]interface{}{"fields": rec.Fields, "labels": rec.Metadata.Labels})
+ if err != nil {
+ return rec // fallback: share (best effort)
+ }
+ wrap := map[string]interface{}{}
+ if err := util.FromJSONBytes(b, &wrap); err != nil {
+ return rec
+ }
+ cp := &event.Event{Timestamp: rec.Timestamp}
+ cp.Fields, _ = wrap["fields"].(map[string]interface{})
+ if lbl, ok := wrap["labels"].(map[string]interface{}); ok {
+ cp.Metadata.Labels = lbl
+ }
+ if cp.Fields == nil {
+ cp.Fields = util.MapStr{}
+ }
+ return cp
+}
diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go
index eb7bc6662..79f7cc93f 100644
--- a/modules/pipeline/tasks.go
+++ b/modules/pipeline/tasks.go
@@ -24,11 +24,16 @@
package pipeline
import (
+ "encoding/json"
"net/http"
+ "strconv"
+ "time"
log "github.com/cihub/seelog"
httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/lane"
"infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
"infini.sh/framework/core/util"
)
@@ -147,6 +152,12 @@ func (module *PipeModule) createPipelineTaskHandler(w http.ResponseWriter, req *
if obj.ID == "" {
obj.ID = obj.Name
}
+ // compile gate BEFORE creating the task — createPipelineTask skips
+ // disabled pipelines entirely, which would otherwise defer the
+ // discovery of a broken chain to the moment it gets enabled
+ if !module.validateAndReject(w, &obj) {
+ return
+ }
err = module.createPipelineTask(obj, true)
if err != nil {
module.WriteError(w, err.Error(), http.StatusBadRequest)
@@ -161,6 +172,13 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req *
id := ps.ByName("id")
_, exists := module.contexts.Load(id)
if exists {
+ // Cancel before deleting the registry entry: the pipeline goroutine
+ // is blocked inside Process (consumer worker loops on IsCanceled,
+ // lane processors on ctx.Done) — clearing the map alone would leave
+ // the goroutine alive forever and the task registry stuck at
+ // STARTED (a zombie "running" task). Same as the auto_reload path:
+ // cancel and wait for release (≤1min), force the delete on timeout.
+ module.stopAndWaitForRelease([]string{id}, time.Minute)
module.deleteTask(id)
module.WriteAckOKJSON(w)
} else {
@@ -171,8 +189,194 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req *
}
// curl -XPOST http://localhost:2900/pipeline/task/echo-test/_start
+// stagesPipelineTaskHandler — GET /pipeline/task/:id/stages
+// Standard per-stage metrics: processor invocations/errors/latency plus
+// live lane carriers (depth-derived).
+func (module *PipeModule) stagesPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ module.WriteJSON(w, util.MapStr{"stages": stageStats(id)}, http.StatusOK)
+}
+
+// flightPipelineTaskHandler — GET /pipeline/task/:id/flight
+// Sampled execution trace of a pipeline: ?trace_id= for one record's
+// journey, ?errors=1 for recent failures, &limit= (default 50, newest
+// first). Memory-only (ring); persistence rides the metrics exporter.
+func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ q := req.URL.Query()
+ limit := 50
+ if v := q.Get("limit"); v != "" {
+ if n, err := strconv.Atoi(v); err == nil {
+ limit = n
+ }
+ }
+ events := flightQuery(id, q.Get("trace_id"), q.Get("errors") == "1", limit)
+ module.WriteJSON(w, util.MapStr{"events": events}, http.StatusOK)
+}
+
+// requestedGeneration parses the optional ?generation= control guard.
+// Returns (0, false) when absent or malformed — legacy unguarded behavior.
+func requestedGeneration(req *http.Request) (int, bool) {
+ g := req.URL.Query().Get("generation")
+ if g == "" {
+ return 0, false
+ }
+ n, err := strconv.Atoi(g)
+ if err != nil || n <= 0 {
+ return 0, false
+ }
+ return n, true
+}
+
+// checkGenerationGuard enforces the fleet control contract (#120): an
+// instruction stamped with a generation applies only when the local task
+// runs exactly that config revision — an instance still on an older
+// generation (hot reload not landed yet) must not act on instructions
+// issued for the new one, and vice versa. Writes a 409 carrying both
+// generations on mismatch; absent/malformed param keeps today's
+// unconditional behavior.
+func (module *PipeModule) checkGenerationGuard(w http.ResponseWriter, req *http.Request, id string) bool {
+ n, ok := requestedGeneration(req)
+ if !ok {
+ return true
+ }
+ cur := module.generationOf(id)
+ if cur != n {
+ module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{
+ "error": "generation mismatch, instruction skipped",
+ "current_generation": cur,
+ "instruction_generation": n,
+ })
+ return false
+ }
+ return true
+}
+
+// pausePipelineTaskHandler — POST /pipeline/task/:id/_pause
+func (module *PipeModule) pausePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ if !module.checkGenerationGuard(w, req, id) {
+ return
+ }
+ if !module.pauseTask(id) {
+ module.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ module.WriteJSON(w, util.MapStr{"paused": true, "id": id}, http.StatusOK)
+}
+
+// resumePipelineTaskHandler — POST /pipeline/task/:id/_resume
+func (module *PipeModule) resumePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ if !module.checkGenerationGuard(w, req, id) {
+ return
+ }
+ if module.cordoned.Load() {
+ module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{"error": "instance is cordoned"})
+ return
+ }
+ if !module.resumeTask(id) {
+ module.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ module.WriteJSON(w, util.MapStr{"resumed": true, "id": id}, http.StatusOK)
+}
+
+// injectPipelineTaskHandler — POST /pipeline/task/:id/_inject
+// Replays a sample into the task's input (queue or lane target).
+type injectRequest struct {
+ Queue string `json:"queue"`
+ Lane string `json:"lane"`
+ Samples []InjectSample `json:"samples"`
+}
+
+// InjectSample accepts raw as either base64 (Go json []byte default) or
+// plain text — callers should not need to pre-encode log lines.
+type InjectSample struct {
+ Raw []byte `json:"raw"`
+ Codec string `json:"codec"`
+}
+
+func (s *InjectSample) UnmarshalJSON(b []byte) error {
+ type alias struct {
+ Raw json.RawMessage `json:"raw"`
+ Codec string `json:"codec"`
+ }
+ var a alias
+ if err := json.Unmarshal(b, &a); err != nil {
+ return err
+ }
+ s.Codec = a.Codec
+ // try standard base64 first ([]byte json encoding); fall back to the
+ // literal string bytes (plain-text payloads)
+ if err := json.Unmarshal(a.Raw, &s.Raw); err != nil {
+ // not base64 — use the raw string content directly
+ var str string
+ if err2 := json.Unmarshal(a.Raw, &str); err2 == nil {
+ s.Raw = []byte(str)
+ } else {
+ s.Raw = a.Raw
+ }
+ }
+ return nil
+}
+
+func (module *PipeModule) injectPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ if _, ok := module.contexts.Load(id); !ok {
+ module.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ var body injectRequest
+ if err := module.DecodeJSON(req, &body); err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ targets := 0
+ if body.Queue != "" {
+ targets++
+ }
+ if body.Lane != "" {
+ targets++
+ }
+ if targets != 1 {
+ module.WriteError(w, "exactly one target (queue or lane) required", http.StatusBadRequest)
+ return
+ }
+ pushed := 0
+ for _, s := range body.Samples {
+ if body.Lane != "" {
+ l := lane.GetOrCreate(lane.LaneConfig{Name: body.Lane})
+ rec, err := pipeline.GetRecordCodec(s.Codec).Decode(s.Raw)
+ if err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if err := l.Emit(rec); err != nil {
+ module.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ pushed++
+ continue
+ }
+ if err := queue.Push(queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: body.Queue}), s.Raw); err != nil {
+ module.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ pushed++
+ }
+ module.WriteJSON(w, util.MapStr{"injected": pushed, "id": id}, http.StatusOK)
+}
+
func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.ByName("id")
+ if !module.checkGenerationGuard(w, req, id) {
+ return
+ }
+ if module.cordoned.Load() {
+ module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{"error": "instance is cordoned"})
+ return
+ }
exists := module.startTask(id)
if exists {
module.WriteAckOKJSON(w)
@@ -186,6 +390,9 @@ func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *h
// curl -XPOST http://localhost:2900/pipeline/task/echo-test/_stop
func (module *PipeModule) stopPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.ByName("id")
+ if !module.checkGenerationGuard(w, req, id) {
+ return
+ }
exists := module.stopTask(id)
if exists {
module.WriteAckOKJSON(w)
@@ -195,3 +402,65 @@ func (module *PipeModule) stopPipelineTaskHandler(w http.ResponseWriter, req *ht
})
}
}
+
+// drainPipelineTaskHandler — POST /pipeline/task/:id/_drain
+// Graceful stop: the consumer finishes the current batch and commits its
+// offset, then the task parks STOPPED (resumable via _start). Honors the
+// ?generation= guard; ?timeout_seconds= bounds the settle wait (default
+// 30, max 600).
+func (module *PipeModule) drainPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ if !module.checkGenerationGuard(w, req, id) {
+ return
+ }
+ if !module.drainTask(id) {
+ module.WriteOpRecordNotFoundJSON(w, id)
+ return
+ }
+ timeout := 30 * time.Second
+ if v := req.URL.Query().Get("timeout_seconds"); v != "" {
+ if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 600 {
+ timeout = time.Duration(n) * time.Second
+ }
+ }
+ state := module.waitTaskSettled(id, timeout)
+ settled := state != pipeline.STARTED && state != pipeline.STARTING && state != pipeline.STOPPING
+ module.WriteJSON(w, util.MapStr{"drained": settled, "id": id, "state": string(state)}, http.StatusOK)
+}
+
+// drainInstanceHandler — POST /pipeline/_drain
+// Graceful-stop every running task on this instance (maintenance path;
+// pair with _cordon so no new work starts while draining). Optional
+// ?timeout_seconds= bounds the shared settle wait (default 60).
+func (module *PipeModule) drainInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ timeout := 60 * time.Second
+ if v := req.URL.Query().Get("timeout_seconds"); v != "" {
+ if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 3600 {
+ timeout = time.Duration(n) * time.Second
+ }
+ }
+ states := module.drainAllTasks(timeout)
+ module.WriteJSON(w, util.MapStr{"drained": states}, http.StatusOK)
+}
+
+// cordonInstanceHandler — POST /pipeline/_cordon
+// Marks the instance cordoned: running tasks drain to completion, but no
+// task starts (auto_start, _start, _resume; hot-reload recreations park
+// STOPPED) until _uncordon.
+func (module *PipeModule) cordonInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ module.cordoned.Store(true)
+ log.Info("pipeline instance cordoned: task starts are rejected until _uncordon")
+ module.WriteJSON(w, util.MapStr{"cordoned": true}, http.StatusOK)
+}
+
+// uncordonInstanceHandler — POST /pipeline/_uncordon
+func (module *PipeModule) uncordonInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ module.cordoned.Store(false)
+ log.Info("pipeline instance uncordoned")
+ module.WriteJSON(w, util.MapStr{"cordoned": false}, http.StatusOK)
+}
+
+// getCordonHandler — GET /pipeline/_cordon
+func (module *PipeModule) getCordonHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ module.WriteJSON(w, util.MapStr{"cordoned": module.cordoned.Load()}, http.StatusOK)
+}
diff --git a/modules/pipeline/validate.go b/modules/pipeline/validate.go
new file mode 100644
index 000000000..63b420061
--- /dev/null
+++ b/modules/pipeline/validate.go
@@ -0,0 +1,154 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+ "net/http"
+
+ log "github.com/cihub/seelog"
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+// ConfigError locates one validation finding in a pipeline config.
+type ConfigError struct {
+ // Stage is the 0-based processor index; -1 means chain/config level.
+ Stage int `json:"stage"`
+ Processor string `json:"processor,omitempty"` // offending processor key ("if" for conditionals)
+ Field string `json:"field,omitempty"`
+ Message string `json:"message"`
+}
+
+// ValidatePipelineConfig compiles the processor chain WITHOUT running it:
+// DSL shape, processor registry existence, parameter unpacking, nested
+// chains (for_each / lane_consumer hosts) and record-contract placement.
+//
+// It is the gate before a config is persisted or deployed — invalid input
+// must be rejected at the control plane, not discovered on the instance
+// after distribution. Compile problems are returned as errors (reject);
+// record-contract mismatches are warnings unless the config opts into
+// StrictContract, matching the runtime compile semantics.
+func ValidatePipelineConfig(cfg *pipeline.PipelineConfigV2) (errs []ConfigError, warns []ConfigError) {
+ if cfg == nil {
+ errs = append(errs, ConfigError{Stage: -1, Message: "config is required"})
+ return
+ }
+ if cfg.Name == "" {
+ errs = append(errs, ConfigError{Stage: -1, Field: "name", Message: "name is required"})
+ }
+ if len(cfg.Processors) == 0 {
+ errs = append(errs, ConfigError{Stage: -1, Field: "processor", Message: "at least one processor is required"})
+ return
+ }
+
+ procCfgs, err := cfg.GetProcessorsConfig()
+ if err != nil {
+ errs = append(errs, ConfigError{Stage: -1, Field: "processor", Message: err.Error()})
+ return
+ }
+
+ // per-stage compile so errors carry the offending step index; a
+ // panicking constructor becomes a validation error instead of
+ // crashing the gate (compile must never take the control plane down)
+ for i, step := range procCfgs {
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ errs = append(errs, ConfigError{
+ Stage: i,
+ Processor: stepKey(step),
+ Message: fmt.Sprintf("constructor panicked: %v", r),
+ })
+ }
+ }()
+ if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil {
+ errs = append(errs, ConfigError{
+ Stage: i,
+ Processor: stepKey(step),
+ Message: err.Error(),
+ })
+ }
+ }()
+ }
+ // whole-chain compile catches issues the standalone steps cannot see
+ if _, err := pipeline.NewPipeline(procCfgs); err != nil && len(errs) == 0 {
+ errs = append(errs, ConfigError{Stage: -1, Message: err.Error()})
+ }
+
+ // contract placement pass: strict compile surfaces mismatches; severity
+ // follows the config's own StrictContract opt-in (default warn-only).
+ // Skipped when the chain already failed to compile — the strict run
+ // would only duplicate those errors, not reveal contract issues.
+ if len(errs) == 0 {
+ if _, err := pipeline.NewPipelineWithScope(procCfgs,
+ pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false); err != nil {
+ ce := ConfigError{Stage: -1, Message: err.Error()}
+ if cfg.StrictContract != nil && *cfg.StrictContract {
+ errs = append(errs, ce)
+ } else {
+ warns = append(warns, ce)
+ }
+ }
+ }
+ return errs, warns
+}
+
+// stepKey names the processor a step config declares ("if" for
+// conditionals, else the single action key).
+func stepKey(step *config.Config) string {
+ fields := step.GetFields()
+ if len(fields) == 0 {
+ return ""
+ }
+ return fields[0]
+}
+
+// validateAndReject writes a structured 400 when the config does not
+// compile; returns true when the request may proceed. The body carries
+// both the standard error wrapper (UIs toast `error.reason`) and the
+// per-stage detail list for richer editors.
+func (module *PipeModule) validateAndReject(w http.ResponseWriter, cfg *pipeline.PipelineConfigV2) bool {
+ errs, warns := ValidatePipelineConfig(cfg)
+ if len(errs) == 0 {
+ return true
+ }
+ summary := "invalid pipeline config: " + errs[0].Message
+ // the registry error lists every valid processor — keep the toast-size
+ // summary short; errors[] carries the full detail
+ if len(summary) > 300 {
+ summary = summary[:300] + "..."
+ }
+ if len(errs) > 1 {
+ summary += fmt.Sprintf(" (+%d more)", len(errs)-1)
+ }
+ module.WriteJSON(w, util.MapStr{
+ "error": util.MapStr{"reason": summary},
+ "status": http.StatusBadRequest,
+ "errors": errs,
+ "warnings": warns,
+ }, http.StatusBadRequest)
+ _ = log.Warnf("rejected invalid pipeline config %q: %d error(s)", cfg.Name, len(errs))
+ return false
+}
+
+// validatePipelineHandler — POST /pipeline/_validate
+// Pre-flight report for editors: compiles the submitted config without
+// persisting anything. Always 200; the `valid` flag carries the verdict.
+func (module *PipeModule) validatePipelineHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
+ var obj pipeline.PipelineConfigV2
+ if err := module.DecodeJSON(req, &obj); err != nil {
+ module.WriteError(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ errs, warns := ValidatePipelineConfig(&obj)
+ module.WriteJSON(w, util.MapStr{
+ "valid": len(errs) == 0,
+ "errors": errs,
+ "warnings": warns,
+ }, http.StatusOK)
+}
diff --git a/modules/pipeline/validate_test.go b/modules/pipeline/validate_test.go
new file mode 100644
index 000000000..f5700936f
--- /dev/null
+++ b/modules/pipeline/validate_test.go
@@ -0,0 +1,173 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ _ "infini.sh/framework/plugins/http"
+ _ "infini.sh/framework/plugins/smtp"
+ "strings"
+ "testing"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/pipeline"
+)
+
+// paramCheckedProc mirrors a real plugin constructor: it rejects configs
+// missing required fields or carrying unknown ones.
+func init() {
+ pipeline.RegisterProcessorPlugin("validate_test_echo", func(cfg *config.Config) (pipeline.Processor, error) {
+ if !cfg.HasField("message") {
+ return nil, errNew("validate_test_echo: `message` is required")
+ }
+ return nopProc{}, nil
+ })
+}
+
+type nopProc struct{}
+
+func (p nopProc) Name() string { return "validate_test_echo" }
+func (p nopProc) Process(*pipeline.Context) error { return nil }
+
+type testError struct{ msg string }
+
+func (e testError) Error() string { return e.msg }
+
+func errNew(msg string) error { return testError{msg} }
+
+func cfgOf(procs ...map[string]interface{}) *pipeline.PipelineConfigV2 {
+ return &pipeline.PipelineConfigV2{Name: "t", Processors: procs}
+}
+
+func TestValidateRejectsUnknownProcessor(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{
+ "no_such_processor_xyz": map[string]interface{}{"x": 1},
+ }))
+ if len(errs) == 0 {
+ t.Fatal("expected rejection for unknown processor")
+ }
+ if errs[0].Processor != "no_such_processor_xyz" {
+ t.Fatalf("expected processor attribution, got %+v", errs[0])
+ }
+ if errs[0].Stage != 0 {
+ t.Fatalf("expected stage 0, got %d", errs[0].Stage)
+ }
+ if !strings.Contains(errs[0].Message, "does not exist") {
+ t.Fatalf("expected registry error, got %q", errs[0].Message)
+ }
+}
+
+func TestValidateRejectsInvalidParameters(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{
+ "validate_test_echo": map[string]interface{}{}, // missing required `message`
+ }))
+ if len(errs) == 0 {
+ t.Fatal("expected rejection for missing required parameter")
+ }
+ if !strings.Contains(errs[0].Message, "`message` is required") {
+ t.Fatalf("expected param error, got %q", errs[0].Message)
+ }
+}
+
+func TestValidateRejectsEmptyChain(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf())
+ if len(errs) == 0 {
+ t.Fatal("expected rejection for empty processor chain")
+ }
+ if errs[0].Field != "processor" {
+ t.Fatalf("expected processor field error, got %+v", errs[0])
+ }
+}
+
+func TestValidateRejectsMissingName(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(&pipeline.PipelineConfigV2{
+ Processors: []map[string]interface{}{
+ {"validate_test_echo": map[string]interface{}{"message": "hi"}},
+ },
+ })
+ if len(errs) == 0 || errs[0].Field != "name" {
+ t.Fatalf("expected name error, got %+v", errs)
+ }
+}
+
+func TestValidateAcceptsValidChain(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(
+ map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "a"}},
+ map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "b"}},
+ ))
+ if len(errs) != 0 {
+ t.Fatalf("valid chain rejected: %+v", errs)
+ }
+}
+
+func TestValidateAttributesStageIndex(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(
+ map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "ok"}},
+ map[string]interface{}{"validate_test_echo": map[string]interface{}{}}, // stage 1 broken
+ ))
+ if len(errs) != 1 {
+ t.Fatalf("expected exactly one error, got %+v", errs)
+ }
+ if errs[0].Stage != 1 {
+ t.Fatalf("expected stage 1 attribution, got %d", errs[0].Stage)
+ }
+}
+
+func TestValidateRejectsMalformedStep(t *testing.T) {
+ // a step with two action keys violates the one-action-per-step DSL rule
+ errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{
+ "validate_test_echo": map[string]interface{}{"message": "x"},
+ "extra_key": map[string]interface{}{"y": 2},
+ }))
+ if len(errs) == 0 {
+ t.Fatal("expected rejection for multi-action step")
+ }
+}
+
+// TestValidateHttpRequiresHosts and TestValidateSmtpRequiresServer pin
+// the required-field checks added by the empty-config audit.
+func TestValidateHttpRequiresHosts(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{
+ "http": map[string]interface{}{"path": "/x"},
+ }))
+ if len(errs) == 0 || !strings.Contains(errs[0].Message, "requires at least one host") {
+ t.Fatalf("expected hosts requirement, got %+v", errs)
+ }
+}
+
+func TestValidateSmtpRequiresServer(t *testing.T) {
+ errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{
+ "smtp": map[string]interface{}{},
+ }))
+ if len(errs) == 0 || !strings.Contains(errs[0].Message, "requires at least one server") {
+ t.Fatalf("expected server requirement, got %+v", errs)
+ }
+}
+
+// TestRuntimeSnapshotReportsFailures pins instance-side failure
+// visibility: a config that fails to compile shows up as FAILED with
+// the reason instead of silently disappearing from the runtime report.
+func TestRuntimeSnapshotReportsFailures(t *testing.T) {
+ m := &PipeModule{}
+ m.recordFailure("ghost-pipeline", errNew("the processor nope does not exist"))
+
+ snap := m.RuntimeSnapshot()
+ got, ok := snap["ghost-pipeline"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected failure entry, got %v", snap)
+ }
+ if got["state"] != "FAILED" {
+ t.Fatalf("expected FAILED, got %v", got["state"])
+ }
+ if got["reason"] != "the processor nope does not exist" {
+ t.Fatalf("expected reason, got %v", got["reason"])
+ }
+
+ // a pipeline that starts clears its stale failure entry
+ m.recordSuccess("ghost-pipeline")
+ snap = m.RuntimeSnapshot()
+ if _, still := snap["ghost-pipeline"]; still {
+ t.Fatal("failure entry should clear after success")
+ }
+}
diff --git a/modules/pipeline/version.go b/modules/pipeline/version.go
new file mode 100644
index 000000000..3609b7fb6
--- /dev/null
+++ b/modules/pipeline/version.go
@@ -0,0 +1,232 @@
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package pipeline
+
+import (
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ log "github.com/cihub/seelog"
+ httprouter "infini.sh/framework/core/api/router"
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/orm"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+// PipelineVersion is the immutable snapshot of one pipeline config at a
+// point in time. Every PUT /pipelines/:id auto-snapshots the previous
+// state before overwriting — no explicit "publish" step; every edit is a
+// version.
+type PipelineVersion struct {
+ orm.ORMObjectBase `json:",inline" elastic_mapping:"-"`
+
+ PipelineID string `json:"pipeline_id" elastic_mapping:"pipeline_id:{type:keyword}"`
+ Version int `json:"version" elastic_mapping:"version:{type:integer}"`
+ // Config is the full PipelineConfigV2 JSON snapshot.
+ Config string `json:"config" elastic_mapping:"config:{type:text}"`
+ Note string `json:"note,omitempty" elastic_mapping:"note:{type:text}"`
+ // UpdatedBy records who made the change (from the request context).
+ UpdatedBy string `json:"updated_by,omitempty" elastic_mapping:"updated_by:{type:keyword}"`
+ CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"`
+}
+
+// maxVersionsPerPipeline bounds the retention (oldest pruned on write).
+const maxVersionsPerPipeline = 50
+
+// saveVersion snapshots the current config before an update lands.
+// Transient pipelines skip versioning — they are ad-hoc executions that
+// produce no durable config worth tracking (saving them wastes storage).
+func (h *PipeModule) saveVersion(octx *orm.Context, current *pipeline.PipelineConfigV2, note string) error {
+ if current.Transient {
+ return nil // transient: no version history
+ }
+ // find the highest existing version number
+ next := h.nextVersion(octx, current.ID) + 1
+ cfgJSON := string(util.MustToJSONBytes(current))
+
+ v := PipelineVersion{
+ PipelineID: current.ID,
+ Version: next,
+ Config: cfgJSON,
+ Note: note,
+ CreatedAt: time.Now().UTC(),
+ }
+ v.ID = fmt.Sprintf("%s-v%d", current.ID, next)
+
+ if err := orm.Save(octx, &v); err != nil {
+ return fmt.Errorf("save version snapshot: %w", err)
+ }
+
+ // prune oldest beyond retention (best effort — failure is a warning)
+ go h.pruneVersions(current.ID)
+ return nil
+}
+
+func (h *PipeModule) nextVersion(octx *orm.Context, pipelineID string) int {
+ orm.WithModel(octx, &PipelineVersion{})
+ q := orm.NewQuery().
+ Filter(orm.TermQuery("pipeline_id", pipelineID)).
+ SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(1)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil || res == nil {
+ return 0
+ }
+ versions, _, _ := decodeVersions(res)
+ if len(versions) == 0 {
+ return 0
+ }
+ return versions[0].Version
+}
+
+func (h *PipeModule) pruneVersions(pipelineID string) {
+ octx := orm.NewContext().DirectAccess()
+ orm.WithModel(octx, &PipelineVersion{})
+ q := orm.NewQuery().
+ Filter(orm.TermQuery("pipeline_id", pipelineID)).
+ SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(maxVersionsPerPipeline + 10)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ return
+ }
+ versions, _, _ := decodeVersions(res)
+ if len(versions) <= maxVersionsPerPipeline {
+ return
+ }
+ for _, old := range versions[maxVersionsPerPipeline:] {
+ _ = orm.Delete(octx, &old)
+ }
+}
+
+func decodeVersions(res *orm.SearchResult) ([]PipelineVersion, int64, error) {
+ return elastic.DecodeHits[PipelineVersion](res)
+}
+
+// ─── endpoints ────────────────────────────────────────────────────────────
+
+// listPipelineVersionsHandler — GET /pipelines/:id/versions
+func (h *PipeModule) listPipelineVersionsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineVersion{})
+ q := orm.NewQuery().
+ Filter(orm.TermQuery("pipeline_id", id)).
+ SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(50)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ versions, total, _ := decodeVersions(res)
+ // strip the full config from the list view (it can be large)
+ type versionSummary struct {
+ Version int `json:"version"`
+ Note string `json:"note,omitempty"`
+ UpdatedBy string `json:"updated_by,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
+ }
+ out := make([]versionSummary, 0, len(versions))
+ for _, v := range versions {
+ out = append(out, versionSummary{v.Version, v.Note, v.UpdatedBy, v.CreatedAt})
+ }
+ h.WriteJSON(w, util.MapStr{"total": total, "versions": out}, http.StatusOK)
+}
+
+// getPipelineVersionHandler — GET /pipelines/:id/versions/:v
+func (h *PipeModule) getPipelineVersionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ vStr := ps.MustGetParameter("version")
+ want, err := strconv.Atoi(vStr)
+ if err != nil {
+ h.WriteError(w, "invalid version number", http.StatusBadRequest)
+ return
+ }
+ v := h.findVersion(req, id, want)
+ if v == nil {
+ h.WriteOpRecordNotFoundJSON(w, vStr)
+ return
+ }
+ h.WriteJSON(w, v, http.StatusOK)
+}
+
+// rollbackPipelineHandler — POST /pipelines/:id/versions/:v/rollback
+// Restores the config from the target version snapshot and writes a new
+// version entry recording the rollback.
+func (h *PipeModule) rollbackPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+ id := ps.MustGetParameter("id")
+ vStr := ps.MustGetParameter("version")
+ want, err := strconv.Atoi(vStr)
+ if err != nil {
+ h.WriteError(w, "invalid version number", http.StatusBadRequest)
+ return
+ }
+
+ target := h.findVersion(req, id, want)
+ if target == nil {
+ h.WriteOpRecordNotFoundJSON(w, vStr)
+ return
+ }
+
+ // snapshot the CURRENT config before rollback (so the rollback itself
+ // is also revertible)
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ current := pipeline.PipelineConfigV2{}
+ current.ID = id
+ exists, err := orm.GetWithSystemFields(octx, ¤t)
+ if err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if exists {
+ if err := h.saveVersion(octx, ¤t, "pre-rollback snapshot"); err != nil {
+ log.Warnf("pipeline %s: pre-rollback snapshot failed: %v", id, err)
+ }
+ }
+
+ // restore the target config
+ restored := pipeline.PipelineConfigV2{}
+ if err := util.FromJSONBytes([]byte(target.Config), &restored); err != nil {
+ h.WriteError(w, fmt.Sprintf("decode version %d config: %v", want, err), http.StatusInternalServerError)
+ return
+ }
+ restored.ID = id // preserve identity
+
+ // compile gate: an old version may reference processors that were
+ // renamed or removed since — restoring it must not break the pipeline
+ if !h.validateAndReject(w, &restored) {
+ return
+ }
+ octx2 := orm.NewContextWithParent(req.Context())
+ octx2.Refresh = orm.WaitForRefresh
+ if err := orm.Save(octx2, &restored); err != nil {
+ h.WriteError(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ h.WriteJSON(w, util.MapStr{
+ "rolled_back_to": want,
+ "id": id,
+ "note": target.Note,
+ }, http.StatusOK)
+}
+
+func (h *PipeModule) findVersion(req *http.Request, pipelineID string, version int) *PipelineVersion {
+ octx := orm.NewContextWithParent(req.Context()).DirectAccess()
+ orm.WithModel(octx, &PipelineVersion{})
+ q := orm.NewQuery().
+ Filter(orm.TermQuery("pipeline_id", pipelineID)).
+ Filter(orm.TermQuery("version", version)).Size(1)
+ res, err := orm.SearchV2(octx, q)
+ if err != nil {
+ return nil
+ }
+ versions, _, _ := decodeVersions(res)
+ if len(versions) == 0 {
+ return nil
+ }
+ return &versions[0]
+}
diff --git a/modules/queue/api.go b/modules/queue/api.go
index d134d87c0..22bf0473c 100644
--- a/modules/queue/api.go
+++ b/modules/queue/api.go
@@ -27,6 +27,7 @@
package queue
import (
+ stderrors "errors"
"fmt"
"infini.sh/framework/core/global"
"infini.sh/framework/core/security"
@@ -73,6 +74,12 @@ func init() {
//reset consumer offset
api.HandleAPIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset)
api.HandleUIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueUpdate))
+ // Consumer management: offset read and consumer delete (clearing stale
+ // consumer groups/offsets, same as Console).
+ api.HandleAPIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset)
+ api.HandleUIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueRead))
+ api.HandleAPIMethod(api.DELETE, "/queue/:id/consumer/:consumer_id", module.QueueDeleteConsumerByID)
+ api.HandleUIMethod(api.DELETE, "/queue/:id/consumer/:consumer_id", module.QueueDeleteConsumerByID, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueUpdate))
//get consumer offset
api.HandleAPIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset)
api.HandleUIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueRead))
@@ -331,8 +338,15 @@ func (module *API) QueueExplore(w http.ResponseWriter, req *http.Request, ps htt
result := util.MapStr{}
status := 200
if err != nil {
- result["error"] = err.Error()
- status = 500
+ // an empty queue (nothing written yet) is normal for
+ // browse/sample: answer "no messages" instead of a 500
+ if stderrors.Is(err, queue1.ErrEmptyQueue) {
+ result["empty"] = true
+ err = nil
+ } else {
+ result["error"] = err.Error()
+ status = 500
+ }
}
if len(messages) > 0 {
if dataIsString {
@@ -396,7 +410,11 @@ func (module *API) QueueExplore(w http.ResponseWriter, req *http.Request, ps htt
return
}
} else {
- log.Errorf("can't acquire consumer [%v] for [%v]", consumer.Key(), qConfig.Name)
+ if stderrors.Is(err, queue1.ErrEmptyQueue) {
+ log.Debugf("queue [%v] is empty, skip consumer [%v]", qConfig.Name, consumer.Key())
+ } else {
+ log.Errorf("can't acquire consumer [%v] for [%v]", consumer.Key(), qConfig.Name)
+ }
}
} else {
diff --git a/modules/queue/disk_queue/consumer.go b/modules/queue/disk_queue/consumer.go
index c18bbd83e..83ffe1c05 100644
--- a/modules/queue/disk_queue/consumer.go
+++ b/modules/queue/disk_queue/consumer.go
@@ -144,7 +144,20 @@ READ_MSG:
// check reader
if d.reader == nil {
- return messages, false, errors.New("reader is nil")
+ //parked at the write head (offset's data files are gone): the
+ //write segment file may exist by now, re-open it and continue;
+ //otherwise report an empty fetch so the consumer keeps waiting
+ //instead of erroring out
+ fileName, exists, _ := SmartGetFileName(d.mCfg, d.queue, d.segment)
+ if !exists && !util.FileExists(fileName) {
+ if len(messages) == 0 && d.cCfg.EOFRetryDelayInMs > 0 {
+ time.Sleep(time.Duration(d.cCfg.EOFRetryDelayInMs) * time.Millisecond)
+ }
+ return messages, false, nil
+ }
+ if rerr := d.ResetOffset(d.segment, d.readPos); rerr != nil || d.reader == nil {
+ return messages, false, nil
+ }
}
//read message size
err = binary.Read(d.reader, binary.BigEndian, &msgSize)
@@ -477,13 +490,15 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error {
}
if segment > d.diskQueue.writeSegmentNum {
- log.Errorf("reading segment [%v] is greater than writing segment [%v]", segment, d.diskQueue.writeSegmentNum)
- return io.EOF
+ log.Warnf("queue:%v,%v, consumer:%v, offset [%v,%v] is beyond the write segment [%v], the queue's data was likely wiped or recreated, reset to the earliest retained data",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), segment, readPos, d.diskQueue.writeSegmentNum)
+ return d.resetToEarliestRetainedData()
}
if segment == d.diskQueue.writeSegmentNum && readPos > d.diskQueue.writePos {
- log.Errorf("reading position [%v] is greater than writing position [%v]", readPos, d.diskQueue.writePos)
- return io.EOF
+ log.Warnf("queue:%v,%v, consumer:%v, read position [%v] is beyond the write position [%v] on segment [%v], restart the segment from the beginning",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), readPos, d.diskQueue.writePos, segment)
+ readPos = 0
}
if d.segment != segment {
@@ -512,13 +527,16 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error {
if d.mCfg.AutoSkipCorruptFile {
nextSegment := d.segment + 1
if nextSegment > d.diskQueue.writeSegmentNum {
- return errors.Errorf("%v not found and next segment(%v) greater than current write segment(%v)", fileName, nextSegment, d.diskQueue.writeSegmentNum)
+ log.Debugf("queue:%v,%v, consumer:%v, offset:%v,%v, file %v is missing and no later segment exists, park at the write head",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, fileName)
+ d.parkAtWriteHead()
+ return nil
}
log.Warnf("queue:%v,%v, consumer:%v, offset:%v,%v, file missing: %v, auto skip to next file",
d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, fileName)
RETRY_NEXT_FILE:
- // there are segments in the middle
- if nextSegment < d.diskQueue.writeSegmentNum {
+ // there are segments in the middle, or the write segment itself
+ if nextSegment <= d.diskQueue.writeSegmentNum {
fileName, exists, next_file_exists = SmartGetFileName(d.mCfg, d.queue, nextSegment)
if exists || util.FileExists(fileName) {
log.Debugf("retry skip to next file: %v, exists", fileName)
@@ -532,7 +550,10 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error {
goto RETRY_NEXT_FILE
}
} else {
- return errors.New(fileName + " not found, next segment greater than current write segment")
+ log.Debugf("queue:%v,%v, consumer:%v, offset:%v,%v, no segment file exists up to the write segment %v, park at the write head",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, d.diskQueue.writeSegmentNum)
+ d.parkAtWriteHead()
+ return nil
}
} else {
return errors.New(fileName + " not found and auto_skip_corrupt_file not enabled.")
@@ -543,13 +564,19 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error {
FIND_NEXT_FILE:
//if next file exists, and current file is not the last file, the file should be completed loaded
- if next_file_exists || d.diskQueue.writeSegmentNum > segment {
+ if next_file_exists || d.diskQueue.writeSegmentNum > d.segment {
d.fileLoadCompleted = true
} else {
d.fileLoadCompleted = false
}
- var err error
+ return d.openReadFile(fileName)
+}
+
+// openReadFile points the consumer's reader at an existing segment file,
+// seeking to the current read position. It is the shared tail of
+// ResetOffset and the self-healing reset paths below.
+func (d *Consumer) openReadFile(fileName string) error {
readFile, err := os.OpenFile(fileName, os.O_RDONLY, 0600)
if err != nil {
log.Error(err)
@@ -582,3 +609,70 @@ FIND_NEXT_FILE:
d.fileName = fileName
return nil
}
+
+// resetToEarliestRetainedData re-anchors the consumer onto the earliest
+// segment file that still exists on disk. It runs when the requested
+// offset segment is beyond the current write segment: the queue's data
+// files were wiped (or the queue was deleted and recreated) while the
+// consumer's persisted offset still references the removed segments.
+// Returning an error there used to wedge pipeline consumers in an
+// acquire-retry panic loop; instead, consume whatever data survives
+// (at-least-once), and when nothing is retained park at the write head
+// and report EOF so the owning worker retries once data arrives.
+func (d *Consumer) resetToEarliestRetainedData() error {
+ for seg := int64(0); seg <= d.diskQueue.writeSegmentNum; seg++ {
+ fileName, exists, nextFileExists := SmartGetFileName(d.mCfg, d.queue, seg)
+ if !exists && !util.FileExists(fileName) {
+ continue
+ }
+ log.Warnf("queue:%v,%v, consumer:%v, reset to the earliest retained segment: %v", d.qCfg.Name, d.queue, d.cCfg.Key(), fileName)
+ if d.readFile != nil {
+ //potential file handler leak
+ d.readFile.Close()
+ }
+ d.segment = seg
+ d.readPos = 0
+ d.maxBytesPerFileRead = 0
+ d.diskQueue.UpdateSegmentConsumerInReading(d.ID, d.segment)
+ if nextFileExists || d.diskQueue.writeSegmentNum > seg {
+ d.fileLoadCompleted = true
+ } else {
+ d.fileLoadCompleted = false
+ }
+ return d.openReadFile(fileName)
+ }
+
+ //nothing retained on disk: park at the write head and wait for new data
+ log.Debugf("queue:%v,%v, consumer:%v, no retained data on disk, park at the write head: segment %v",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), d.diskQueue.writeSegmentNum)
+ d.parkAtWriteHead()
+ return nil
+}
+
+// parkAtWriteHead leaves the consumer waiting for new data at the current
+// write position, with no reader attached: every data file the offset
+// referenced is gone, so there is nothing to read until the writer
+// creates the write segment file. FetchMessages re-opens the segment as
+// soon as it appears, so callers see a healthy (if idle) consumer
+// instead of an error that would panic-loop the owning worker.
+func (d *Consumer) parkAtWriteHead() {
+ if d.readFile != nil {
+ d.readFile.Close()
+ d.readFile = nil
+ }
+ d.reader = nil
+ d.segment = d.diskQueue.writeSegmentNum
+ d.readPos = 0
+ d.maxBytesPerFileRead = 0
+ d.lastFileSize = 0
+ d.fileLoadCompleted = false
+ d.diskQueue.UpdateSegmentConsumerInReading(d.ID, d.segment)
+
+ //idle workers get recycled and re-acquire on a regular cadence, which
+ //would replay this warning forever — rate-limit it to once a minute
+ if now := time.Now().UnixMilli(); now-d.diskQueue.parkWarnAt.Load() > 60*1000 {
+ d.diskQueue.parkWarnAt.Store(now)
+ log.Warnf("queue:%v,%v, consumer:%v, offset:%v,%v, data files are missing (queue wiped or recreated), park at the write head: segment %v, waiting for new data",
+ d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, d.diskQueue.writeSegmentNum)
+ }
+}
diff --git a/modules/queue/disk_queue/consumer_selfheal_test.go b/modules/queue/disk_queue/consumer_selfheal_test.go
new file mode 100644
index 000000000..f2329a38a
--- /dev/null
+++ b/modules/queue/disk_queue/consumer_selfheal_test.go
@@ -0,0 +1,222 @@
+package queue
+
+import (
+ "os"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/queue"
+)
+
+// Regression tests for consumers whose persisted offset points at segment
+// files that no longer exist (queue wiped / deleted and recreated, or
+// retention removed files under a lagging consumer). AcquireConsumer used
+// to return "not found, next segment greater than current write segment"
+// and wedge pipeline consumers in an acquire-retry panic loop; the
+// consumer must instead self-heal onto whatever data is still on disk.
+
+func newSelfhealTestQueue(t *testing.T, name string) (*DiskBasedQueue, string) {
+ t.Helper()
+ global.Env().SystemConfig.PathConfig.Data = t.TempDir()
+ dataPath := GetDataPath(name)
+ _ = os.MkdirAll(dataPath, 0755)
+
+ cfg := &DiskQueueConfig{
+ MinMsgSize: 1,
+ MaxMsgSize: 104857600,
+ MaxBytesPerFile: 1024, // small segments: multiple files quickly
+ SyncEveryRecords: 1,
+ SyncTimeoutInMS: 10,
+ WriteTimeoutInMS: 1000,
+ EOFRetryDelayInMs: 100,
+ ReadChanBuffer: 10,
+ WriteChanBuffer: 10,
+ AutoSkipCorruptFile: true,
+ }
+ return NewDiskQueueByConfig(name, dataPath, cfg), dataPath
+}
+
+func putTestRecords(t *testing.T, d *DiskBasedQueue, n int, payload string) {
+ t.Helper()
+ for i := 0; i < n; i++ {
+ res := d.Put([]byte(payload))
+ if res.Error != nil {
+ t.Fatalf("put: %v", res.Error)
+ }
+ }
+ time.Sleep(500 * time.Millisecond)
+}
+
+// Offset lands inside the range of segments, but every file up to (not
+// including) the current write segment was wiped: the skip-forward walk
+// must include the write segment itself, which it previously never
+// checked.
+func TestAcquireConsumerSkipsToWriteSegment(t *testing.T) {
+ name := "test-selfheal-skip"
+ d, _ := newSelfhealTestQueue(t, name)
+ defer d.Close()
+
+ putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef")
+ if d.writeSegmentNum < 2 {
+ t.Fatalf("expected several segments, got write segment %v", d.writeSegmentNum)
+ }
+
+ // wipe every segment file below the last written one (old data gone,
+ // fresh data still sits in the newest segment file)
+ for seg := int64(0); seg < d.writeSegmentNum-1; seg++ {
+ if err := os.Remove(GetFileName(name, seg)); err != nil && !os.IsNotExist(err) {
+ t.Fatalf("remove segment %v: %v", seg, err)
+ }
+ }
+
+ c, err := d.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-skip"}, queue.NewOffset(1, 512))
+ if err != nil {
+ t.Fatalf("acquire with wiped offset should self-heal, got: %v", err)
+ }
+ defer c.Close()
+
+ msgs, _, err := c.FetchMessages(&queue.Context{}, 500)
+ if err != nil {
+ t.Fatalf("fetch: %v", err)
+ }
+ if len(msgs) == 0 {
+ t.Fatal("expected the records retained in the write segment to be consumable")
+ }
+ t.Logf("self-healed onto write segment %v, fetched %v messages", d.writeSegmentNum, len(msgs))
+}
+
+// Offset is beyond the write segment entirely (queue deleted and
+// recreated, writer restarted from a low segment): the consumer must
+// re-anchor onto the earliest retained segment instead of erroring.
+func TestAcquireConsumerResetsToEarliestRetainedData(t *testing.T) {
+ name := "test-selfheal-reset"
+ d, dataPath := newSelfhealTestQueue(t, name)
+
+ putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef")
+ d.Close()
+
+ // wipe the whole queue dir except meta, then reopen the writer and
+ // produce fresh data into low segments — the "queue recreated" state
+ entries, err := os.ReadDir(dataPath)
+ if err != nil {
+ t.Fatalf("readdir: %v", err)
+ }
+ for _, e := range entries {
+ if e.Name() != "meta.dat" {
+ if err := os.Remove(dataPath + string(os.PathSeparator) + e.Name()); err != nil {
+ t.Fatalf("remove %v: %v", e.Name(), err)
+ }
+ }
+ }
+
+ d2 := NewDiskQueueByConfig(name, dataPath, &DiskQueueConfig{
+ MinMsgSize: 1,
+ MaxMsgSize: 104857600,
+ MaxBytesPerFile: 1048576,
+ SyncEveryRecords: 1,
+ SyncTimeoutInMS: 10,
+ WriteTimeoutInMS: 1000,
+ EOFRetryDelayInMs: 100,
+ ReadChanBuffer: 10,
+ WriteChanBuffer: 10,
+ AutoSkipCorruptFile: true,
+ })
+ defer d2.Close()
+ putTestRecords(t, d2, 5, "post-wipe message")
+
+ // stale offset far beyond the recreated queue's write segment
+ c, err := d2.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-reset"}, queue.NewOffset(7, 104857842))
+ if err != nil {
+ t.Fatalf("acquire with offset beyond retained data should self-heal, got: %v", err)
+ }
+ defer c.Close()
+
+ // the writer flushes asynchronously: poll until the 5 post-wipe
+ // messages are all consumed
+ total := 0
+ deadline := time.Now().Add(5 * time.Second)
+ for total < 5 && time.Now().Before(deadline) {
+ msgs, _, err := c.FetchMessages(&queue.Context{}, 10)
+ if err != nil {
+ t.Fatalf("fetch: %v", err)
+ }
+ total += len(msgs)
+ time.Sleep(100 * time.Millisecond)
+ }
+ if total != 5 {
+ t.Fatalf("expected the 5 post-wipe messages, got %v", total)
+ }
+}
+
+// No data file survives at all (fully wiped queue, nothing written since):
+// acquiring with a stale offset must succeed with the consumer parked at
+// the write head — FetchMessages reports empty instead of erroring — and
+// consumption must resume in place once the writer produces data.
+func TestAcquireConsumerParksAtWriteHeadWhenNoData(t *testing.T) {
+ name := "test-selfheal-park"
+ d, dataPath := newSelfhealTestQueue(t, name)
+
+ putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef")
+ d.Close()
+
+ // wipe the whole queue dir except meta and reopen the writer with no
+ // data on disk
+ entries, err := os.ReadDir(dataPath)
+ if err != nil {
+ t.Fatalf("readdir: %v", err)
+ }
+ for _, e := range entries {
+ if e.Name() != "meta.dat" {
+ if err := os.Remove(dataPath + string(os.PathSeparator) + e.Name()); err != nil {
+ t.Fatalf("remove %v: %v", e.Name(), err)
+ }
+ }
+ }
+
+ d2 := NewDiskQueueByConfig(name, dataPath, &DiskQueueConfig{
+ MinMsgSize: 1,
+ MaxMsgSize: 104857600,
+ MaxBytesPerFile: 1048576,
+ SyncEveryRecords: 1,
+ SyncTimeoutInMS: 10,
+ WriteTimeoutInMS: 1000,
+ EOFRetryDelayInMs: 100,
+ ReadChanBuffer: 10,
+ WriteChanBuffer: 10,
+ AutoSkipCorruptFile: true,
+ })
+ defer d2.Close()
+ time.Sleep(500 * time.Millisecond)
+
+ // stale offset beyond everything: parks at the write head
+ c, err := d2.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-park", EOFRetryDelayInMs: 50}, queue.NewOffset(20, 5))
+ if err != nil {
+ t.Fatalf("acquire with no retained data should park, got: %v", err)
+ }
+ defer c.Close()
+
+ msgs, _, err := c.FetchMessages(&queue.Context{}, 10)
+ if err != nil {
+ t.Fatalf("fetch on parked consumer should be a clean empty, got: %v", err)
+ }
+ if len(msgs) != 0 {
+ t.Fatalf("expected empty fetch on parked consumer, got %v", len(msgs))
+ }
+
+ // data arrives: the parked consumer picks it up without re-acquire
+ putTestRecords(t, d2, 3, "post-park message")
+ total := 0
+ deadline := time.Now().Add(5 * time.Second)
+ for total < 3 && time.Now().Before(deadline) {
+ msgs, _, err = c.FetchMessages(&queue.Context{}, 10)
+ if err != nil {
+ t.Fatalf("fetch after data arrived: %v", err)
+ }
+ total += len(msgs)
+ time.Sleep(100 * time.Millisecond)
+ }
+ if total != 3 {
+ t.Fatalf("expected 3 post-park messages, got %v", total)
+ }
+}
diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go
index 8f5949a5f..f4ebc0f86 100644
--- a/modules/queue/disk_queue/diskqueue.go
+++ b/modules/queue/disk_queue/diskqueue.go
@@ -60,6 +60,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
log "github.com/cihub/seelog"
@@ -91,6 +92,10 @@ type DiskBasedQueue struct {
exitFlag int32
needSync bool
+ //last time a consumer parked at the write head logged a warning,
+ //rate-limited because idle workers re-acquire (and re-park) regularly
+ parkWarnAt atomic.Int64
+
consumerMode bool
// read related
diff --git a/modules/queue/disk_queue/empty_concurrent_test.go b/modules/queue/disk_queue/empty_concurrent_test.go
index 79b098346..8e7083321 100644
--- a/modules/queue/disk_queue/empty_concurrent_test.go
+++ b/modules/queue/disk_queue/empty_concurrent_test.go
@@ -23,7 +23,7 @@ func (m *concKV) GetValue(b string, k []byte) ([]byte, error) {
return nil, nil
}
func (m *concKV) GetCompressedValue(b string, k []byte) ([]byte, error) { return m.GetValue(b, k) }
-func (m *concKV) AddValueCompress(b string, k, v []byte) error { return m.AddValue(b, k, v) }
+func (m *concKV) AddValueCompress(b string, k, v []byte) error { return m.AddValue(b, k, v) }
func (m *concKV) AddValue(b string, k, v []byte) error {
m.data[string(k)] = append([]byte(nil), v...)
return nil
diff --git a/modules/queue/disk_queue/empty_test.go b/modules/queue/disk_queue/empty_test.go
index af4bf684a..31d1ab2f8 100644
--- a/modules/queue/disk_queue/empty_test.go
+++ b/modules/queue/disk_queue/empty_test.go
@@ -12,10 +12,12 @@ import (
type memKV struct{ data map[string][]byte }
-func (m *memKV) Open() error { return nil }
-func (m *memKV) Close() error { return nil }
+func (m *memKV) Open() error { return nil }
+func (m *memKV) Close() error { return nil }
func (m *memKV) GetValue(b string, k []byte) ([]byte, error) {
- if v, ok := m.data[string(k)]; ok { return append([]byte(nil), v...), nil }
+ if v, ok := m.data[string(k)]; ok {
+ return append([]byte(nil), v...), nil
+ }
return nil, nil
}
func (m *memKV) GetCompressedValue(b string, k []byte) ([]byte, error) { return m.GetValue(b, k) }
@@ -44,15 +46,15 @@ func TestEmptyRemovesConsumedSegments(t *testing.T) {
_ = os.MkdirAll(dataPath, 0755)
cfg := &DiskQueueConfig{
- MinMsgSize: 1,
- MaxMsgSize: 104857600,
- MaxBytesPerFile: 1024, // small segments: multiple files quickly
- SyncEveryRecords: 1,
- SyncTimeoutInMS: 10,
- WriteTimeoutInMS: 1000,
+ MinMsgSize: 1,
+ MaxMsgSize: 104857600,
+ MaxBytesPerFile: 1024, // small segments: multiple files quickly
+ SyncEveryRecords: 1,
+ SyncTimeoutInMS: 10,
+ WriteTimeoutInMS: 1000,
EOFRetryDelayInMs: 100,
- ReadChanBuffer: 10,
- WriteChanBuffer: 10,
+ ReadChanBuffer: 10,
+ WriteChanBuffer: 10,
}
d := NewDiskQueueByConfig(name, dataPath, cfg)
defer d.Close()
diff --git a/modules/queue/disk_queue/module.go b/modules/queue/disk_queue/module.go
index f8996db3e..961bb10fd 100644
--- a/modules/queue/disk_queue/module.go
+++ b/modules/queue/disk_queue/module.go
@@ -390,7 +390,7 @@ func (module *DiskQueue) AcquireConsumer(qconfig *queue.QueueConfig, consumer *q
q1 := q.(*DiskBasedQueue)
if q1.writeSegmentNum == 0 && q1.writePos == 0 {
//empty queue, no need to create consumer
- return nil, errors.New("empty queue")
+ return nil, queue.ErrEmptyQueue
}
return q1.AcquireConsumer(qconfig, consumer, offset)
}
diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go
index 4a9057553..29560dfb1 100644
--- a/modules/security/access_token/authentication.go
+++ b/modules/security/access_token/authentication.go
@@ -241,6 +241,11 @@ func CreateAPIToken(user *security.UserSessionInfo, tokenName, tokenDesc, typeNa
ctx.DirectAccess()
ctx.Refresh = orm.WaitForRefresh
ctx.PermissionScope(security.PermissionScopePlatform)
+ // the elastic ORM's data-op hooks resolve the index from the ctx
+ // model — without this the create panics "model is not set" on any
+ // ES-backed manager (sqlite resolves from the object, so only
+ // native+elastic deployments hit it)
+ orm.WithModel(ctx, &accessToken)
if err := orm.Create(ctx, &accessToken); err != nil {
return nil, err
@@ -618,6 +623,7 @@ func ListTokens() ([]*security.AccessToken, error) {
if isNative() {
ctx := orm.NewContext().DirectAccess()
ctx.PermissionScope(security.PermissionScopePlatform)
+ orm.WithModel(ctx, &security.AccessToken{})
res, err := orm.SearchV2(ctx, orm.NewQuery().Size(1000))
if err != nil {
return nil, err
@@ -658,6 +664,7 @@ func DeleteTokenByID(tokenID string) error {
if isNative() {
ctx := orm.NewContext().DirectAccess()
ctx.PermissionScope(security.PermissionScopePlatform)
+ orm.WithModel(ctx, &security.AccessToken{})
t := security.AccessToken{}
t.ID = tokenID
return orm.Delete(ctx, &t)
diff --git a/modules/sqlite/orm/query_builder.go b/modules/sqlite/orm/query_builder.go
index 7c9b04f4c..5a65bb287 100644
--- a/modules/sqlite/orm/query_builder.go
+++ b/modules/sqlite/orm/query_builder.go
@@ -166,11 +166,16 @@ func clauseToSQL(clause *orm.Clause, resolve FieldResolver) (string, []interface
}
}
- // must_not is combined with AND NOT
+ // must_not is combined with AND NOT. A predicate over a MISSING
+ // field yields NULL under SQL three-valued logic (NOT NULL = NULL =
+ // row dropped), but Elasticsearch semantics say a missing field never
+ // matches the inner term — so must_not passes and the row survives.
+ // IFNULL folds that NULL to false before the NOT, restoring the
+ // engine's contract for sparse fields.
for _, sub := range clause.MustNotClauses {
sql, args := clauseToSQL(sub, resolve)
if sql != "" {
- parts = append(parts, fmt.Sprintf("NOT (%s)", sql))
+ parts = append(parts, fmt.Sprintf("NOT (IFNULL(%s, 0))", sql))
allArgs = append(allArgs, args...)
}
}
diff --git a/modules/stats/simple.go b/modules/stats/simple.go
index 7a9877b81..1639ddd49 100755
--- a/modules/stats/simple.go
+++ b/modules/stats/simple.go
@@ -40,6 +40,9 @@ import (
log "github.com/cihub/seelog"
"github.com/segmentio/encoding/json"
+ gcpu "github.com/shirou/gopsutil/v4/cpu"
+ gmem "github.com/shirou/gopsutil/v4/mem"
+ gnet "github.com/shirou/gopsutil/v4/net"
"github.com/shirou/gopsutil/v4/process"
"infini.sh/framework/core/api"
httprouter "infini.sh/framework/core/api/router"
@@ -388,6 +391,21 @@ func (s *Stats) StatsAll() string {
systemStats["store"] = int64(storage) //maybe too heavy to call frequently
}
+ // host-level utilization + network counters (cumulative since boot,
+ // consumers compute rates from deltas between polls). Best-effort:
+ // any failure just omits the keys, never fails the whole response.
+ if hostPercents, err := gcpu.Percent(0, false); err == nil && len(hostPercents) > 0 {
+ systemStats["host_cpu_percent"] = int64(hostPercents[0])
+ }
+ if vm, err := gmem.VirtualMemory(); err == nil && vm != nil {
+ systemStats["host_mem_total_bytes"] = int64(vm.Total)
+ systemStats["host_mem_used_bytes"] = int64(vm.Used)
+ }
+ if ioCounters, err := gnet.IOCounters(false); err == nil && len(ioCounters) > 0 {
+ systemStats["net_in_bytes"] = int64(ioCounters[0].BytesRecv)
+ systemStats["net_out_bytes"] = int64(ioCounters[0].BytesSent)
+ }
+
result["system"] = systemStats
return util.ToJson(result, false)
diff --git a/modules/task/task.go b/modules/task/task.go
index a3d33e9cc..8e327f909 100644
--- a/modules/task/task.go
+++ b/modules/task/task.go
@@ -25,11 +25,11 @@ package task
import (
"infini.sh/framework/core/api"
- "infini.sh/framework/core/security"
httprouter "infini.sh/framework/core/api/router"
"infini.sh/framework/core/env"
"infini.sh/framework/core/global"
"infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/security"
"infini.sh/framework/core/task"
"infini.sh/framework/core/util"
"net/http"
diff --git a/plugins/elastic/indexing_merge/indexing_merge.go b/plugins/elastic/indexing_merge/indexing_merge.go
index 791d25ac5..3c8eea16c 100644
--- a/plugins/elastic/indexing_merge/indexing_merge.go
+++ b/plugins/elastic/indexing_merge/indexing_merge.go
@@ -46,6 +46,7 @@ import (
"sync"
"time"
+ "github.com/buger/jsonparser"
log "github.com/cihub/seelog"
"github.com/savsgio/gotils/bytes"
"infini.sh/framework/core/config"
@@ -58,6 +59,7 @@ import (
"infini.sh/framework/core/util"
"infini.sh/framework/lib/bytebufferpool"
"infini.sh/framework/modules/elastic/common"
+ "strings"
)
type IndexingMergeProcessor struct {
@@ -119,14 +121,14 @@ func New(c *config.Config) (pipeline.Processor, error) {
}
if cfg.InputQueue == "" {
- panic(errors.New("input_queue can't be nil"))
+ return nil, errors.New("indexing_merge processor requires input_queue")
}
if cfg.OutputQueue.Name == "" {
- panic(errors.New("name of output_queue can't be nil"))
+ return nil, errors.New("indexing_merge processor requires output_queue.name")
}
if cfg.IndexName == "" {
- panic(errors.New("index name can't be nil"))
+ return nil, errors.New("indexing_merge processor requires index_name")
}
if cfg.FailureQueue == "" {
@@ -271,17 +273,16 @@ READ_DOCS:
if len(processor.config.KeyFields) > 0 {
id_part = fmt.Sprintf(", \"_id\":\"%v\"", processor.fingerprintID(pop))
} else if processor.config.KeyField != "" {
- source := util.MapStr{}
- err := util.FromJSONBytes(pop, &source)
+ // zero-alloc path query: extracting ONE field
+ // must not float-parse the whole document (embeddings!)
+ v, _, _, err := jsonparser.Get(pop, strings.Split(processor.config.KeyField, ".")...)
if err != nil {
- panic(err)
+ log.Warnf("indexing_merge: key_field %q not found: %v", processor.config.KeyField, err)
+ } else if s, ok := trimJSONString(v); ok {
+ id_part = fmt.Sprintf(", \"_id\":\"%v\"", s)
+ } else {
+ id_part = fmt.Sprintf(", \"_id\":\"%v\"", string(v))
}
-
- v, err := source.GetValue(processor.config.KeyField)
- if err != nil {
- panic(err)
- }
- id_part = fmt.Sprintf(", \"_id\":\"%v\"", v)
}
// default to index
writeOpType := "index"
@@ -370,9 +371,13 @@ CLEAN_BUFFER:
// first-class doc fields)
// - a top-level @timestamp is derived from payload.timestamp /
// payload.observed_timestamp / the envelope timestamp / now — data
-// streams reject documents without a date @timestamp
-// - timestamp mirrors @timestamp (conventional sort field)
-// - metadata.file (agent collection origin) is kept as top-level "file"
+// streams reject documents without a date @timestamp. @timestamp is the
+// document's one true timestamp (house standard): the mirrored top-level
+// "timestamp" and any payload "timestamp" are folded into it and removed.
+// - a bare document (no "payload" key) keeps its own @timestamp when
+// present; only a missing one is stamped with now.
+// - metadata.file (agent collection origin) lives in @agent.file_* — the
+// former top-level "file" promotion is gone (zero readers).
//
// Documents that are not envelopes (no "payload" key) pass through
// unchanged except for the @timestamp stamp.
@@ -396,6 +401,11 @@ func normalizeDataStreamDoc(doc []byte) []byte {
ts = v
}
}
+ if ts == "" {
+ if v, ok := m["@timestamp"].(string); ok { // bare doc already carrying one
+ ts = v
+ }
+ }
if ts == "" {
ts = time.Now().UTC().Format(time.RFC3339Nano)
}
@@ -411,12 +421,9 @@ func normalizeDataStreamDoc(doc []byte) []byte {
}
}
out["@timestamp"] = ts
- out["timestamp"] = ts
+ delete(out, "timestamp") // @timestamp is the single source of truth
if meta, ok := m["metadata"].(map[string]interface{}); ok {
- if f, ok := meta["file"].(map[string]interface{}); ok {
- out["file"] = f
- }
if rt, ok := meta["log_type"].(string); ok && rt != "" {
out["log_type"] = rt
}
@@ -443,24 +450,19 @@ func normalizeDataStreamDoc(doc []byte) []byte {
func (processor *IndexingMergeProcessor) fingerprintID(pop []byte) string {
h := fnv.New64a()
- source := util.MapStr{}
- if err := util.FromJSONBytes(pop, &source); err != nil {
- // undecodable message - hash the raw bytes as-is
- h.Write(pop)
- return fmt.Sprintf("%x", h.Sum64())
- }
-
+ // zero-alloc path queries: fingerprinting must not
+ // float-parse the whole document (embeddings ride these messages)
matched := false
for _, f := range processor.config.KeyFields {
h.Write([]byte{'|'})
- v, err := source.GetValue(f)
- if err != nil || v == nil {
+ v, _, _, err := jsonparser.Get(pop, strings.Split(f, ".")...)
+ if err != nil {
// keep positional meaning for missing fields
h.Write([]byte("\x00"))
continue
}
matched = true
- h.Write([]byte(util.ToString(v)))
+ h.Write(v) // raw JSON token: any scalar type hashes deterministically
}
if !matched {
// none of the key fields exist on this message: fall back to a
@@ -470,3 +472,12 @@ func (processor *IndexingMergeProcessor) fingerprintID(pop []byte) string {
}
return fmt.Sprintf("%x", h.Sum64())
}
+
+// trimJSONString strips JSON string quotes from a raw token ("" and false
+// when the token is not a quoted string).
+func trimJSONString(v []byte) (string, bool) {
+ if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' {
+ return string(v[1 : len(v)-1]), true
+ }
+ return "", false
+}
diff --git a/plugins/elastic/indexing_merge/indexing_merge_test.go b/plugins/elastic/indexing_merge/indexing_merge_test.go
index cc7ab3277..c920d1c92 100644
--- a/plugins/elastic/indexing_merge/indexing_merge_test.go
+++ b/plugins/elastic/indexing_merge/indexing_merge_test.go
@@ -31,16 +31,22 @@ func TestNormalizeDataStreamDoc(t *testing.T) {
t.Fatalf("payload key should not survive: %v", doc)
}
- // @timestamp: payload 无显式 timestamp 时 observed_timestamp 优先于信封级;
- // timestamp 镜像之。
- if doc["@timestamp"] != "2026-08-23T14:38:32.982744Z" || doc["timestamp"] != "2026-08-23T14:38:32.982744Z" {
- t.Fatalf("timestamp fields: @=%v ts=%v", doc["@timestamp"], doc["timestamp"])
+ // @timestamp: when the payload carries no explicit timestamp,
+ // observed_timestamp wins over the envelope-level one. @timestamp is
+ // the only real timestamp — the legacy "timestamp" mirror is no longer
+ // written.
+ if doc["@timestamp"] != "2026-08-23T14:38:32.982744Z" {
+ t.Fatalf("@timestamp = %v", doc["@timestamp"])
+ }
+ if _, ok := doc["timestamp"]; ok {
+ t.Fatalf("timestamp mirror must not be written: %v", doc["timestamp"])
}
- // metadata promotion.
- file, ok := doc["file"].(map[string]interface{})
- if !ok || file["path"] != "/var/log/system.log" {
- t.Fatalf("file metadata not promoted: %v", doc["file"])
+ // metadata promotion: file origin no longer promotes to a top-level
+ // "file" (collection provenance lives on @agent.file_*); only the
+ // log_type/log_kind labels promote.
+ if _, ok := doc["file"]; ok {
+ t.Fatalf("top-level file promotion is gone: %v", doc["file"])
}
if doc["log_type"] != "text" || doc["log_kind"] != "server" {
t.Fatalf("label fields not promoted: %v", doc)
@@ -64,11 +70,14 @@ func TestNormalizeDataStreamDocTimestampPriority(t *testing.T) {
if doc["@timestamp"] != tc.want {
t.Fatalf("%s: @timestamp = %v, want %v", tc.name, doc["@timestamp"], tc.want)
}
+ if _, ok := doc["timestamp"]; ok {
+ t.Fatalf("%s: payload/envelope timestamp must fold into @timestamp only: %v", tc.name, doc)
+ }
}
}
// TestNormalizeDataStreamDocBareDoc: non-envelope docs keep their fields
-// and only get the @timestamp stamp.
+// and only get the @timestamp stamp (single truth — no mirror).
func TestNormalizeDataStreamDocBareDoc(t *testing.T) {
out := normalizeDataStreamDoc([]byte(`{"message":"raw","level":"info"}`))
var doc util.MapStr
@@ -76,8 +85,19 @@ func TestNormalizeDataStreamDocBareDoc(t *testing.T) {
if doc["message"] != "raw" || doc["level"] != "info" {
t.Fatalf("bare doc fields lost: %v", doc)
}
- if doc["timestamp"] == "" || doc["@timestamp"] == "" {
- t.Fatalf("bare doc missing timestamp stamps: %v", doc)
+ if doc["@timestamp"] == "" {
+ t.Fatalf("bare doc missing @timestamp: %v", doc)
+ }
+ if _, ok := doc["timestamp"]; ok {
+ t.Fatalf("bare doc must not gain a timestamp mirror: %v", doc)
+ }
+
+ // bare doc already carrying @timestamp keeps it (no re-stamp)
+ keep := normalizeDataStreamDoc([]byte(`{"message":"kept","@timestamp":"2026-05-05T00:00:00Z"}`))
+ var kd util.MapStr
+ _ = json.Unmarshal(keep, &kd)
+ if kd["@timestamp"] != "2026-05-05T00:00:00Z" {
+ t.Fatalf("bare doc @timestamp must be kept: %v", kd["@timestamp"])
}
}
diff --git a/plugins/elastic/json_indexing/json_indexing.go b/plugins/elastic/json_indexing/json_indexing.go
index 812eab31e..ee2411ee7 100644
--- a/plugins/elastic/json_indexing/json_indexing.go
+++ b/plugins/elastic/json_indexing/json_indexing.go
@@ -100,7 +100,7 @@ func New(c *config.Config) (pipeline.Processor, error) {
}
if cfg.InputQueue == "" {
- panic(errors.New("input_queue can't be nil"))
+ return nil, errors.New("json_indexing processor requires input_queue")
}
if cfg.FailureQueue == "" {
diff --git a/plugins/elastic/lane_bulk/bulk_indexing_lane.go b/plugins/elastic/lane_bulk/bulk_indexing_lane.go
new file mode 100644
index 000000000..303243f3b
--- /dev/null
+++ b/plugins/elastic/lane_bulk/bulk_indexing_lane.go
@@ -0,0 +1,313 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello#infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+/* ©INFINI, All Rights Reserved.
+ * mail: contact#infini.ltd */
+
+package lane_bulk
+
+import (
+ "fmt"
+ "sync"
+ "time"
+
+ log "github.com/cihub/seelog"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/errors"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/stats"
+ "infini.sh/framework/core/util"
+)
+
+// BulkIndexingLaneProcessor is the memory-path twin of bulk_indexing: it
+// subscribes to a lane of merged bulk records (merge_to_bulk_lane's output)
+// and sends each payload to Elasticsearch. Receipts of a batch settle when
+// its sends complete (the SubscribeBatch settlement contract), which is
+// also the settlement point for the merge side's EmitAck receipts.
+//
+// Failure policy: a failed send retries after retry_delay_in_ms. Without a
+// failure_queue retries continue indefinitely — a down cluster stalls the
+// lane, which backpressures the merge, which stalls input queue consumption
+// (no silent loss, matching legacy bulk_indexing's stall semantics). With a
+// failure_queue set, a payload diverts there after max_retries attempts and
+// the lane keeps flowing.
+type BulkIndexingLaneProcessor struct {
+ config SenderConfig
+ inputLane lane.Lane
+ failureQ *queue.QueueConfig
+ bulkProcessor elastic.BulkProcessor
+ bufferPool *elastic.BulkBufferPool
+}
+
+func (processor *BulkIndexingLaneProcessor) Name() string {
+ return "bulk_indexing_lane"
+}
+
+type SenderConfig struct {
+ Lane string `config:"lane"`
+ Elasticsearch string `config:"elasticsearch"`
+
+ Bulk elastic.BulkProcessorConfig `config:"bulk"`
+
+ FailureQueue string `config:"failure_queue"`
+ MaxRetries int `config:"max_retries"`
+ RetryDelayInMs int `config:"retry_delay_in_ms"`
+
+ // WorkerSize sends one batch's payloads concurrently across workers;
+ // the batch function waits for every send before returning, keeping
+ // receipt settlement exact.
+ WorkerSize int `config:"worker_size"`
+
+ // BatchFlushMs flushes a partial batch after this idle window. Must
+ // be > 0: without a timer, a batch below batch_max never reaches its
+ // size trigger and its records would sit undelivered.
+ BatchFlushMs int `config:"batch_flush_ms"`
+}
+
+func init() {
+ pipeline.RegisterProcessorPluginWithConfigMetadata("bulk_indexing_lane", NewBulkIndexingLane, SenderConfig{})
+}
+
+func NewBulkIndexingLane(c *config.Config) (pipeline.Processor, error) {
+ cfg := SenderConfig{
+ RetryDelayInMs: 3000,
+ WorkerSize: 1,
+ BatchFlushMs: 100,
+ }
+
+ if err := c.Unpack(&cfg); err != nil {
+ return nil, fmt.Errorf("failed to unpack the configuration of bulk_indexing_lane processor: %s", err)
+ }
+
+ if cfg.Lane == "" {
+ return nil, errors.New("bulk_indexing_lane processor requires lane")
+ }
+ if cfg.Elasticsearch == "" {
+ return nil, errors.New("bulk_indexing_lane processor requires elasticsearch")
+ }
+ if cfg.RetryDelayInMs <= 0 {
+ cfg.RetryDelayInMs = 3000
+ }
+ if cfg.WorkerSize <= 0 {
+ cfg.WorkerSize = 1
+ }
+ if cfg.BatchFlushMs <= 0 {
+ cfg.BatchFlushMs = 100
+ }
+ if cfg.FailureQueue != "" && cfg.MaxRetries <= 0 {
+ // a failure queue without a retry budget diverts on first failure;
+ // make the budget explicit instead of guessing
+ return nil, errors.New("bulk_indexing_lane processor requires max_retries > 0 when failure_queue is set")
+ }
+
+ // GetOrCreate (not Get+error): tolerate the merge task not having run
+ // yet (startup ordering), and re-create the lane when a previous task
+ // instance closed it on exit — a bare Get would keep returning the
+ // closed instance and the task would silently idle forever. Capacity
+ // matches merge_to_bulk_lane's auto-create budget (merged bulks are
+ // large; an explicit `lane:` stanza overrides when present).
+ inputLane := lane.GetOrCreate(lane.LaneConfig{Name: cfg.Lane, Capacity: autoLaneCapacity})
+
+ var failureQ *queue.QueueConfig
+ if cfg.FailureQueue != "" {
+ failureQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.FailureQueue})
+ }
+
+ return &BulkIndexingLaneProcessor{
+ config: cfg,
+ inputLane: inputLane,
+ failureQ: failureQ,
+ bulkProcessor: elastic.NewBulkProcessor("bulk_indexing_lane", cfg.Elasticsearch, cfg.Bulk),
+ bufferPool: elastic.NewBulkBufferPool("bulk_indexing_lane", 1024*1024*1024, 100000),
+ }, nil
+}
+
+// Process subscribes (once) and blocks until the pipeline task cancels —
+// the host drives the lane for its whole lifetime. On exit the lane is
+// DELETED (closed + unregistered), not merely closed: Close alone leaves a
+// permanently-closed instance in the registry that wedges the producer
+// (merge_to_bulk_lane's emits fail "closed" forever) and breaks this task's
+// own restart (a re-Get would return the dead instance and the subscriber
+// would exit immediately). Delete + the producers'/consumers' GetOrCreate
+// re-creation makes a task stop/restart a recoverable cycle: the drain
+// semantics of Close are preserved (buffered records flush through the
+// batch function, in-flight receipts settle).
+func (processor *BulkIndexingLaneProcessor) Process(ctx *pipeline.Context) error {
+ meta := processor.waitMetadata(ctx)
+ if meta == nil {
+ return nil
+ }
+
+ // batch width tracks the worker fan-out: batch_max payloads are picked
+ // up per flush (size trigger, or batch_flush_ms idle trigger — lone
+ // records below batch_max rely on the timer) and sent
+ // worker_size-at-a-time
+ batchMax := processor.config.WorkerSize * 2
+ _ = processor.inputLane.SubscribeBatch(batchMax, time.Duration(processor.config.BatchFlushMs)*time.Millisecond, func(recs []*event.Event) {
+ processor.sendBatch(ctx, meta, recs)
+ })
+
+ <-ctx.Done()
+ _ = lane.Delete(processor.config.Lane, true)
+ return nil
+}
+
+// sendBatch delivers one micro-batch: records are dealt across workers,
+// every send completes (or diverts) before returning — receipts settle only
+// after that (SubscribeBatch contract), so settlement means "delivered or
+// salvaged", never "still pending".
+func (processor *BulkIndexingLaneProcessor) sendBatch(ctx *pipeline.Context, meta *elastic.ElasticsearchMetadata, recs []*event.Event) {
+ workers := processor.config.WorkerSize
+ if workers > len(recs) {
+ workers = len(recs)
+ }
+ if workers <= 1 {
+ for _, rec := range recs {
+ processor.sendRecord(ctx, meta, rec)
+ }
+ return
+ }
+
+ var wg sync.WaitGroup
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func(chunk []*event.Event) {
+ defer wg.Done()
+ for _, rec := range chunk {
+ processor.sendRecord(ctx, meta, rec)
+ }
+ }(recs[w:])
+ }
+ wg.Wait()
+}
+
+// waitMetadata blocks until the cluster metadata is available (startup
+// ordering: the elastic module may finish after pipeline tasks) or the task
+// cancels. Stalling here is safe — nothing has been consumed yet.
+func (processor *BulkIndexingLaneProcessor) waitMetadata(ctx *pipeline.Context) *elastic.ElasticsearchMetadata {
+ for {
+ if ctx.IsCanceled() {
+ return nil
+ }
+ if meta := elastic.GetMetadata(processor.config.Elasticsearch); meta != nil {
+ return meta
+ }
+ log.Warnf("bulk_indexing_lane: cluster metadata [%v] not ready, waiting", processor.config.Elasticsearch)
+ select {
+ case <-time.After(2 * time.Second):
+ case <-ctx.Done():
+ return nil
+ }
+ }
+}
+
+// sendRecord delivers one merged bulk payload with the configured retry
+// policy. On task cancellation each in-flight record gets ONE final attempt
+// (the legacy final-submit behavior) and unresolved payloads divert to the
+// failure queue when one is configured.
+func (processor *BulkIndexingLaneProcessor) sendRecord(ctx *pipeline.Context, meta *elastic.ElasticsearchMetadata, rec *event.Event) {
+ bulk, ok := bulkPayloadOf(rec)
+ if !ok {
+ log.Warnf("bulk_indexing_lane: record without bulk payload (trace %q), skip", pipeline.TraceID(rec))
+ stats.Increment("bulk_indexing_lane", "skipped")
+ return
+ }
+
+ id := pipeline.TraceID(rec)
+ if id == "" {
+ id = "bulk-" + util.GetUUID()
+ }
+
+ // shutdown drain: one best-effort attempt, then salvage — retrying
+ // forever here would hang the task teardown
+ final := ctx.IsCanceled() || global.ShuttingDown()
+
+ for attempt := 0; ; attempt++ {
+ if !final && (ctx.IsCanceled() || global.ShuttingDown()) {
+ final = true
+ }
+
+ host := meta.GetActivePreferredSeedHost()
+ if elastic.IsHostDead(host) || !elastic.IsHostAvailable(host) {
+ host = meta.GetActiveHost()
+ }
+
+ buf := processor.bufferPool.AcquireBulkBuffer()
+ buf.Queue = processor.config.Elasticsearch
+ buf.WriteMessageID(id)
+ buf.Write(bulk)
+ buf.SafetyEndWithNewline()
+
+ continueNext, statsRet, _, err := processor.bulkProcessor.Bulk(ctx.Context, "bulk_indexing_lane", meta, host, buf)
+ buf.ResetData()
+ processor.bufferPool.ReturnBulkBuffer(buf)
+
+ for code, n := range statsRet {
+ stats.IncrementBy("bulk_indexing_lane", "response_code_"+util.ToString(code), int64(n))
+ }
+
+ if err == nil {
+ stats.Increment("bulk_indexing_lane", "sent")
+ stats.IncrementBy("bulk_indexing_lane", "sent_bytes", int64(len(bulk)))
+ if !continueNext {
+ log.Warnf("bulk_indexing_lane: cluster suggested backoff (continue=false)")
+ }
+ return
+ }
+
+ stats.Increment("bulk_indexing_lane", "failed")
+ log.Errorf("bulk_indexing_lane: bulk send failed (host: %v, attempt: %v, err: %v)", host, attempt+1, err)
+
+ // only a configured failure queue ends the retry loop early;
+ // without one a hard-down cluster stalls the lane (intended)
+ if final || (processor.failureQ != nil && attempt+1 >= processor.config.MaxRetries) {
+ break
+ }
+
+ select {
+ case <-time.After(time.Duration(processor.config.RetryDelayInMs) * time.Millisecond):
+ case <-ctx.Done():
+ continue // loop top re-checks cancellation and flags final
+ }
+ }
+
+ if processor.failureQ != nil {
+ if err := queue.Push(processor.failureQ, bulk); err != nil {
+ log.Errorf("bulk_indexing_lane: push to failure queue %q failed: %v", processor.failureQ.Name, err)
+ } else {
+ stats.Increment("bulk_indexing_lane", "diverted")
+ }
+ return
+ }
+ if ctx.IsCanceled() || global.ShuttingDown() {
+ // no salvage path configured on the shutdown drain — visible in
+ // stats rather than silent
+ stats.Increment("bulk_indexing_lane", "lost_on_shutdown")
+ }
+}
diff --git a/plugins/elastic/lane_bulk/lane_bulk_test.go b/plugins/elastic/lane_bulk/lane_bulk_test.go
new file mode 100644
index 000000000..d4b9e163d
--- /dev/null
+++ b/plugins/elastic/lane_bulk/lane_bulk_test.go
@@ -0,0 +1,639 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software. For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello#infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+/* ©INFINI, All Rights Reserved.
+ * mail: contact#infini.ltd */
+
+package lane_bulk
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/kv"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/util"
+)
+
+// ─── fake in-memory queue backend ────────────────────────────────────────────
+
+type memQueue struct {
+ mu sync.Mutex
+ data map[string][][]byte
+}
+
+func (q *memQueue) Name() string { return "mem" }
+func (q *memQueue) Init(string) error {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ if q.data == nil {
+ q.data = map[string][][]byte{}
+ }
+ return nil
+}
+func (q *memQueue) Close(string) error { return nil }
+func (q *memQueue) Destroy(string) error { return nil }
+func (q *memQueue) GetStorageSize(string) uint64 { return 0 }
+func (q *memQueue) GetQueues() []string { return nil }
+
+func (q *memQueue) Push(id string, v []byte) error {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ if q.data == nil {
+ q.data = map[string][][]byte{}
+ }
+ q.data[id] = append(q.data[id], v)
+ return nil
+}
+
+func (q *memQueue) Pop(id string, timeout time.Duration) ([]byte, bool) {
+ q.mu.Lock()
+ if items := q.data[id]; len(items) > 0 {
+ v := items[0]
+ q.data[id] = items[1:]
+ q.mu.Unlock()
+ return v, false
+ }
+ q.mu.Unlock()
+ d := timeout
+ if d > 200*time.Millisecond {
+ d = 200 * time.Millisecond
+ }
+ time.Sleep(d)
+ return nil, true
+}
+
+func (q *memQueue) Depth(id string) int64 {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ return int64(len(q.data[id]))
+}
+
+var memQueueOnce sync.Once
+
+// memKV backs the queue-config registry in unit tests (GetOrInitConfig
+// persists configs through the kv store; without a handler it panics).
+type memKV struct {
+ data map[string][]byte
+}
+
+func (m *memKV) Open() error { return nil }
+func (m *memKV) Close() error { return nil }
+func (m *memKV) GetValue(bucket string, key []byte) ([]byte, error) {
+ if v, ok := m.data[string(key)]; ok {
+ return append([]byte(nil), v...), nil
+ }
+ return nil, nil
+}
+func (m *memKV) GetCompressedValue(bucket string, key []byte) ([]byte, error) {
+ return m.GetValue(bucket, key)
+}
+func (m *memKV) AddValueCompress(bucket string, key, value []byte) error {
+ return m.AddValue(bucket, key, value)
+}
+func (m *memKV) AddValue(bucket string, key, value []byte) error {
+ m.data[string(key)] = append([]byte(nil), value...)
+ return nil
+}
+func (m *memKV) ExistsKey(bucket string, key []byte) (bool, error) {
+ _, ok := m.data[string(key)]
+ return ok, nil
+}
+func (m *memKV) DeleteKey(bucket string, key []byte) error {
+ delete(m.data, string(key))
+ return nil
+}
+
+// installMemQueue wires the fake queue backend and kv store so
+// GetOrInitConfig-created queues (empty type) resolve end to end.
+func installMemQueue() *memQueue {
+ q := &memQueue{}
+ memQueueOnce.Do(func() {
+ queue.RegisterDefaultHandler(q)
+ kv.Register("lane-bulk-test-kv", &memKV{data: map[string][]byte{}})
+ })
+ return q
+}
+
+// ─── payload codec ───────────────────────────────────────────────────────────
+
+func TestBulkPayloadCodecRoundTrip(t *testing.T) {
+ payload := []byte("{\"index\":{}}\n{\"a\":1}\n")
+
+ rec, err := bulkPayloadCodec{}.Decode(payload)
+ if err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if got := pipeline.CodecHint(rec); got != "bulk" {
+ t.Errorf("codec hint = %q, want bulk", got)
+ }
+ out, err := bulkPayloadCodec{}.Encode(rec)
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+ if string(out) != string(payload) {
+ t.Errorf("round-trip mismatch:\n got: %q\nwant: %q", out, payload)
+ }
+
+ if _, err := (bulkPayloadCodec{}).Decode(nil); err == nil {
+ t.Error("empty payload should error")
+ }
+ bare := &event.Event{Fields: util.MapStr{"not_bulk": 1}}
+ if _, err := (bulkPayloadCodec{}).Encode(bare); err == nil {
+ t.Error("record without payload field should error")
+ }
+}
+
+// ─── merge formatting helpers ────────────────────────────────────────────────
+
+func TestFingerprintID(t *testing.T) {
+ p := &MergeToBulkLaneProcessor{config: MergeConfig{KeyFields: []string{"file.path", "file.offset"}}}
+
+ doc := []byte(`{"file":{"path":"/var/log/a.log","offset":42},"message":"x"}`)
+
+ a := p.fingerprintID(doc)
+ b := p.fingerprintID(doc)
+ if a != b {
+ t.Errorf("fingerprint not deterministic: %v vs %v", a, b)
+ }
+
+ // a different offset must yield a different id
+ doc2 := []byte(`{"file":{"path":"/var/log/a.log","offset":43},"message":"x"}`)
+ if p.fingerprintID(doc2) == a {
+ t.Error("offset change should change the fingerprint")
+ }
+
+ // no key field matches: content-hash fallback, distinct docs stay distinct
+ c1 := p.fingerprintID([]byte(`{"message":"one"}`))
+ c2 := p.fingerprintID([]byte(`{"message":"two"}`))
+ if c1 == c2 {
+ t.Error("content fallback collapsed distinct documents")
+ }
+}
+
+func TestTrimJSONString(t *testing.T) {
+ if s, ok := trimJSONString([]byte(`"hello"`)); !ok || s != "hello" {
+ t.Errorf("quoted token: %q %v", s, ok)
+ }
+ if _, ok := trimJSONString([]byte(`42`)); ok {
+ t.Error("non-string token should not trim")
+ }
+}
+
+func TestNormalizeDataStreamDoc(t *testing.T) {
+ env := []byte(`{"metadata":{"file":{"path":"/a.log"}},"payload":{"message":"hi","timestamp":"2026-10-15T00:00:00Z"},"timestamp":"2026-10-15T00:00:00Z"}`)
+
+ out := normalizeDataStreamDoc(env)
+ m := util.MapStr{}
+ if err := util.FromJSONBytes(out, &m); err != nil {
+ t.Fatalf("normalized doc is not JSON: %v", err)
+ }
+ if m["message"] != "hi" {
+ t.Errorf("payload.message not promoted: %v", m)
+ }
+ if m["@timestamp"] != "2026-10-15T00:00:00Z" {
+ t.Errorf("@timestamp = %v", m["@timestamp"])
+ }
+ // @timestamp is the only real timestamp: the payload's timestamp folds
+ // into @timestamp and is dropped — no top-level mirror is written.
+ if _, ok := m["timestamp"]; ok {
+ t.Errorf("timestamp mirror must not survive: %v", m["timestamp"])
+ }
+ // file origin no longer promotes to a top-level "file"
+ // (collection provenance lives on @agent.file_*)
+ if _, ok := m["file"]; ok {
+ t.Errorf("top-level file promotion is gone: %v", m["file"])
+ }
+
+ // bare docs pass through with a stamped @timestamp
+ bare := normalizeDataStreamDoc([]byte(`{"message":"plain"}`))
+ m2 := util.MapStr{}
+ if err := util.FromJSONBytes(bare, &m2); err != nil {
+ t.Fatalf("bare doc: %v", err)
+ }
+ if m2["message"] != "plain" {
+ t.Errorf("bare doc message lost: %v", m2)
+ }
+ if m2["@timestamp"] == nil {
+ t.Error("bare doc missing stamped @timestamp")
+ }
+}
+
+// ─── config validation ───────────────────────────────────────────────────────
+
+func TestNewMergeValidation(t *testing.T) {
+ cases := []struct {
+ name string
+ cfg map[string]interface{}
+ errMsg string
+ }{
+ {"empty", map[string]interface{}{}, "exactly one of input_queue | input_lane"},
+ {"both", map[string]interface{}{"input_queue": "q", "input_lane": "l"}, "exactly one of input_queue | input_lane"},
+ {"no lane", map[string]interface{}{"input_queue": "q"}, "requires output_lane"},
+ {"no index", map[string]interface{}{"input_queue": "q", "output_lane": "l"}, "requires index_name"},
+ }
+ for _, c := range cases {
+ cfg, err := config.NewConfigFrom(c.cfg)
+ if err != nil {
+ t.Fatalf("%s: build config: %v", c.name, err)
+ }
+ _, err = NewMergeToBulkLane(cfg)
+ if err == nil || !strings.Contains(err.Error(), c.errMsg) {
+ t.Errorf("%s: err = %v, want contains %q", c.name, err, c.errMsg)
+ }
+ }
+}
+
+func TestNewSenderValidation(t *testing.T) {
+ cfg, err := config.NewConfigFrom(map[string]interface{}{})
+ if err != nil {
+ t.Fatalf("build config: %v", err)
+ }
+ if _, err := NewBulkIndexingLane(cfg); err == nil || !strings.Contains(err.Error(), "requires lane") {
+ t.Errorf("empty config: err = %v", err)
+ }
+
+ // failure_queue without max_retries is rejected
+ cfg2, _ := config.NewConfigFrom(map[string]interface{}{
+ "lane": "missing", "elasticsearch": "es", "failure_queue": "dead",
+ })
+ if _, err := NewBulkIndexingLane(cfg2); err == nil || !strings.Contains(err.Error(), "max_retries") {
+ t.Errorf("failure queue without budget: err = %v", err)
+ }
+
+ // a missing lane is auto-created: the old "not found" rejection wedged
+ // task restarts (consumer exits delete the lane; a restart racing its
+ // producers had nothing to bind and the pipeline died)
+ cfg3, _ := config.NewConfigFrom(map[string]interface{}{
+ "lane": "test-sender-auto-lane", "elasticsearch": "es",
+ })
+ if _, err := NewBulkIndexingLane(cfg3); err != nil {
+ t.Errorf("auto-create lane: err = %v", err)
+ }
+ if l := lane.Get("test-sender-auto-lane"); l == nil {
+ t.Error("auto-created lane not registered")
+ }
+ _ = lane.Delete("test-sender-auto-lane", true)
+}
+
+// ─── merge end-to-end over a lane ────────────────────────────────────────────
+
+func TestMergeToBulkLaneEndToEnd(t *testing.T) {
+ installMemQueue()
+
+ const laneName = "test-merge-lane"
+ const queueName = "test-merge-queue"
+ l, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 16})
+ if err != nil {
+ t.Fatalf("create lane: %v", err)
+ }
+ defer lane.Delete(laneName, true)
+
+ qcfg := queue.GetOrInitConfig(queueName)
+ docs := []string{
+ `{"id":"1","message":"alpha"}`,
+ `{"id":"2","message":"beta"}`,
+ `{"id":"3","message":"gamma"}`,
+ }
+ for _, d := range docs {
+ if err := queue.Push(qcfg, []byte(d)); err != nil {
+ t.Fatalf("push: %v", err)
+ }
+ }
+
+ cfg, err := config.NewConfigFrom(map[string]interface{}{
+ "input_queue": queueName,
+ "output_lane": laneName,
+ "index_name": "test-logs",
+ "key_field": "id",
+ "idle_timeout_in_seconds": 1,
+ "bulk_size_in_kb": 1,
+ })
+ if err != nil {
+ t.Fatalf("build config: %v", err)
+ }
+ proc, err := NewMergeToBulkLane(cfg)
+ if err != nil {
+ t.Fatalf("new processor: %v", err)
+ }
+
+ stdCtx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ pctx := &pipeline.Context{Context: stdCtx}
+
+ type delivery struct {
+ payload string
+ traceID string
+ codec string
+ }
+ bulks := make(chan delivery, 8)
+ l.SubscribeBatch(4, 50*time.Millisecond, func(recs []*event.Event) {
+ for _, rec := range recs {
+ if b, ok := bulkPayloadOf(rec); ok {
+ bulks <- delivery{payload: string(b), traceID: pipeline.TraceID(rec), codec: pipeline.CodecHint(rec)}
+ }
+ }
+ })
+
+ processDone := make(chan error, 1)
+ go func() { processDone <- proc.Process(pctx) }()
+
+ var combined string
+ var first delivery
+ deadline := time.After(10 * time.Second)
+collect:
+ for {
+ select {
+ case d := <-bulks:
+ if first.payload == "" {
+ first = d
+ }
+ combined += d.payload
+ if strings.Count(combined, `"message"`) >= len(docs) {
+ break collect
+ }
+ case <-deadline:
+ t.Fatalf("timed out waiting for merged bulk, got: %q", combined)
+ }
+ }
+
+ // action lines target the right index and carry the key_field-derived ids
+ if !strings.Contains(combined, `"_index" : "test-logs"`) {
+ t.Errorf("action line missing index:\n%s", combined)
+ }
+ for _, id := range []string{`"_id":"1"`, `"_id":"2"`, `"_id":"3"`} {
+ if !strings.Contains(combined, id) {
+ t.Errorf("action line missing %s:\n%s", id, combined)
+ }
+ }
+ // every doc body survived the merge
+ for _, d := range docs {
+ if !strings.Contains(combined, d) {
+ t.Errorf("doc body missing from bulk:\n%s\nwant: %s", combined, d)
+ }
+ }
+ // record metadata: codec hint routes spill encoding, trace id correlates
+ if first.codec != "bulk" {
+ t.Errorf("codec hint = %q, want bulk", first.codec)
+ }
+ if first.traceID == "" {
+ t.Error("merged record missing trace id")
+ }
+
+ // cancellation stops the merge host
+ cancel()
+ select {
+ case <-processDone:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Process did not return after cancel")
+ }
+
+ // nothing hit the salvage path: the failure queue stays empty
+ if d := queue.Depth(queue.GetOrInitConfig(queueName + "_failure")); d != 0 {
+ t.Errorf("failure queue depth = %d, want 0", d)
+ }
+}
+
+// ─── consumer stop/restart cycle over the lane ──────────────────────────────
+
+// TestBulkLaneSurvivesConsumerRestart pins the gateway wedge fix: when the
+// sink task (bulk_indexing_lane) exits it DELETES the lane; the merge host
+// must re-create it on the next flush and keep flowing (no "closed" emit
+// errors, nothing salvaged), and a restarted consumer binds the fresh
+// instance. Before the fix every flush after a sink-task stop salvaged to
+// the failure queue with "lane closed" errors, forever.
+func TestBulkLaneSurvivesConsumerRestart(t *testing.T) {
+ installMemQueue()
+
+ const laneName = "test-restart-lane"
+ const queueName = "test-restart-queue"
+ if _, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 16}); err != nil {
+ t.Fatalf("create lane: %v", err)
+ }
+ defer lane.Delete(laneName, true)
+
+ qcfg := queue.GetOrInitConfig(queueName)
+
+ cfg, err := config.NewConfigFrom(map[string]interface{}{
+ "input_queue": queueName,
+ "output_lane": laneName,
+ "index_name": "test-logs",
+ "idle_timeout_in_seconds": 1,
+ "bulk_size_in_kb": 1,
+ })
+ if err != nil {
+ t.Fatalf("build config: %v", err)
+ }
+ proc, err := NewMergeToBulkLane(cfg)
+ if err != nil {
+ t.Fatalf("new processor: %v", err)
+ }
+
+ stdCtx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ processDone := make(chan error, 1)
+ go func() { processDone <- proc.Process(&pipeline.Context{Context: stdCtx}) }()
+
+ bulks := make(chan string, 8)
+ subscribe := func(l lane.Lane) {
+ l.SubscribeBatch(4, 50*time.Millisecond, func(recs []*event.Event) {
+ for _, rec := range recs {
+ if b, ok := bulkPayloadOf(rec); ok {
+ bulks <- string(b)
+ }
+ }
+ })
+ }
+ subscribe(lane.Get(laneName))
+
+ // phase 1: normal flow through the first lane instance
+ if err := queue.Push(qcfg, []byte(`{"id":"1","message":"before-restart"}`)); err != nil {
+ t.Fatal(err)
+ }
+ waitBulk(t, bulks, "before-restart")
+
+ // the sink task exiting: bulk_indexing_lane's Process deletes the lane
+ // on ctx.Done — exactly this call
+ if err := lane.Delete(laneName, true); err != nil {
+ t.Fatalf("delete lane: %v", err)
+ }
+
+ // phase 2: the merge's next flush re-creates the lane and emits into it
+ if err := queue.Push(qcfg, []byte(`{"id":"2","message":"after-restart"}`)); err != nil {
+ t.Fatal(err)
+ }
+
+ // the "restarted consumer": wait for the producer to re-create the lane
+ // instance, then bind it
+ var fresh lane.Lane
+ deadline := time.Now().Add(10 * time.Second)
+ for fresh = lane.Get(laneName); fresh == nil && time.Now().Before(deadline); fresh = lane.Get(laneName) {
+ time.Sleep(50 * time.Millisecond)
+ }
+ if fresh == nil {
+ t.Fatal("lane was not re-created after consumer exit")
+ }
+ subscribe(fresh)
+ waitBulk(t, bulks, "after-restart")
+
+ cancel()
+ select {
+ case <-processDone:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Process did not return after cancel")
+ }
+
+ // nothing hit the salvage path — the wedge symptom was every flush
+ // salvaging with "lane closed" errors
+ if d := queue.Depth(queue.GetOrInitConfig(queueName + "_failure")); d != 0 {
+ t.Errorf("failure queue depth = %d, want 0", d)
+ }
+}
+
+func waitBulk(t *testing.T, bulks chan string, want string) {
+ t.Helper()
+ deadline := time.After(10 * time.Second)
+ for {
+ select {
+ case b := <-bulks:
+ if strings.Contains(b, want) {
+ return
+ }
+ case <-deadline:
+ t.Fatalf("timed out waiting for bulk containing %q", want)
+ }
+ }
+}
+
+// ─── block-on-full backpressure ──────────────────────────────────────────────
+
+func TestMergeToBulkLaneBlocksWhenLaneFull(t *testing.T) {
+ installMemQueue()
+
+ const laneName = "test-block-lane"
+ const queueName = "test-block-queue"
+ l, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 1})
+ if err != nil {
+ t.Fatalf("create lane: %v", err)
+ }
+ defer lane.Delete(laneName, true)
+
+ // big bodies force frequent flushes: ~600B docs against a 1KB bulk
+ // threshold yields a merged bulk roughly every two docs
+ qcfg := queue.GetOrInitConfig(queueName)
+ bigBody := strings.Repeat("x", 600)
+ const docCount = 20
+ for i := 0; i < docCount; i++ {
+ doc := `{"id":"` + fmt.Sprint(i) + `","message":"` + bigBody + `"}`
+ if err := queue.Push(qcfg, []byte(doc)); err != nil {
+ t.Fatalf("push: %v", err)
+ }
+ }
+
+ cfg, err := config.NewConfigFrom(map[string]interface{}{
+ "input_queue": queueName,
+ "output_lane": laneName,
+ "index_name": "test-logs",
+ "idle_timeout_in_seconds": 1,
+ "bulk_size_in_kb": 1,
+ })
+ if err != nil {
+ t.Fatalf("build config: %v", err)
+ }
+ proc, err := NewMergeToBulkLane(cfg)
+ if err != nil {
+ t.Fatalf("new processor: %v", err)
+ }
+
+ stdCtx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ pctx := &pipeline.Context{Context: stdCtx}
+
+ // the sole subscriber holds its first delivery until released: the
+ // lane (capacity 1) saturates and the merge must BLOCK in Emit
+ release := make(chan struct{})
+ var deliveries int32
+ l.SubscribeBatch(1, 0, func(recs []*event.Event) {
+ atomic.AddInt32(&deliveries, 1)
+ <-release
+ })
+
+ processDone := make(chan error, 1)
+ go func() { processDone <- proc.Process(pctx) }()
+
+ // saturation point: one record held by the subscriber, one in the
+ // channel, further emissions blocked — input queue consumption stalls
+ saturated := waitFor(time.Second*20, func() bool {
+ return atomic.LoadInt32(&deliveries) >= 1 && l.Depth() >= 1
+ })
+ if !saturated {
+ t.Fatalf("lane did not saturate (deliveries=%d depth=%d)", atomic.LoadInt32(&deliveries), l.Depth())
+ }
+ time.Sleep(1500 * time.Millisecond) // idle window passes while blocked
+ if d := queue.Depth(qcfg); d == 0 {
+ t.Error("input queue drained while the lane was saturated — blocking backpressure not engaged")
+ }
+
+ // release the subscriber: flow resumes, the input queue drains, and
+ // only then does cancellation stop the host
+ close(release)
+ if !waitFor(20*time.Second, func() bool { return queue.Depth(qcfg) == 0 }) {
+ t.Fatalf("input queue did not drain after release, depth=%d", queue.Depth(qcfg))
+ }
+ cancel()
+ select {
+ case <-processDone:
+ case <-time.After(15 * time.Second):
+ t.Fatal("Process did not return after cancel")
+ }
+}
+
+func waitFor(timeout time.Duration, cond func() bool) bool {
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if cond() {
+ return true
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ return cond()
+}
+
+// ─── sender guard path ───────────────────────────────────────────────────────
+
+func TestSenderSkipsPayloadlessRecord(t *testing.T) {
+ p := &BulkIndexingLaneProcessor{}
+ pctx := &pipeline.Context{Context: context.Background()}
+
+ // a record without the bulk payload skips before any ES interaction
+ // (meta stays nil): the guard must hold without panicking
+ rec := &event.Event{Fields: util.MapStr{"something": "else"}}
+ p.sendRecord(pctx, nil, rec)
+}
diff --git a/plugins/elastic/lane_bulk/merge_to_bulk_lane.go b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go
new file mode 100644
index 000000000..a1ff6d064
--- /dev/null
+++ b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go
@@ -0,0 +1,524 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello#infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+/* ©INFINI, All Rights Reserved.
+ * mail: contact#infini.ltd */
+
+package lane_bulk
+
+import (
+ "fmt"
+ "hash/fnv"
+ "runtime"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/buger/jsonparser"
+ log "github.com/cihub/seelog"
+
+ "infini.sh/framework/core/config"
+ "infini.sh/framework/core/elastic"
+ "infini.sh/framework/core/errors"
+ "infini.sh/framework/core/global"
+ "infini.sh/framework/core/lane"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/queue"
+ "infini.sh/framework/core/stats"
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/util"
+ "infini.sh/framework/lib/bytebufferpool"
+ "infini.sh/framework/modules/elastic/common"
+)
+
+// autoLaneCapacity bounds the lane auto-created when no `lane:` stanza
+// defines the output lane: enough slack to smooth sender hiccups, small
+// enough that block-on-full engages at a sane memory ceiling.
+const autoLaneCapacity = 64
+
+// MergeToBulkLaneProcessor is the memory-path twin of indexing_merge: it
+// pops JSON documents off the input disk queue, merges them into ES bulk
+// NDJSON payloads (bulk_size / idle_timeout), and emits each merged payload
+// as ONE record into the output lane — no output disk queue in between.
+//
+// Durability note: the input queue pop is the point of no return (same
+// semantics as indexing_merge); a crash loses whatever is buffered in
+// memory. Records travel without ack receipts — a future commit-gated
+// variant can switch to EmitAck and wait on settlement without changing
+// this processor's configuration contract.
+//
+// Backpressure: lane capacity bounds in-flight bulks; a full lane BLOCKS
+// (the default overflow policy) — the merge loop stalls, then input queue
+// consumption stalls. That is the intended posture for the memory path:
+// pressure propagates back to the durable entry point instead of paying
+// intermediate disk IO. (`overflow: spill:` remains a lane-level
+// opt-out; spilled payloads are raw bulk NDJSON the legacy bulk_indexing
+// can consume.)
+type MergeToBulkLaneProcessor struct {
+ config MergeConfig
+ failureQ *queue.QueueConfig
+ outputLane lane.Lane
+ // laneCfg is the effective output-lane config (stanza-bound or the
+ // auto-create default) — the blueprint for GetOrCreate re-creation.
+ laneCfg lane.LaneConfig
+}
+
+func (processor *MergeToBulkLaneProcessor) Name() string {
+ return "merge_to_bulk_lane"
+}
+
+type MergeConfig struct {
+ InputQueue string `config:"input_queue"`
+ // InputLane is the in-memory input (mutually exclusive with InputQueue):
+ // records arrive as event.Event, their Fields are serialized into
+ // document bytes and take the same aggregation path — written directly
+ // by an upstream output{lane}, in memory end to end (the staging queue
+ // is the only disk landing point).
+ InputLane string `config:"input_lane"`
+ OutputLane string `config:"output_lane"`
+ IndexName string `config:"index_name"`
+ TypeName string `config:"type_name"`
+ WriteOpType string `config:"write_op_type"`
+ KeyField string `config:"key_field"`
+ KeyFields []string `config:"key_fields"`
+
+ BulkSizeInKB int `config:"bulk_size_in_kb"`
+ BulkSizeInMB int `config:"bulk_size_in_mb"`
+ IdleTimeoutInSeconds int `config:"idle_timeout_in_seconds"`
+ NumOfWorkers int `config:"worker_size"`
+
+ // Elasticsearch is only used to default type_name from the cluster's
+ // doc type; the merge itself never talks to Elasticsearch.
+ Elasticsearch string `config:"elasticsearch"`
+
+ FailureQueue string `config:"failure_queue"`
+}
+
+func init() {
+ pipeline.RegisterProcessorPluginWithConfigMetadata("merge_to_bulk_lane", NewMergeToBulkLane, MergeConfig{})
+}
+
+func NewMergeToBulkLane(c *config.Config) (pipeline.Processor, error) {
+ cfg := MergeConfig{
+ BulkSizeInMB: 10,
+ NumOfWorkers: 1,
+ IdleTimeoutInSeconds: 5,
+ }
+
+ if err := c.Unpack(&cfg); err != nil {
+ return nil, fmt.Errorf("failed to unpack the configuration of merge_to_bulk_lane processor: %s", err)
+ }
+
+ if (cfg.InputQueue == "") == (cfg.InputLane == "") {
+ return nil, errors.New("merge_to_bulk_lane processor requires exactly one of input_queue | input_lane")
+ }
+ if cfg.OutputLane == "" {
+ return nil, errors.New("merge_to_bulk_lane processor requires output_lane")
+ }
+ if cfg.IndexName == "" {
+ return nil, errors.New("merge_to_bulk_lane processor requires index_name")
+ }
+
+ if cfg.FailureQueue == "" {
+ fq := cfg.InputQueue
+ if fq == "" {
+ fq = cfg.InputLane
+ }
+ cfg.FailureQueue = fq + "_failure"
+ }
+
+ // Bind to the static lane definition (top-level `lane:` stanza) when
+ // present — its capacity and overflow policy govern this producer.
+ // Without one, create a block-on-full lane with a MODEST capacity:
+ // merged bulks are large, and the 4096-record lane default would bound
+ // memory at capacity × bulk_size (40GB at 10MB bulks) — blocking must
+ // arrive at a realistic ceiling. Define an explicit stanza to tune the
+ // budget (capacity × bulk_size_in_mb is the worst case).
+ outputLane := lane.Get(cfg.OutputLane)
+ var laneCfg lane.LaneConfig
+ if outputLane != nil {
+ laneCfg = outputLane.Config()
+ } else {
+ log.Warnf("merge_to_bulk_lane: lane %q not defined under lane:, creating block-on-full lane with capacity %d; define an explicit stanza to tune the memory budget", cfg.OutputLane, autoLaneCapacity)
+ laneCfg = lane.LaneConfig{Name: cfg.OutputLane, Capacity: autoLaneCapacity}
+ outputLane = lane.GetOrCreate(laneCfg)
+ }
+
+ return &MergeToBulkLaneProcessor{
+ config: cfg,
+ failureQ: queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.FailureQueue}),
+ outputLane: outputLane,
+ laneCfg: laneCfg,
+ }, nil
+}
+
+func (processor *MergeToBulkLaneProcessor) Process(ctx *pipeline.Context) error {
+ defer func() {
+ if !global.Env().IsDebug {
+ if r := recover(); r != nil {
+ var v string
+ switch r := r.(type) {
+ case error:
+ v = r.Error()
+ case runtime.Error:
+ v = r.Error()
+ case string:
+ v = r
+ }
+ log.Error("error in merge_to_bulk_lane,", v)
+ }
+ }
+ }()
+
+ bulkSizeInByte := 1048576 * processor.config.BulkSizeInMB
+ if processor.config.BulkSizeInKB > 0 {
+ bulkSizeInByte = 1024 * processor.config.BulkSizeInKB
+ }
+
+ wg := sync.WaitGroup{}
+ for i := 0; i < processor.config.NumOfWorkers; i++ {
+ wg.Add(1)
+ go processor.newBulkWorker(ctx, bulkSizeInByte, &wg)
+ }
+
+ wg.Wait()
+
+ return nil
+}
+
+func (processor *MergeToBulkLaneProcessor) newBulkWorker(ctx *pipeline.Context, bulkSizeInByte int, wg *sync.WaitGroup) {
+ defer func() {
+ if !global.Env().IsDebug {
+ if r := recover(); r != nil {
+ var v string
+ switch r := r.(type) {
+ case error:
+ v = r.Error()
+ case runtime.Error:
+ v = r.Error()
+ case string:
+ v = r
+ }
+ log.Error("error in merge_to_bulk_lane worker,", v)
+ ctx.RecordError(fmt.Errorf("merge_to_bulk_lane panic: %v", r))
+ }
+ }
+ wg.Done()
+ }()
+
+ log.Trace("start merge_to_bulk_lane bulk worker")
+
+ mainBuf := bytebufferpool.Get("lane_bulk_merge")
+ defer bytebufferpool.Put("lane_bulk_merge", mainBuf)
+
+ idleDuration := time.Duration(processor.config.IdleTimeoutInSeconds) * time.Second
+
+ typeName := processor.config.TypeName
+ if typeName == "" && processor.config.Elasticsearch != "" {
+ // tolerant defaulting: an unavailable cluster leaves _type unset
+ // (legal in the action line) instead of failing the merge task
+ if meta := elastic.GetMetadata(processor.config.Elasticsearch); meta != nil {
+ typeName = common.GetClusterDocType(processor.config.Elasticsearch)
+ }
+ }
+
+ writeOpType := processor.config.WriteOpType
+ if writeOpType == "" {
+ writeOpType = "index"
+ }
+
+ var lastCommit time.Time = time.Now()
+
+ // Input source: with input_lane, subscribe to the lane and serialize
+ // Fields into document bytes; with input_queue, PopTimeout the disk
+ // queue directly (the original path).
+ docs := make(chan []byte, 256)
+ var laneCancel func()
+ if processor.config.InputLane != "" {
+ l := lane.Get(processor.config.InputLane)
+ if l == nil {
+ l = lane.GetOrCreate(lane.LaneConfig{Name: processor.config.InputLane, Capacity: autoLaneCapacity})
+ }
+ laneCancel = l.SubscribeBatch(64, 200*time.Millisecond, func(recs []*event.Event) {
+ for _, rec := range recs {
+ if rec == nil {
+ continue
+ }
+ b, err := util.ToJSONBytes(rec.Fields)
+ if err != nil {
+ log.Warnf("merge_to_bulk_lane: skip record (serialize fields): %v", err)
+ continue
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case docs <- b:
+ }
+ }
+ })
+ defer laneCancel()
+ }
+
+READ_DOCS:
+ for {
+
+ if ctx.IsCanceled() {
+ goto CLEAN_BUFFER
+ }
+
+ var pop []byte
+ if processor.config.InputLane != "" {
+ select {
+ case <-ctx.Done():
+ goto CLEAN_BUFFER
+ case pop = <-docs:
+ }
+ } else {
+ var err error
+ pop, _, err = queue.PopTimeout(queue.GetOrInitConfig(processor.config.InputQueue), idleDuration)
+ if err != nil {
+ log.Error(err)
+ panic(err)
+ }
+ }
+
+ if len(pop) > 0 {
+
+ var id_part = ""
+ if len(processor.config.KeyFields) > 0 {
+ id_part = fmt.Sprintf(", \"_id\":\"%v\"", processor.fingerprintID(pop))
+ } else if processor.config.KeyField != "" {
+ // zero-alloc path query: extracting ONE field
+ // must not float-parse the whole document (embeddings!)
+ v, _, _, err := jsonparser.Get(pop, strings.Split(processor.config.KeyField, ".")...)
+ if err != nil {
+ log.Warnf("merge_to_bulk_lane: key_field %q not found: %v", processor.config.KeyField, err)
+ } else if s, ok := trimJSONString(v); ok {
+ id_part = fmt.Sprintf(", \"_id\":\"%v\"", s)
+ } else {
+ id_part = fmt.Sprintf(", \"_id\":\"%v\"", string(v))
+ }
+ }
+
+ if typeName != "" {
+ mainBuf.WriteString(fmt.Sprintf("{ \"%s\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" %v } }\n", writeOpType, processor.config.IndexName, typeName, id_part))
+ } else {
+ mainBuf.WriteString(fmt.Sprintf("{ \"%s\" : { \"_index\" : \"%s\" %v } }\n", writeOpType, processor.config.IndexName, id_part))
+ }
+
+ util.WalkBytesAndReplace(pop, util.NEWLINE, util.SPACE)
+
+ // Data stream (create) documents must carry a top-level date
+ // @timestamp; the otel envelope keeps the timestamp at the
+ // envelope level and the record body under "payload", which
+ // neither reaches the doc top level. Normalize envelope docs
+ // for data stream semantics; bare docs pass through as-is.
+ if writeOpType == "create" {
+ pop = normalizeDataStreamDoc(pop)
+ }
+
+ mainBuf.Write(pop)
+ mainBuf.WriteString("\n")
+ }
+
+ //submit no matter the size of bulk after idle timeout
+ if time.Since(lastCommit) > idleDuration && mainBuf.Len() > 0 {
+ if global.Env().IsDebug {
+ log.Trace("hit idle timeout, ", idleDuration.String())
+ }
+ goto CLEAN_BUFFER
+ }
+
+ if mainBuf.Len() > (bulkSizeInByte) {
+ if global.Env().IsDebug {
+ log.Trace("hit buffer size, ", mainBuf.Len())
+ }
+ goto CLEAN_BUFFER
+ }
+
+ }
+
+CLEAN_BUFFER:
+
+ lastCommit = time.Now()
+
+ if mainBuf.Len() > 0 {
+ mainBuf.WriteByte('\n')
+
+ // the record outlives this reusable buffer — hand over an
+ // exact-size copy, then reset
+ bulk := make([]byte, mainBuf.Len())
+ copy(bulk, mainBuf.Bytes())
+ mainBuf.Reset()
+
+ rec := newBulkRecord(bulk, "bulk-"+util.GetUUID())
+ // Re-acquire per flush: the consumer task (bulk_indexing_lane)
+ // DELETES the lane on exit; GetOrCreate re-creates it when absent
+ // or closed. A stale cached handle would emit-fail "closed"
+ // forever across the consumer's stop/restart cycle — re-acquiring
+ // here makes that cycle self-healing (bulks buffer in the fresh
+ // lane while the consumer is down, then drain on its restart).
+ processor.outputLane = lane.GetOrCreate(processor.laneCfg)
+ // Emit (not EmitAck): there is no commit gate on this host to
+ // attach receipts to, and Emit is the variant that reports the
+ // closed-lane failure the salvage path below needs
+ if err := processor.outputLane.Emit(rec); err != nil {
+ // closed lane (sender task stopped first): salvage the merged
+ // payload on the failure queue so a replay path exists
+ log.Errorf("merge_to_bulk_lane: emit to lane %q failed: %v", processor.config.OutputLane, err)
+ stats.Increment("merge_to_bulk_lane", "salvaged")
+ if err := queue.Push(processor.failureQ, bulk); err != nil {
+ log.Errorf("merge_to_bulk_lane: push to failure queue %q failed: %v", processor.failureQ.Name, err)
+ }
+ } else {
+ stats.Increment("merge_to_bulk_lane", "emitted")
+ stats.IncrementBy("merge_to_bulk_lane", "emitted_bytes", int64(len(bulk)))
+ }
+
+ log.Trace("clean buffer")
+ }
+
+ if ctx.IsCanceled() {
+ return
+ }
+ goto READ_DOCS
+}
+
+// fingerprintID derives a deterministic document id: the configured dot-path
+// fields are looked up in the queue message, joined and hashed (FNV-1a 64).
+// When NONE of the fields resolve, the raw message bytes are hashed instead —
+// the fallback keeps distinct documents apart for sources lacking e.g. file
+// metadata. Deterministic ids make at-least-once replays idempotent when
+// combined with write_op_type=create: duplicated deliveries turn into
+// ES version conflicts instead of duplicate documents.
+//
+// NOTE the fields must be stable across redeliveries: per-attempt stamps like
+// ingest timestamps change between retries and defeat dedup; content-derived
+// or source-location fields (file path/offset) do not.
+func (processor *MergeToBulkLaneProcessor) fingerprintID(pop []byte) string {
+ h := fnv.New64a()
+
+ // zero-alloc path queries: fingerprinting must not
+ // float-parse the whole document (embeddings ride these messages)
+ matched := false
+ for _, f := range processor.config.KeyFields {
+ h.Write([]byte{'|'})
+ v, _, _, err := jsonparser.Get(pop, strings.Split(f, ".")...)
+ if err != nil {
+ // keep positional meaning for missing fields
+ h.Write([]byte("\x00"))
+ continue
+ }
+ matched = true
+ h.Write(v) // raw JSON token: any scalar type hashes deterministically
+ }
+ if !matched {
+ // none of the key fields exist on this message: fall back to a
+ // content hash so unrelated messages don't collapse into one id
+ h.Reset()
+ h.Write(pop)
+ }
+ return fmt.Sprintf("%x", h.Sum64())
+}
+
+// trimJSONString strips JSON string quotes from a raw token ("" and false
+// when the token is not a quoted string).
+func trimJSONString(v []byte) (string, bool) {
+ if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' {
+ return string(v[1 : len(v)-1]), true
+ }
+ return "", false
+}
+
+// normalizeDataStreamDoc turns one otel envelope (the queue's LogEvent JSON:
+// {"metadata":{...},"payload":{...},"timestamp":"..."}) into a data stream
+// compatible document:
+//
+// - payload fields are promoted to the top level (message etc. become
+// first-class doc fields)
+// - a top-level @timestamp is derived from payload.timestamp /
+// payload.observed_timestamp / the envelope timestamp / now — data
+// streams reject documents without a date @timestamp. @timestamp is the
+// document's one true timestamp (house standard): the mirrored top-level
+// "timestamp" and any payload "timestamp" are folded into it and removed.
+// - a bare document (no "payload" key) keeps its own @timestamp when
+// present; only a missing one is stamped with now.
+// - metadata.file (agent collection origin) lives in @agent.file_* — the
+// former top-level "file" promotion is gone (zero readers).
+//
+// Documents that are not envelopes (no "payload" key) pass through
+// unchanged except for the @timestamp stamp.
+func normalizeDataStreamDoc(doc []byte) []byte {
+ var m util.MapStr
+ if err := util.FromJSONBytes(doc, &m); err != nil {
+ return doc
+ }
+
+ ts := ""
+ if payload, ok := m["payload"].(map[string]interface{}); ok {
+ for _, k := range []string{"timestamp", "@timestamp", "observed_timestamp"} {
+ if v, ok := payload[k].(string); ok && v != "" {
+ ts = v
+ break
+ }
+ }
+ }
+ if ts == "" {
+ if v, ok := m["timestamp"].(string); ok {
+ ts = v
+ }
+ }
+ if ts == "" {
+ if v, ok := m["@timestamp"].(string); ok { // bare doc already carrying one
+ ts = v
+ }
+ }
+ if ts == "" {
+ ts = time.Now().UTC().Format(time.RFC3339Nano)
+ }
+
+ out := util.MapStr{}
+ if payload, ok := m["payload"].(map[string]interface{}); ok {
+ for k, v := range payload {
+ out[k] = v
+ }
+ } else {
+ for k, v := range m {
+ out[k] = v
+ }
+ }
+ out["@timestamp"] = ts
+ delete(out, "timestamp") // @timestamp is the single source of truth
+
+ if meta, ok := m["metadata"].(map[string]interface{}); ok {
+ if rt, ok := meta["log_type"].(string); ok && rt != "" {
+ out["log_type"] = rt
+ }
+ // per-pattern label (e.g. server|deprecation|slowlog|gc) — virtual
+ // streams split on this field.
+ if lk, ok := meta["log_kind"].(string); ok && lk != "" {
+ out["log_kind"] = lk
+ }
+ }
+ return util.MustToJSONBytes(out)
+}
diff --git a/plugins/elastic/lane_bulk/payload.go b/plugins/elastic/lane_bulk/payload.go
new file mode 100644
index 000000000..bd1cd8ac2
--- /dev/null
+++ b/plugins/elastic/lane_bulk/payload.go
@@ -0,0 +1,121 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Framework is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+// - Website: infinilabs.com
+// - Email: hello#infini.ltd
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+/* ©INFINI, All Rights Reserved.
+ * mail: contact#infini.ltd */
+
+// Package lane_bulk provides the in-memory counterpart of the
+// indexing_merge + bulk_indexing pair: merged bulk payloads travel a lane
+// (zero serialization, bounded memory, optional spill-to-disk degradation)
+// instead of round-tripping the output disk queue.
+//
+// merge_to_bulk_lane input disk queue → merged bulk records → lane
+// bulk_indexing_lane lane → Elasticsearch bulk send
+//
+// The two processors are additive: the legacy queue-based pair keeps
+// running unchanged; pipelines opt into the memory path per task.
+package lane_bulk
+
+import (
+ "fmt"
+ "time"
+
+ "infini.sh/framework/core/event"
+ "infini.sh/framework/core/pipeline"
+ "infini.sh/framework/core/util"
+)
+
+// PayloadField is the record field carrying one merged bulk NDJSON payload.
+const PayloadField = "bulk"
+
+// codecName names the record codec for merged bulk payloads.
+const codecName = "bulk"
+
+// bulkPayloadCodec keeps lane records and queue payloads byte-identical:
+// Encode returns the wrapped bulk NDJSON as-is, so a lane configured with
+// `overflow: spill:` degrades records onto a disk queue that the
+// legacy bulk_indexing processor can consume directly (fast/slow paths run
+// the equivalent "send to ES" step, satisfying the spill equivalence rule).
+type bulkPayloadCodec struct{}
+
+func (bulkPayloadCodec) Name() string { return codecName }
+func (bulkPayloadCodec) Namespace() string { return codecName }
+
+func (bulkPayloadCodec) Decode(data []byte) (*event.Event, error) {
+ if len(data) == 0 {
+ return nil, fmt.Errorf("bulk codec: empty payload")
+ }
+ rec := &event.Event{
+ Timestamp: time.Now().UTC(),
+ Fields: util.MapStr{PayloadField: data},
+ Metadata: event.EventMetadata{Labels: util.MapStr{pipeline.CodecLabel: codecName}},
+ }
+ return rec, nil
+}
+
+func (bulkPayloadCodec) Encode(rec *event.Event) ([]byte, error) {
+ if rec == nil {
+ return nil, fmt.Errorf("bulk codec: nil record")
+ }
+ switch v := rec.Fields[PayloadField].(type) {
+ case []byte:
+ return v, nil
+ case string:
+ return []byte(v), nil
+ default:
+ return nil, fmt.Errorf("bulk codec: record field [%s] is not a payload", PayloadField)
+ }
+}
+
+func init() {
+ pipeline.MustRegisterRecordCodec(bulkPayloadCodec{})
+}
+
+// newBulkRecord wraps one merged bulk payload as a lane record. The bytes
+// must be caller-owned (not alias a reusable buffer): the record outlives
+// the flush that produced it. The codec label routes spill encoding and
+// keeps queue round-trips symmetric.
+func newBulkRecord(bulk []byte, traceID string) *event.Event {
+ rec := &event.Event{
+ Timestamp: time.Now().UTC(),
+ Fields: util.MapStr{PayloadField: bulk},
+ Metadata: event.EventMetadata{Labels: util.MapStr{pipeline.CodecLabel: codecName}},
+ }
+ pipeline.SetTraceID(rec, traceID)
+ return rec
+}
+
+// bulkPayloadOf extracts the merged bulk bytes from a lane record.
+func bulkPayloadOf(rec *event.Event) ([]byte, bool) {
+ if rec == nil {
+ return nil, false
+ }
+ switch v := rec.Fields[PayloadField].(type) {
+ case []byte:
+ return v, len(v) > 0
+ case string:
+ return []byte(v), v != ""
+ default:
+ return nil, false
+ }
+}
diff --git a/plugins/elastic/merge_to_bulk/merge_to_bulk.go b/plugins/elastic/merge_to_bulk/merge_to_bulk.go
index 3dc5265e3..d906660fd 100644
--- a/plugins/elastic/merge_to_bulk/merge_to_bulk.go
+++ b/plugins/elastic/merge_to_bulk/merge_to_bulk.go
@@ -86,11 +86,11 @@ func New(c *config.Config) (pipeline.Processor, error) {
}
if cfg.OutputQueue.Name == "" {
- panic(errors.New("name of output_queue can't be nil"))
+ return nil, errors.New("merge_to_bulk processor requires output_queue.name")
}
if cfg.IndexName == "" {
- panic(errors.New("index name can't be nil"))
+ return nil, errors.New("merge_to_bulk processor requires index_name")
}
diff := &IndexingMergeToBulkProcessor{
@@ -102,8 +102,6 @@ func New(c *config.Config) (pipeline.Processor, error) {
if cfg.IndexName != "" {
labels["_index"] = cfg.IndexName
- } else {
- panic("index name is empty")
}
if cfg.TypeName == "" {
@@ -138,7 +136,7 @@ func New(c *config.Config) (pipeline.Processor, error) {
producer, err := queue.AcquireProducer(queueConfig)
if err != nil {
- panic(err)
+ return nil, fmt.Errorf("merge_to_bulk processor failed to acquire output queue producer: %w", err)
}
diff.producer = producer
diff --git a/plugins/http/http.go b/plugins/http/http.go
index 7e73b70fa..a1d1fbb75 100644
--- a/plugins/http/http.go
+++ b/plugins/http/http.go
@@ -119,6 +119,11 @@ func New(c *config.Config) (pipeline.Processor, error) {
if err := c.Unpack(&cfg); err != nil {
return nil, fmt.Errorf("failed to unpack the configuration of http_replicator processor: %s", err)
}
+ // no hosts = the send loop never executes and every record fails;
+ // templates like $[[env.HOSTS]] stay non-empty strings here
+ if len(cfg.Hosts) == 0 {
+ return nil, fmt.Errorf("http processor requires at least one host (hosts)")
+ }
processor := &HTTPProcessor{
config: &cfg,
diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go
index 2a7190d00..af55c2249 100755
--- a/plugins/queue/consumer/consumer.go
+++ b/plugins/queue/consumer/consumer.go
@@ -29,6 +29,7 @@ import (
"infini.sh/framework/core/errors"
"infini.sh/framework/core/locker"
"runtime"
+ "runtime/debug"
"sync"
"time"
@@ -36,9 +37,11 @@ import (
log "github.com/cihub/seelog"
"infini.sh/framework/core/config"
"infini.sh/framework/core/global"
+ "infini.sh/framework/core/lane"
"infini.sh/framework/core/pipeline"
"infini.sh/framework/core/queue"
"infini.sh/framework/core/util"
+ modpipeline "infini.sh/framework/modules/pipeline"
)
type QueueConsumerProcessor struct {
@@ -60,23 +63,23 @@ type MessageHandlerAPI interface {
}
type Config struct {
- NumOfSlices int `config:"num_of_slices"`
- Slices []int `config:"slices"`
- enabledSlice map[int]int
- IdleTimeoutInSecond int `config:"idle_timeout_in_seconds"`
- MaxConnectionPerHost int `config:"max_connection_per_node"`
- QueueLabels map[string]interface{} `config:"queues,omitempty"`
- Selector queue.QueueSelector `config:"queue_selector"`
+ NumOfSlices int `config:"num_of_slices"`
+ Slices []int `config:"slices"`
+ enabledSlice map[int]int
+ IdleTimeoutInSecond int `config:"idle_timeout_in_seconds"`
+ MaxConnectionPerHost int `config:"max_connection_per_node"`
+ QueueLabels map[string]interface{} `config:"queues,omitempty"`
+ Selector queue.QueueSelector `config:"queue_selector"`
// ForceQueueType enforces the backend type (e.g. kafka) of the queues
// consumed by this processor. Used by cross-instance transport segments:
// the writer (Agent) and the reader (Gateway) must land on the same
// queue implementation.
- ForceQueueType string `config:"force_queue_type"`
- Consumer *queue.ConsumerConfig `config:"consumer"`
- MaxWorkers int `config:"max_worker_size"`
- DetectActiveQueue bool `config:"detect_active_queue"`
- DetectIntervalInMs int `config:"detect_interval"`
- QuitDetectAfterIdleInMs int `config:"quite_detect_after_idle_in_ms"`
+ ForceQueueType string `config:"force_queue_type"`
+ Consumer *queue.ConsumerConfig `config:"consumer"`
+ MaxWorkers int `config:"max_worker_size"`
+ DetectActiveQueue bool `config:"detect_active_queue"`
+ DetectIntervalInMs int `config:"detect_interval"`
+ QuitDetectAfterIdleInMs int `config:"quite_detect_after_idle_in_ms"`
MessageProcessors []*config.Config `config:"processor"`
@@ -91,6 +94,16 @@ type Config struct {
WaitingAfter []string `config:"waiting_after"`
RetryDelayIntervalInMs int `config:"retry_delay_interval"`
AutoCommitOffset bool `config:"auto_commit_offset"`
+ // AckTimeoutInMs bounds the commit-gate wait: lane outputs may attach
+ // receipts to the context; the offset only advances when all settle.
+ // Unsettled at timeout = sub-chain failure (redelivery, at-least-once).
+ AckTimeoutInMs int64 `config:"ack_timeout_ms"`
+
+ // Bridge sugar (disk → memory in one block): when OutputLane is set,
+ // fetched payloads decode via Codec and emit into the lane with ack
+ // receipts — replacing the consumer+for_each+lane_output sandwich.
+ OutputLane string `config:"output_lane"`
+ Codec string `config:"codec"`
}
const name = "consumer"
@@ -110,6 +123,7 @@ func New(c *config.Config) (pipeline.Processor, error) {
QueueField: "queue_name",
MessageField: "messages",
AutoCommitOffset: true,
+ AckTimeoutInMs: 30000,
Selector: queue.QueueSelector{
Labels: map[string]interface{}{},
},
@@ -169,7 +183,9 @@ func New(c *config.Config) (pipeline.Processor, error) {
processor, err := pipeline.NewPipeline(runner.config.MessageProcessors)
if err != nil {
- panic(err)
+ // config error, not a runtime fault — surface it so the compile
+ // gate rejects the pipeline instead of the host panicking
+ return nil, fmt.Errorf("%s processor requires a non-empty inner processor chain: %w", name, err)
}
processor.SkipCatchError = true //skip catch internal error
@@ -177,7 +193,7 @@ func New(c *config.Config) (pipeline.Processor, error) {
pool, err := pipeline.NewPoolWithTag(name, runner.config.MaxWorkers)
if err != nil {
- panic(err)
+ return nil, fmt.Errorf("%s processor failed to create worker pool: %w", name, err)
}
runner.pool = pool
@@ -258,7 +274,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error {
var lastDispatch = time.Now()
for {
- if c.IsCanceled() {
+ if c.IsCanceled() || c.IsDraining() {
return
}
@@ -276,7 +292,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error {
//log.Errorf("filter queue by:%v, num of queues:%v", processor.config.Selector.ToString(), len(cfgs))
for _, v := range cfgs {
- if c.IsCanceled() {
+ if c.IsCanceled() || c.IsDraining() {
return
}
@@ -536,7 +552,7 @@ func (processor *QueueConsumerProcessor) NewSlicedWorker(ctx *pipeline.Context,
v = r.(string)
}
if v != "empty queue" {
- log.Errorf("worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v", workerID, qConfig.ID, sliceID, initOffset, offset, v)
+ log.Errorf("worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v\n%s", workerID, qConfig.ID, sliceID, initOffset, offset, v, debug.Stack())
ctx.Failed(fmt.Errorf("panic in slice worker: %+v", r))
}
if parentContext != nil {
@@ -607,11 +623,14 @@ READ_DOCS:
//TODO
//consumerInstance.ResetOffset(queue.ConvertOffset(offset))
- EOF := false
- for {
- if global.ShuttingDown() || ctx.IsCanceled() || ctx.IsFailed() || parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled()) {
- goto CLEAN_BUFFER
- }
+ EOF := false
+ for {
+ // drain (graceful stop): break to CLEAN_BUFFER so the current
+ // offset commits, then return instead of fetching more — the
+ // batch boundary is the safe exit point for maintenance stops
+ if global.ShuttingDown() || ctx.IsCanceled() || ctx.IsFailed() || ctx.IsDraining() || parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled() || parentContext.IsDraining()) {
+ goto CLEAN_BUFFER
+ }
if len(processor.config.WaitingAfter) > 0 {
for _, v := range processor.config.WaitingAfter {
@@ -718,6 +737,13 @@ READ_DOCS:
if err != nil {
panic(err)
}
+ // commit gate: lane receipts must settle before the offset
+ // advances; a timeout is a sub-chain failure (redelivery)
+ if ackSet := pipeline.AckSetOf(&newCtx); ackSet != nil && ackSet.Pending() > 0 {
+ if werr := ackSet.Wait(context.Background(), time.Duration(processor.config.AckTimeoutInMs)*time.Millisecond); werr != nil {
+ panic(werr)
+ }
+ }
// Only consume the batch when the pipeline is not stopping: a
// batch interrupted mid-processing keeps the previous offset so
// it is re-delivered next run (at-least-once).
@@ -782,7 +808,30 @@ CLEAN_BUFFER:
}
log.Tracef("slice_worker, goto READ_DOCS, return on queue:[%v], slice_id:%v", qConfig.Name, sliceID)
- if !ctx.IsCanceled() && !(parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled())) {
+ if !ctx.IsCanceled() && !ctx.IsDraining() && !(parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled() || parentContext.IsDraining())) {
goto READ_DOCS
}
}
+
+// bridgeToLane decodes each fetched payload via the configured codec and
+// emits it into the output lane with an acknowledgment receipt attached
+// to the context's AckSet (the commit gate consumes it upstream).
+func (processor *QueueConsumerProcessor) bridgeToLane(ctx *pipeline.Context, messages []queue.Message) error {
+ l := lane.GetOrCreate(lane.LaneConfig{Name: processor.config.OutputLane})
+ codec := modpipeline.GetRecordCodec(processor.config.Codec)
+ as := pipeline.AttachAckSet(ctx)
+ for i := range messages {
+ if len(messages[i].Data) == 0 {
+ continue
+ }
+ rec, err := codec.Decode(messages[i].Data)
+ if err != nil {
+ return fmt.Errorf("bridge decode failed at offset %v: %w", messages[i].Offset, err)
+ }
+ if pipeline.TraceID(rec) == "" {
+ pipeline.SetTraceID(rec, fmt.Sprintf("tr-%v-%v", messages[i].Offset.Position, time.Now().UnixNano()))
+ }
+ as.AddReceipt(l.EmitAck(rec))
+ }
+ return nil
+}
diff --git a/plugins/replay/replay.go b/plugins/replay/replay.go
index 0d9855efd..5a760750e 100644
--- a/plugins/replay/replay.go
+++ b/plugins/replay/replay.go
@@ -204,8 +204,12 @@ func ReplayLines(req *fasthttp.Request, res *fasthttp.Response, ctx *pipeline.Co
err := execute(req, res, buffer)
if err != nil {
log.Error(err, req.String())
+ req.Reset()
+ res.Reset()
panic(err)
}
+ req.Reset()
+ res.Reset()
buffer.Reset()
requestIsSet = false
}
@@ -256,18 +260,20 @@ func ReplayLines(req *fasthttp.Request, res *fasthttp.Response, ctx *pipeline.Co
err := execute(req, res, buffer)
if err != nil {
log.Error(err, req.String())
+ req.Reset()
+ res.Reset()
panic(err)
}
+ req.Reset()
+ res.Reset()
}
return count, nil, false
}
func execute(req *fasthttp.Request, res *fasthttp.Response, buffer *bytebufferpool.ByteBuffer) error {
- defer func() {
- req.Reset()
- res.Reset()
- }()
+ // reset is the caller's job: it logs the request when execute fails,
+ // and must do so while the request is still intact.
if buffer.Len() > 0 {
if util.ContainStr(string(req.Header.RequestURI()), "_bulk") {
buffer.WriteString(newline)
diff --git a/plugins/smtp/smtp.go b/plugins/smtp/smtp.go
index 68141c43f..16ee3ac71 100644
--- a/plugins/smtp/smtp.go
+++ b/plugins/smtp/smtp.go
@@ -129,6 +129,20 @@ func New(c *config.Config) (pipeline.Processor, error) {
if err := c.Unpack(&cfg); err != nil {
return nil, fmt.Errorf("failed to unpack the configuration of index_diff processor: %s", err)
}
+ if len(cfg.Servers) == 0 {
+ return nil, fmt.Errorf("smtp processor requires at least one server")
+ }
+ for name, sc := range cfg.Servers {
+ if sc == nil {
+ return nil, fmt.Errorf("smtp server %q is empty", name)
+ }
+ if sc.Server.Host == "" {
+ return nil, fmt.Errorf("smtp server %q requires host", name)
+ }
+ if sc.Server.Port <= 0 {
+ return nil, fmt.Errorf("smtp server %q requires a positive port", name)
+ }
+ }
processor := &SMTPProcessor{
config: &cfg,