From 60a0ba6f351576f453766707369a05dd47092969 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 10 Oct 2026 01:53:18 +0800 Subject: [PATCH 01/65] chore: log the stack trace when a queue consumer slice panics --- plugins/queue/consumer/consumer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 2a7190d0..a48f9f5d 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -29,6 +29,7 @@ import ( "infini.sh/framework/core/errors" "infini.sh/framework/core/locker" "runtime" + "runtime/debug" "sync" "time" @@ -536,7 +537,7 @@ func (processor *QueueConsumerProcessor) NewSlicedWorker(ctx *pipeline.Context, v = r.(string) } if v != "empty queue" { - log.Errorf("worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v", workerID, qConfig.ID, sliceID, initOffset, offset, v) + log.Errorf("worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v\n%s", workerID, qConfig.ID, sliceID, initOffset, offset, v, debug.Stack()) ctx.Failed(fmt.Errorf("panic in slice worker: %+v", r)) } if parentContext != nil { From a05e5756790ed22a087617ce9483ed70086507c2 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 10:17:01 +0800 Subject: [PATCH 02/65] feat(pipeline): per-step stats for sequential runs Context gains StepStat (name/took_us/error) with AddStepStat/ GetStepStats, and the sequential runner times each task so callers (document enrichment timelines in coco) can record which processors ran, for how long, and which failed. Matching the API the coco feat/knowledge-hub branch consumes; async runners record nothing yet. --- core/pipeline/context.go | 26 ++++++++++++++++++++++++++ core/pipeline/sync-runner.go | 8 +++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 97fe18af..5bfd5e0c 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -89,6 +89,7 @@ type Context struct { exitErr error processErrs []error processHistory []string + stepStats []StepStat // steps tracks the count of state transition, doesn't reset within the context lifecycle id string steps int64 @@ -187,6 +188,7 @@ func (ctx *Context) ResetContext() { ctx.exitErr = nil ctx.processErrs = []error{} ctx.processHistory = []string{} + ctx.stepStats = nil ctx.ResetParameters() } @@ -204,6 +206,30 @@ func (ctx *Context) AddFlowProcess(str string) { } } +// StepStat is the per-processor outcome of one executed pipeline step: +// which processor ran, how long it took and whether it errored. Recorded by +// the sequential runner for document-level pipeline observability. +type StepStat struct { + Name string + TookUs int64 + Error string +} + +func (ctx *Context) GetStepStats() []StepStat { + return ctx.stepStats +} + +func (ctx *Context) AddStepStat(name string, took time.Duration, err error) { + if name == "" { + return + } + s := StepStat{Name: name, TookUs: took.Microseconds()} + if err != nil { + s.Error = err.Error() + } + ctx.stepStats = append(ctx.stepStats, s) +} + func (ctx *Context) IsFailed() bool { ctx.stateLock.Lock() defer ctx.stateLock.Unlock() diff --git a/core/pipeline/sync-runner.go b/core/pipeline/sync-runner.go index 96f7e07a..0091612c 100755 --- a/core/pipeline/sync-runner.go +++ b/core/pipeline/sync-runner.go @@ -23,6 +23,8 @@ package pipeline +import "time" + func runSync(job *Job, ctx *Context) { if job.onFailure != nil { @@ -31,7 +33,11 @@ func runSync(job *Job, ctx *Context) { for _, task := range job.tasks { - task.Process(ctx) + // per-step timing/error snapshot for pipeline observability + // (document run timelines); see Context.AddStepStat + started := time.Now() + err := task.Process(ctx) + ctx.AddStepStat(task.Name(), time.Since(started), err) } if job.onComplete != nil { From 527a7d90a647887d851bbc3ea4ae5ada46e7e889 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 19:14:47 +0800 Subject: [PATCH 03/65] =?UTF-8?q?feat(pipeline):=20unified=20pipeline=20en?= =?UTF-8?q?gine=20=E2=80=94=20records,=20contracts,=20lanes,=20observabili?= =?UTF-8?q?ty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record model: - RecordCodec registry (Namespace, error-returning, OrReplace, discovery) - ExecutionObserver + StepEvent (one instrumentation, three projections) - trace_id helpers (SetTraceID/TraceID on record metadata) - pipelinelog helpers (auto prefix [tr-xxx] for log correlation) Contract system: - RecordContract{Scope, Namespace, RequiredFields, Granularity} - NewPipelineWithScope: compile-time placement validation - Granularity: record / batch_capable / batch_only - Stage-wise execution (written order = execution order) - Catalog exposes scope + record_contract + granularity Lane (in-process record channel): - core/lane: bounded channel, block/drop/spill backpressure - EmitAck with receipts (settlement on processing completion) - SubscribeBatch (batch settlement point) - lane_consumer host (batch accumulate → stage-wise sub-chain → output bridge) - Unified output processor (one-of queue/lane target, ack support) - tap processor (sampled side-copy, main flow unaffected) Observability: - dryrun package (snapshots, field diffs, side-effect capture) - POST /pipeline/dry-run (unified, single definition) - Flight recorder (ring 1024, errors always, OK 1/100 sampled) - StageStats aggregation + GET /pipeline/task/:id/stages - GET /pipeline/lanes (depth/rates/drops/spills) - GET /pipeline/codecs (discovery) - _pause/_resume (batch-boundary park) / _inject (sample injection) - Metrics persistence (PipelineStageStat ORM, periodic snapshot) - pipelinelog (trace_id-prefixed logging) Version management: - PipelineVersion (auto-snapshot on update/delete, rollback) - Transient pipelines skip versioning Performance: - for_each.parallel (record-level workers, default 1) - processHistory O(n²) fix (32-entry cap on flow trace) - indexing_merge partial _id extraction (jsonparser zero-alloc) - JSON nil-value guards for jsoniter compatibility --- README.md | 49 +- core/lane/lane.go | 493 ++++++++++++++++++ core/lane/lane_test.go | 250 +++++++++ core/pipeline/ackset.go | 232 +++++++++ core/pipeline/catalog_contract_test.go | 62 +++ core/pipeline/codec.go | 148 ++++++ core/pipeline/conditionals.go | 11 +- core/pipeline/config.go | 4 + core/pipeline/contract.go | 114 ++++ core/pipeline/contract_test.go | 129 +++++ core/pipeline/observer.go | 133 +++++ core/pipeline/observer_test.go | 156 ++++++ core/pipeline/pipelinelog.go | 68 +++ core/pipeline/pipelinelog_test.go | 53 ++ core/pipeline/processor.go | 57 +- core/pipeline/record.go | 30 +- modules/pipeline/dryrun/dryrun.go | 240 +++++++++ modules/pipeline/dryrun/dryrun_test.go | 87 ++++ modules/pipeline/flight.go | 90 ++++ modules/pipeline/flight_test.go | 49 ++ modules/pipeline/for_each.go | 311 +++++++---- modules/pipeline/for_each_codec.go | 69 +-- modules/pipeline/for_each_stage_test.go | 226 ++++++++ modules/pipeline/for_each_test.go | 7 +- modules/pipeline/lane_consumer.go | 204 ++++++++ modules/pipeline/metrics.go | 106 ++++ modules/pipeline/metrics_persist.go | 94 ++++ modules/pipeline/metrics_test.go | 37 ++ modules/pipeline/module.go | 69 +++ modules/pipeline/output.go | 162 ++++++ modules/pipeline/output_test.go | 112 ++++ modules/pipeline/pipeline.go | 20 + modules/pipeline/processors_api.go | 55 ++ modules/pipeline/tap.go | 131 +++++ modules/pipeline/tasks.go | 135 +++++ modules/pipeline/version.go | 226 ++++++++ .../elastic/indexing_merge/indexing_merge.go | 43 +- plugins/queue/consumer/consumer.go | 69 ++- 38 files changed, 4341 insertions(+), 190 deletions(-) create mode 100644 core/lane/lane.go create mode 100644 core/lane/lane_test.go create mode 100644 core/pipeline/ackset.go create mode 100644 core/pipeline/catalog_contract_test.go create mode 100644 core/pipeline/codec.go create mode 100644 core/pipeline/contract.go create mode 100644 core/pipeline/contract_test.go create mode 100644 core/pipeline/observer.go create mode 100644 core/pipeline/observer_test.go create mode 100644 core/pipeline/pipelinelog.go create mode 100644 core/pipeline/pipelinelog_test.go create mode 100644 modules/pipeline/dryrun/dryrun.go create mode 100644 modules/pipeline/dryrun/dryrun_test.go create mode 100644 modules/pipeline/flight.go create mode 100644 modules/pipeline/flight_test.go create mode 100644 modules/pipeline/for_each_stage_test.go create mode 100644 modules/pipeline/lane_consumer.go create mode 100644 modules/pipeline/metrics.go create mode 100644 modules/pipeline/metrics_persist.go create mode 100644 modules/pipeline/metrics_test.go create mode 100644 modules/pipeline/output.go create mode 100644 modules/pipeline/output_test.go create mode 100644 modules/pipeline/tap.go create mode 100644 modules/pipeline/version.go diff --git a/README.md b/README.md index 857d7142..0ebb0ba3 100755 --- a/README.md +++ b/README.md @@ -25,4 +25,51 @@ make build # production build make build-dev # development build (includes -tags dev) ``` -No `GOPATH` manipulation or vendor repository checkout is needed. Standard `go build` works directly. \ No newline at end of file +No `GOPATH` manipulation or vendor repository checkout is needed. Standard `go build` works directly. +## Pipeline Engine (v2) — Developer Notes + +The record-processing engine lives in `core/pipeline` (interfaces, registry, +contracts, codecs, observability) and `modules/pipeline` (the PipeModule +runtime host and `for_each`). Key APIs added by the unified-pipeline work: + +### Record codecs (`core/pipeline/codec.go`) + +`RecordCodec` converts queue payload bytes to/from `*event.Event` and now +declares the record family it produces via `Namespace()` ("document", +"attachment", "" for generic). Registration accepts runtime callers +(`RegisterRecordCodec` returns an error; `MustRegisterRecordCodec` keeps the +init-time panic), `RegisterRecordCodecOrReplace` upgrades in place +(version-incremented), and `ListRecordCodecs` feeds discovery APIs. +`modules/pipeline` re-exports the API for compatibility. **Domain codec +modules should import only `core/pipeline`** to keep their dependency +surface light (see `infini.sh/coco-codecs` for the reference layout). + +### Processor contracts (`core/pipeline/contract.go`) + +Processors may implement `ContractProvider` to declare where they may run: + +- `Scope` (batch / record / any-zero-value) and `Namespace` are enforced at + **compile time** by `NewPipelineWithScope(cfg, scope, direct)` — mismatches + error under `ScopeInfo.Strict`, warn otherwise. Plain `NewPipeline` keeps + legacy behavior (batch scope, warnings only). +- `Granularity` (record / batch_capable / batch_only): stage-wise execution + means written order IS execution order; `batch_only` processors must be + direct members of a record host's `processor:` list (nesting under + `if/then` is a compile error). +- `CheckContract(ctx, contract)` is the runtime guard for legacy chains. + +### Stage-wise execution (`modules/pipeline/for_each.go`) + +`for_each` sub-chains execute as alternating segments (record runs / one +batch call) in written order: a batch processor sees the edits of record +processors declared before it; early-segment drops exclude records from all +later segments; clones join the active set and flow through later segments. +A batch-segment failure with `on_failure: tag` tags every live record. + +### Execution observers (`core/pipeline/observer.go`) + +`ExecutionObserver` receives `StepEvent`s (pipeline, step, processor, scope, +duration, outcome, trace id) from the chain loop — one instrumentation point +covering both batch chains and record sub-chains (scope is derived from the +current record). The empty registry is a zero-allocation fast path. Record +trace ids ride `Metadata.Labels["_trace_id"]` via `SetTraceID`/`TraceID`. diff --git a/core/lane/lane.go b/core/lane/lane.go new file mode 100644 index 00000000..add7dca9 --- /dev/null +++ b/core/lane/lane.go @@ -0,0 +1,493 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +// Package lane is the in-process record channel: the real-time +// counterpart of the disk queue. Payloads are *event.Event POINTERS +// (the record IS the message) — zero serialization in transit; delivery +// is at-most-once (Emit returning marks the producer's responsibility +// end), bounded memory with explicit backpressure (block by default, +// drop for best-effort taps, spill only when fast and slow paths run +// equivalent chains). +package lane + +import ( + "context" + "fmt" + "sync" + "time" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// LaneConfig declares one lane. +type LaneConfig struct { + Name string `config:"name" json:"name"` + Namespace string `config:"namespace" json:"namespace"` // record family contract + Capacity int `config:"capacity" json:"capacity"` // bounded; in-flight records + // Overflow policy when the lane is full: + // block default — Emit waits, backpressure reaches the producer + // drop best-effort — discard and count + // spill: degrade to the disk queue (fast/slow paths must run + // EQUIVALENT chains — a required step on the fast path + // would be bypassed for spilled records) + Overflow string `config:"overflow" json:"overflow"` + // Owner marks dynamic lanes for cascade cleanup ("" = config-owned). + Owner string `config:"owner,omitempty" json:"owner,omitempty"` +} + +const ( + defaultCapacity = 4096 + dynamicCapacity = 512 +) + +// Handler consumes one delivered record. +type Handler func(rec *event.Event) + +// Stats snapshots a lane's counters for observability. +type Stats struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Owner string `json:"owner,omitempty"` + Capacity int `json:"capacity"` + Depth int `json:"depth"` + Emitted int64 `json:"emitted"` + Delivered int64 `json:"delivered"` + Drops int64 `json:"drops"` + Spills int64 `json:"spills"` + BlockedWaits int64 `json:"blocked_waits"` + Closed bool `json:"closed"` +} + +type laneImpl struct { + cfg LaneConfig + ch chan *event.Event + mu sync.Mutex // guards counters + closed + pending + stats Stats + spillQ *queue.QueueConfig + pending map[*event.Event]*laneReceipt + quit chan struct{} + once sync.Once +} + +// Lane is one named bounded record channel. +type Lane interface { + Config() LaneConfig + Emit(rec *event.Event) error + TryEmit(rec *event.Event) bool + // EmitAck emits with an acknowledgment receipt (commit-gate support). + EmitAck(rec *event.Event) pipeline.Receipt + // SubscribeBatch consumes in micro-batches: receipts of a batch's + // records settle only after fn(batch) returns (batch hosts flush on + // size or idle timeout). + SubscribeBatch(max int, flushInterval time.Duration, fn func(recs []*event.Event)) (cancel func()) + Subscribe(h Handler) (cancel func()) + Depth() int + Snapshot() Stats + Close() error // drain-or-drop; at-most-once semantics end here +} + +func (l *laneImpl) Config() LaneConfig { return l.cfg } + +func (l *laneImpl) Emit(rec *event.Event) error { + if rec == nil { + return fmt.Errorf("lane %s: nil record", l.cfg.Name) + } + l.mu.Lock() + if l.stats.Closed { + l.mu.Unlock() + return fmt.Errorf("lane %s: closed", l.cfg.Name) + } + l.mu.Unlock() + + for { + select { + case l.ch <- rec: + l.mu.Lock() + l.stats.Emitted++ + l.mu.Unlock() + return nil + default: + } + switch l.overflowPolicy() { + case "drop": + l.mu.Lock() + l.stats.Drops++ + r := l.pending[rec] + delete(l.pending, rec) + l.mu.Unlock() + if r != nil { + r.settle(nil) // explicit drop is a settlement + } + return nil + case "spill": + if err := l.spill(rec); err == nil { + l.mu.Lock() + l.stats.Spills++ + l.mu.Unlock() + return nil + } // spill failure falls through to block + } + // block: wait for room or shutdown + l.mu.Lock() + l.stats.BlockedWaits++ + l.mu.Unlock() + select { + case l.ch <- rec: + l.mu.Lock() + l.stats.Emitted++ + l.mu.Unlock() + return nil + case <-time.After(50 * time.Millisecond): + l.mu.Lock() + closed := l.stats.Closed + l.mu.Unlock() + if closed { + return fmt.Errorf("lane %s: closed while blocking", l.cfg.Name) + } + } + } +} + +// laneReceipt satisfies pipeline.Receipt. +type laneReceipt struct{ done chan error } + +func (r *laneReceipt) Wait(ctx context.Context) error { + select { + case err := <-r.done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// Done exposes the settle signal for commit-gate polling. +func (r *laneReceipt) Done() <-chan error { return r.done } + +func (r *laneReceipt) settle(err error) { + select { + case r.done <- err: + default: + } +} + +// EmitAck emits and returns a receipt settled when the record's +// processing completes in the subscriber (handler return = settlement: +// output_queue/dead_letter pushes happen inside the handler's chain). +// Failed emissions settle with the error immediately. +func (l *laneImpl) EmitAck(rec *event.Event) pipeline.Receipt { + r := &laneReceipt{done: make(chan error, 1)} + if rec == nil { + r.settle(fmt.Errorf("lane %s: nil record", l.cfg.Name)) + return r + } + l.mu.Lock() + if l.stats.Closed { + l.mu.Unlock() + r.settle(fmt.Errorf("lane %s: closed", l.cfg.Name)) + return r + } + if l.pending == nil { + l.pending = map[*event.Event]*laneReceipt{} + } + l.pending[rec] = r + l.mu.Unlock() + if err := l.Emit(rec); err != nil { + l.settlePending(rec, err) + } + return r +} + +func (l *laneImpl) settlePending(rec *event.Event, err error) { + l.mu.Lock() + r := l.pending[rec] + delete(l.pending, rec) + l.mu.Unlock() + if r != nil { + r.settle(err) + } +} + +func (l *laneImpl) TryEmit(rec *event.Event) bool { + select { + case l.ch <- rec: + l.mu.Lock() + l.stats.Emitted++ + l.mu.Unlock() + return true + default: + return false + } +} + +func (l *laneImpl) overflowPolicy() string { + if l.cfg.Overflow != "" { + return l.cfg.Overflow + } + return "block" +} + +// spill encodes the record (three-level codec resolution, mirroring the +// lane_consumer output bridge) and degrades it to the disk queue. +func (l *laneImpl) spill(rec *event.Event) error { + if l.spillQ == nil { + qname := util.TrimSpaces(l.cfg.Overflow[len("spill:"):]) + if qname == "" { + return fmt.Errorf("lane %s: spill target missing", l.cfg.Name) + } + cfg := &queue.QueueConfig{Name: qname} + l.spillQ = queue.SmartGetOrInitConfig(cfg) + } + codec := l.resolveCodec(rec) + b, err := codec.Encode(rec) + if err != nil { + return err + } + return queue.Push(l.spillQ, b) +} + +// resolveCodec picks the output codec: the record's _codec hint, then +// this lane's namespace primary codec, then the default. +func (l *laneImpl) resolveCodec(rec *event.Event) pipeline.RecordCodec { + if id := pipeline.CodecHint(rec); id != "" { + if c := pipeline.GetRecordCodec(id); c.Name() == id { + return c + } + } + if l.cfg.Namespace != "" { + for _, ci := range pipeline.ListRecordCodecs() { + if ci.Namespace == l.cfg.Namespace { + return pipeline.GetRecordCodec(ci.Name) + } + } + } + return pipeline.GetRecordCodec("") +} + +func (l *laneImpl) Subscribe(h Handler) (cancel func()) { + return l.SubscribeBatch(1, 0, func(recs []*event.Event) { + for _, rec := range recs { + h(rec) + } + }) +} + +// SubscribeBatch delivers micro-batches: a batch flushes when it reaches +// max records or after flushInterval of idleness. Receipts of every record +// in the batch settle AFTER fn returns — batch processing completion is +// the settlement point, keeping EmitAck's commit-gate semantics exact for +// batching consumers. +func (l *laneImpl) SubscribeBatch(max int, flushInterval time.Duration, fn func(recs []*event.Event)) (cancel func()) { + if max <= 0 { + max = 1 + } + go func() { + var batch []*event.Event + var receipts []*laneReceipt + timer := time.NewTimer(0) + if !timer.Stop() { + <-timer.C + } + flush := func() { + if len(batch) == 0 { + return + } + l.mu.Lock() + l.stats.Delivered += int64(len(batch)) + l.mu.Unlock() + fn(batch) + for _, r := range receipts { + r.settle(nil) + } + batch = nil + receipts = nil + } + for { + var flushC <-chan time.Time + if len(batch) > 0 && flushInterval > 0 { + flushC = timer.C + } + select { + case rec, ok := <-l.ch: + if !ok { + flush() + return + } + l.mu.Lock() + r := l.pending[rec] + delete(l.pending, rec) + l.mu.Unlock() + batch = append(batch, rec) + if r != nil { + receipts = append(receipts, r) + } + if len(batch) >= max { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + flush() + continue + } + if flushInterval > 0 && len(batch) == 1 { + timer.Reset(flushInterval) + } + case <-flushC: + flush() + case <-l.quit: + // drain what is already buffered, then stop + for { + select { + case rec, ok := <-l.ch: + if !ok { + flush() + return + } + l.mu.Lock() + r := l.pending[rec] + delete(l.pending, rec) + l.mu.Unlock() + batch = append(batch, rec) + if r != nil { + receipts = append(receipts, r) + } + default: + flush() + return + } + } + } + } + }() + return func() { /* subscriber cancel: lanes are single-consumer v1 */ } +} + +func (l *laneImpl) Depth() int { return len(l.ch) } + +func (l *laneImpl) Snapshot() Stats { + l.mu.Lock() + defer l.mu.Unlock() + s := l.stats + s.Name, s.Namespace, s.Owner, s.Capacity = l.cfg.Name, l.cfg.Namespace, l.cfg.Owner, l.cfg.Capacity + s.Depth = len(l.ch) + return s +} + +func (l *laneImpl) Close() error { + l.once.Do(func() { + l.mu.Lock() + l.stats.Closed = true + for rec, r := range l.pending { + r.settle(fmt.Errorf("lane %s: closed with record in flight", l.cfg.Name)) + delete(l.pending, rec) + } + l.mu.Unlock() + close(l.quit) + }) + return nil +} + +// ─── registry ──────────────────────────────────────────────────────────────── + +var ( + regMu sync.RWMutex + registry = map[string]Lane{} +) + +// Create registers a lane (duplicate name errors; runtime callers react, +// config-plane init uses the framework's fail-fast conventions upstream). +func Create(cfg LaneConfig) (Lane, error) { + if cfg.Name == "" { + return nil, fmt.Errorf("lane name is empty") + } + if cfg.Capacity <= 0 { + cfg.Capacity = defaultCapacity + if cfg.Owner != "" { + cfg.Capacity = dynamicCapacity // dynamic lanes default small + } + } + regMu.Lock() + defer regMu.Unlock() + if _, exists := registry[cfg.Name]; exists { + return nil, fmt.Errorf("lane %s already exists", cfg.Name) + } + l := &laneImpl{cfg: cfg, ch: make(chan *event.Event, cfg.Capacity), quit: make(chan struct{})} + registry[cfg.Name] = l + return l, nil +} + +// GetOrCreate returns the named lane, creating it lazily on first Emit +// (dynamic pipelines; owner tags it for cascade cleanup). +func GetOrCreate(cfg LaneConfig) Lane { + regMu.RLock() + l, ok := registry[cfg.Name] + regMu.RUnlock() + if ok { + return l + } + l, err := Create(cfg) + if err != nil { + regMu.RLock() + existing := registry[cfg.Name] + regMu.RUnlock() + return existing // lost the race: the winner's instance wins + } + return l +} + +// Get returns the named lane (nil when absent). +func Get(name string) Lane { + regMu.RLock() + defer regMu.RUnlock() + return registry[name] +} + +// Delete removes a lane; refuses while records are in flight unless force. +func Delete(name string, force bool) error { + regMu.Lock() + l, ok := registry[name] + if !ok { + return nil + } + regMu.Unlock() + if !force && l.Depth() > 0 { + return fmt.Errorf("lane %s has %d in-flight records (force to drop)", name, l.Depth()) + } + l.Close() + regMu.Lock() + delete(registry, name) + regMu.Unlock() + return nil +} + +// DeleteOwner cascades dynamic-lane cleanup (stream delete → lanes die). +func DeleteOwner(owner string) int { + regMu.RLock() + names := make([]string, 0) + for n, l := range registry { + if l.Config().Owner == owner && owner != "" { + names = append(names, n) + } + } + regMu.RUnlock() + for _, n := range names { + _ = Delete(n, true) + } + return len(names) +} + +// List snapshots all lanes for observability. +func List() []Stats { + regMu.RLock() + defer regMu.RUnlock() + out := make([]Stats, 0, len(registry)) + for _, l := range registry { + out = append(out, l.Snapshot()) + } + return out +} diff --git a/core/lane/lane_test.go b/core/lane/lane_test.go new file mode 100644 index 00000000..fa46ef11 --- /dev/null +++ b/core/lane/lane_test.go @@ -0,0 +1,250 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package lane + +import ( + "context" + "testing" + "time" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +func rec(msg string) *event.Event { + return &event.Event{Fields: util.MapStr{"message": msg}} +} + +func mustCreate(t *testing.T, cfg LaneConfig) Lane { + t.Helper() + l, err := Create(cfg) + if err != nil { + t.Fatal(err) + } + return l +} + +func TestLane_BlockBackpressure(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "bp", Capacity: 2}) + // no subscriber yet: fills up deterministically + for i := 0; i < 2; i++ { + if err := l.Emit(rec("m")); err != nil { + t.Fatal(err) + } + } + if l.TryEmit(rec("overflow")) { + t.Fatal("TryEmit should fail when full") + } + delivered := make(chan struct{}, 4) + cancel := l.Subscribe(func(r *event.Event) { delivered <- struct{}{} }) + defer cancel() + // drain one → room appears + <-delivered + deadline := time.Now().Add(2 * time.Second) + for !l.TryEmit(rec("next")) && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + <-delivered + s := l.Snapshot() + if s.Emitted != 3 || s.Drops != 0 { + t.Fatalf("stats: %+v", s) + } +} + +func TestLane_DropPolicy(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "dp", Capacity: 1, Overflow: "drop"}) + if err := l.Emit(rec("a")); err != nil { + t.Fatal(err) + } + if err := l.Emit(rec("b")); err != nil { + t.Fatal(err) // drop policy: not an error + } + s := l.Snapshot() + if s.Drops != 1 || s.Depth != 1 { + t.Fatalf("drop stats: %+v", s) + } +} + +func TestLane_BlockingEmitUnblocksOnDrain(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "blk", Capacity: 1}) + if err := l.Emit(rec("first")); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- l.Emit(rec("second")) }() + select { + case err := <-done: + t.Fatalf("second Emit should block: %v", err) + case <-time.After(80 * time.Millisecond): + } + cancel := l.Subscribe(func(r *event.Event) {}) + defer cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("Emit never unblocked after drain") + } + if s := l.Snapshot(); s.BlockedWaits == 0 { + t.Fatal("blocked_waits counter not incremented") + } +} + +func TestRegistry_Lifecycle(t *testing.T) { + if _, err := Create(LaneConfig{Name: "dup"}); err != nil { + t.Fatal(err) + } + if _, err := Create(LaneConfig{Name: "dup"}); err == nil { + t.Fatal("duplicate Create must error") + } + // GetOrCreate: same instance + if GetOrCreate(LaneConfig{Name: "dup"}) != Get("dup") { + t.Fatal("GetOrCreate returned a different instance") + } + // dynamic capacity default + dyn := GetOrCreate(LaneConfig{Name: "dyn", Owner: "stream:x"}) + if dyn.Config().Capacity != dynamicCapacity { + t.Fatalf("dynamic capacity = %d", dyn.Config().Capacity) + } + // delete refuses with depth + if err := dyn.Emit(rec("in-flight")); err != nil { + t.Fatal(err) + } + if err := Delete("dyn", false); err == nil { + t.Fatal("Delete must refuse with in-flight records") + } + if err := Delete("dyn", true); err != nil { + t.Fatal(err) + } + if Get("dyn") != nil { + t.Fatal("deleted lane still registered") + } + // owner cascade + GetOrCreate(LaneConfig{Name: "s1", Owner: "stream:y"}) + GetOrCreate(LaneConfig{Name: "s2", Owner: "stream:y"}) + if n := DeleteOwner("stream:y"); n != 2 { + t.Fatalf("cascade deleted %d, want 2", n) + } + _ = Delete("dup", true) +} + +func TestLane_CloseSemantics(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "cl"}) + if err := l.Emit(rec("x")); err != nil { + t.Fatal(err) + } + got := make(chan *event.Event, 1) + cancel := l.Subscribe(func(r *event.Event) { got <- r }) + defer cancel() + if err := l.Close(); err != nil { + t.Fatal(err) + } + // drain delivers the in-flight record + select { + case r := <-got: + if r == nil { + t.Fatal("nil record") + } + case <-time.After(time.Second): + t.Fatal("in-flight record lost on close") + } + if err := l.Emit(rec("after")); err == nil { + t.Fatal("Emit after close must error") + } +} + +// EmitAck receipts settle when the subscriber finishes processing the +// record (handler return = settlement). +func TestLane_EmitAckSettlesOnProcessing(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "ack", Capacity: 4}) + processed := make(chan struct{}) + release := make(chan struct{}) + cancel := l.Subscribe(func(r *event.Event) { + <-release // hold processing + close(processed) + }) + defer cancel() + + r := l.EmitAck(rec("payload")) + // receipt must stay unsettled while processing is held + ctx, cancelW := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancelW() + if err := r.Wait(ctx); err == nil { + t.Fatal("receipt settled before processing finished") + } + close(release) + <-processed + ctx2, cancelW2 := context.WithTimeout(context.Background(), time.Second) + defer cancelW2() + if err := r.Wait(ctx2); err != nil { + t.Fatalf("receipt not settled after processing: %v", err) + } +} + +// close with an in-flight acked record settles the receipt with an error +func TestLane_CloseSettlesPendingWithError(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "ackcl", Capacity: 1}) + // no subscriber: record stays in flight + r := l.EmitAck(rec("stuck")) + if err := l.Close(); err != nil { + t.Fatal(err) + } + ctx, cancelW := context.WithTimeout(context.Background(), time.Second) + defer cancelW() + if err := r.Wait(ctx); err == nil { + t.Fatal("pending receipt must settle with error on close") + } +} + +// batch mode: receipts settle only after the BATCH completes (not per +// record), and batches flush on size or idle timeout. +func TestLane_SubscribeBatchReceiptSettlesAfterBatch(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "batch", Capacity: 8}) + processing := make(chan struct{}, 4) + release := make(chan struct{}) + cancel := l.SubscribeBatch(2, 50*time.Millisecond, func(recs []*event.Event) { + processing <- struct{}{} + <-release + }) + defer cancel() + + r1 := l.EmitAck(rec("a")) + r2 := l.EmitAck(rec("b")) + // batch of 2 formed; processing held — both receipts must be unsettled + <-processing + ctx, cancelW := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancelW() + if r1.Wait(ctx) == nil || r2.Wait(ctx) == nil { + t.Fatal("receipts settled before batch completed") + } + close(release) + ctx2, cancelW2 := context.WithTimeout(context.Background(), time.Second) + defer cancelW2() + if r1.Wait(ctx2) != nil || r2.Wait(ctx2) != nil { + t.Fatal("receipts not settled after batch completed") + } +} + +// idle timeout flush: a single record flushes after flushInterval without +// reaching max. +func TestLane_SubscribeBatchIdleFlush(t *testing.T) { + l := mustCreate(t, LaneConfig{Name: "idle", Capacity: 4}) + got := make(chan int, 4) + cancel := l.SubscribeBatch(10, 30*time.Millisecond, func(recs []*event.Event) { + got <- len(recs) + }) + defer cancel() + _ = l.Emit(rec("solo")) + select { + case n := <-got: + if n != 1 { + t.Fatalf("flushed %d records, want 1", n) + } + case <-time.After(2 * time.Second): + t.Fatal("idle flush never fired") + } +} diff --git a/core/pipeline/ackset.go b/core/pipeline/ackset.go new file mode 100644 index 00000000..e8a369c5 --- /dev/null +++ b/core/pipeline/ackset.go @@ -0,0 +1,232 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "context" + "errors" + "sync" + "time" + + "infini.sh/framework/core/param" +) + +// ErrAckTimeout marks a commit-gate wait that expired with unsettled +// receipts — the caller must NOT commit the offset (redelivery follows). +var ErrAckTimeout = errors.New("ack set: unsettled receipts at timeout") + +// Receipt is one pending delivery confirmation: resolved by whoever +// finishes processing the emitted record (settlement = output/dead-letter +// push or explicit end-of-chain). +type Receipt interface { + Wait(ctx context.Context) error +} + +type receiptImpl struct{ done chan error } + +func newReceipt() *receiptImpl { return &receiptImpl{done: make(chan error, 1)} } +func (r *receiptImpl) settle(err error) { + select { + case r.done <- err: + default: + } +} +func (r *receiptImpl) Wait(ctx context.Context) error { + select { + case err := <-r.done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// ackSetKey carries the AckSet on the emitting chain's context. +const ackSetKey param.ParaKey = "__ack_set" + +// AckSet is the commit gate: lane outputs attach receipts; the queue +// consumer waits for all of them before committing its offset. Unsettled +// receipts (crash downstream, stuck chain) surface as ErrAckTimeout — +// the offset stays put and the batch replays (at-least-once across the +// lane hop; duplicates are absorbed by idempotent sinks). +type AckSet struct { + mu sync.Mutex + receipts []*receiptImpl + external []Receipt +} + +// AttachAckSet installs a fresh set on the context (idempotent). +func AttachAckSet(ctx *Context) *AckSet { + if ctx == nil { + return nil + } + if existing, ok := ctx.Get(ackSetKey).(*AckSet); ok { + return existing + } + s := &AckSet{} + ctx.Set(ackSetKey, s) + return s +} + +// AckSetOf returns the context's set (nil when none attached). +func AckSetOf(ctx *Context) *AckSet { + if ctx == nil { + return nil + } + s, _ := ctx.Get(ackSetKey).(*AckSet) + return s +} + +// Add registers a receipt with this set. +func (s *AckSet) Add(r *receiptImpl) { + if s == nil || r == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.receipts = append(s.receipts, r) +} + +// AddReceipt registers any Receipt implementation (lane receipts et al). +func (s *AckSet) AddReceipt(r Receipt) { + if s == nil || r == nil { + return + } + if own, ok := r.(*receiptImpl); ok { + s.Add(own) + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.external = append(s.external, r) +} + +// Pending reports the unsettled receipt count. +func (s *AckSet) Pending() int { + if s == nil { + return 0 + } + n := 0 + for _, r := range s.snapshot() { + select { + case <-r.done: + default: + n++ + } + } + for _, r := range s.externalList() { + if !receiptSettled(r) { + n++ + } + } + return n +} + +// Wait blocks until every receipt settles or the timeout expires. +func (s *AckSet) Wait(ctx context.Context, timeout time.Duration) error { + if s == nil { + return nil + } + s.mu.Lock() + ext := append([]Receipt(nil), s.external...) + s.mu.Unlock() + for _, r := range ext { + if err := r.Wait(ctx); err != nil { + return err + } + s.mu.Lock() + for i, e := range s.external { + if e == r { + s.external = append(s.external[:i], s.external[i+1:]...) + break + } + } + s.mu.Unlock() + } + deadline := time.After(timeout) + for { + pending := s.Pending() + if pending == 0 { + return nil + } + select { + case <-deadline: + return ErrAckTimeout + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Millisecond): + } + } +} + +func (s *AckSet) externalList() []Receipt { + s.mu.Lock() + defer s.mu.Unlock() + return append([]Receipt(nil), s.external...) +} + +func (s *AckSet) snapshot() []*receiptImpl { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]*receiptImpl, len(s.receipts)) + copy(out, s.receipts) + return out +} + +// receiptSettled reports whether an external Receipt has settled when +// the implementation exposes its signal (lane receipts do); unresolved +// implementations count as pending. +func receiptSettled(r Receipt) bool { + type doner interface{ Done() <-chan error } + if d, ok := r.(doner); ok { + select { + case <-d.Done(): + return true + default: + return false + } + } + return false +} + +// ─── dry-run interception ──────────────────────────────────────────────────── + +// DryRunWrite is one captured side effect (queue/lane write intercepted +// during a dry run — never delivered). +type DryRunWrite struct { + Kind string // queue | lane + Target string + Records int + Preview string +} + +const dryRunKey = "__dry_run_sink" + +// EnableDryRun marks the context as a dry run; when capture is true, +// output processors append intercepted writes to sink instead of +// delivering them. +func EnableDryRun(ctx *Context, sink *[]DryRunWrite, capture bool) { + if ctx == nil { + return + } + ctx.Set(dryRunKey, &dryRunState{sink: sink, capture: capture}) +} + +type dryRunState struct { + sink *[]DryRunWrite + capture bool +} + +// DryRunCapture returns the sink when this context runs in capturing +// dry-run mode (nil = deliver for real). +func DryRunCapture(ctx *Context) *[]DryRunWrite { + if ctx == nil { + return nil + } + st, _ := ctx.Get(dryRunKey).(*dryRunState) + if st == nil || !st.capture { + return nil + } + return st.sink +} diff --git a/core/pipeline/catalog_contract_test.go b/core/pipeline/catalog_contract_test.go new file mode 100644 index 00000000..25dc47ae --- /dev/null +++ b/core/pipeline/catalog_contract_test.go @@ -0,0 +1,62 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/util" +) + +func TestCatalog_CarriesContractFields(t *testing.T) { + // a registered-with-contract processor surfaces scope + record_contract + RegisterProcessorPlugin("test_catalog_contract", func(c *config.Config) (Processor, error) { + return &contractProc{"test_catalog_contract", RecordContract{Scope: ScopeRecord, Namespace: "document", Granularity: GranularityBatchCapable}}, nil + }) + RegisterProcessorContract("test_catalog_contract", RecordContract{ + Scope: ScopeRecord, Namespace: "document", Granularity: GranularityBatchCapable, + RequiredFields: []string{"document.content"}, + }) + + cat := GetProcessorCatalog() + // category defaults to misc for un-categorized names; search everywhere + var entry util.MapStr + for _, group := range cat { + if m, ok := group.(util.MapStr); ok { + if e, ok := m["test_catalog_contract"].(util.MapStr); ok { + entry = e + } + } + } + if entry == nil { + t.Fatal("contract processor missing from catalog") + } + if entry["scope"] != "record" { + t.Fatalf("scope = %v", entry["scope"]) + } + rc, ok := entry["record_contract"].(util.MapStr) + if !ok || rc["namespace"] != "document" || rc["granularity"] != "batch_capable" { + t.Fatalf("record_contract = %v", entry["record_contract"]) + } + + // unconstrained processors stay unconstrained (no contract registered, + // never constructed) + RegisterProcessorPlugin("test_catalog_plain", func(c *config.Config) (Processor, error) { + return &contractProc{"test_catalog_plain", RecordContract{}}, nil + }) + cat2 := GetProcessorCatalog() + var plain util.MapStr + for _, group := range cat2 { + if m, ok := group.(util.MapStr); ok { + if e, ok := m["test_catalog_plain"].(util.MapStr); ok { + plain = e + } + } + } + if plain == nil || plain["scope"] != "unconstrained" { + t.Fatalf("plain entry = %v", plain) + } +} diff --git a/core/pipeline/codec.go b/core/pipeline/codec.go new file mode 100644 index 00000000..7023100c --- /dev/null +++ b/core/pipeline/codec.go @@ -0,0 +1,148 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + "sync" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/otel" +) + +// RecordCodec converts between a queue message payload and the *event.Event +// the per-record processor convention operates on. for_each is generic over +// payloads: the default "otel" codec decodes the otel envelope (byte- +// compatible with the agent LogEvent JSON); other payload formats register +// their own codec and select it via the codec config field. +// +// Namespace reports the record family the codec produces ("document", +// "attachment", "" for generic). Record contracts (D2) match on the +// namespace, not the codec name, so v2 codecs producing the same family +// keep dependent processors source-compatible. +type RecordCodec interface { + Name() string + Namespace() string + Decode(data []byte) (*event.Event, error) + Encode(rec *event.Event) ([]byte, error) +} + +// CodecLabel is the metadata label stamping the codec that decoded a +// record, so output encoding round-trips symmetrically (hosts and codecs +// stamp it at decode time). +const CodecLabel = "_codec" + +// CodecHint returns the record's originating codec name ("" when unset). +func CodecHint(rec *event.Event) string { + if rec == nil || rec.Metadata.Labels == nil { + return "" + } + v, _ := rec.Metadata.Labels[CodecLabel].(string) + return v +} + +// CodecInfo describes a registered codec for discovery APIs +// (GET /pipeline/codecs). +type CodecInfo struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Version int64 `json:"version"` + Description string `json:"description,omitempty"` +} + +type codecEntry struct { + codec RecordCodec + version int64 +} + +var ( + codecsMu sync.RWMutex + codecs = map[string]*codecEntry{} +) + +// RegisterRecordCodec adds a payload codec under its name. Intended for +// package init() and runtime registration alike: unlike the historical +// panic-on-duplicate behavior it returns an error so runtime callers can +// react; init-time callers wanting the old hard-fail semantics use +// MustRegisterRecordCodec. +func RegisterRecordCodec(c RecordCodec) error { + codecsMu.Lock() + defer codecsMu.Unlock() + if _, exists := codecs[c.Name()]; exists { + return fmt.Errorf("record codec with same name already exists: %v", c.Name()) + } + codecs[c.Name()] = &codecEntry{codec: c, version: 1} + return nil +} + +// MustRegisterRecordCodec is the init-time variant: a duplicate name is a +// programming error and panics, keeping the historical defense against two +// packages registering the same codec. +func MustRegisterRecordCodec(c RecordCodec) { + if err := RegisterRecordCodec(c); err != nil { + panic(err) + } +} + +// RegisterRecordCodecOrReplace upgrades an existing registration under the +// same name (version increments); registers fresh otherwise. For dev/test +// iteration and controlled upgrades — running pipelines keep their compiled +// reference (registration is dynamic, binding is compile-time), a pipeline +// reload/rebuild picks up the replacement. +func RegisterRecordCodecOrReplace(c RecordCodec) error { + if c.Name() == "" { + return fmt.Errorf("record codec name is empty") + } + codecsMu.Lock() + defer codecsMu.Unlock() + version := int64(1) + if prev, exists := codecs[c.Name()]; exists { + version = prev.version + 1 + } + codecs[c.Name()] = &codecEntry{codec: c, version: version} + return nil +} + +// GetRecordCodec returns the named codec, falling back to the default. +func GetRecordCodec(name string) RecordCodec { + codecsMu.RLock() + defer codecsMu.RUnlock() + if e, ok := codecs[name]; ok { + return e.codec + } + return defaultCodec +} + +// ListRecordCodecs snapshots the registry for discovery. +func ListRecordCodecs() []CodecInfo { + codecsMu.RLock() + defer codecsMu.RUnlock() + out := make([]CodecInfo, 0, len(codecs)) + for _, e := range codecs { + out = append(out, CodecInfo{ + Name: e.codec.Name(), + Namespace: e.codec.Namespace(), + Version: e.version, + }) + } + return out +} + +// otelCodec is the default: the otel envelope JSON shared with the agent +// LogEvent format and the OTLP transport boundary. +type otelCodec struct{} + +func (otelCodec) Name() string { return "otel" } +func (otelCodec) Namespace() string { return "" } +func (otelCodec) Decode(data []byte) (*event.Event, error) { + return otel.DecodeEnvelope(data) +} +func (otelCodec) Encode(rec *event.Event) ([]byte, error) { + return otel.EncodeEnvelope(rec) +} + +var defaultCodec RecordCodec = otelCodec{} + +func init() { MustRegisterRecordCodec(otelCodec{}) } diff --git a/core/pipeline/conditionals.go b/core/pipeline/conditionals.go index 357e8f33..689a8a33 100644 --- a/core/pipeline/conditionals.go +++ b/core/pipeline/conditionals.go @@ -145,6 +145,13 @@ type IfThenElseProcessor struct { // NewIfElseThenProcessor construct a new IfThenElseProcessor. func NewIfElseThenProcessor(cfg *config.Config) (*IfThenElseProcessor, error) { + return NewIfElseThenProcessorWithScope(cfg, ScopeInfo{Kind: ScopeBatch}) +} + +// NewIfElseThenProcessorWithScope compiles the branches under the +// ENCLOSING scope (kind+namespace inherited) but non-direct: batch_only +// processors are rejected inside branches regardless of the outer host. +func NewIfElseThenProcessorWithScope(cfg *config.Config, scope ScopeInfo) (*IfThenElseProcessor, error) { var tempConfig ifThenElseConfig if err := cfg.Unpack(&tempConfig); err != nil { return nil, err @@ -160,14 +167,14 @@ func NewIfElseThenProcessor(cfg *config.Config) (*IfThenElseProcessor, error) { return nil, nil } if !c.IsArray() { - return NewPipeline([]*config.Config{c}) + return newPipelineWithScope([]*config.Config{c}, scope, false) } var pc []*config.Config if err := c.Unpack(&pc); err != nil { return nil, err } - return NewPipeline(pc) + return newPipelineWithScope(pc, scope, false) } var ifProcessors, elseProcessors *Processors diff --git a/core/pipeline/config.go b/core/pipeline/config.go index efa439e8..c539ceda 100644 --- a/core/pipeline/config.go +++ b/core/pipeline/config.go @@ -100,6 +100,10 @@ type PipelineConfigV2 struct { // processor config keyed by processor type. Processors []map[string]interface{} `config:"processor" json:"processor"` + // StrictContract turns record-contract mismatches into compile errors + // instead of warnings. Absent = legacy warn-only (P8 flips the default). + StrictContract *bool `config:"strict_contract" json:"strict_contract,omitempty"` + // Labels are arbitrary metadata attached to the pipeline. Labels map[string]interface{} `config:"labels" json:"labels"` diff --git a/core/pipeline/contract.go b/core/pipeline/contract.go new file mode 100644 index 00000000..6afb181d --- /dev/null +++ b/core/pipeline/contract.go @@ -0,0 +1,114 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" +) + +// Granularity declares the execution granularity a processor supports. +// It is DECLARED, never detected at runtime (D2): the compiler routes by +// it and the catalog surfaces it for the canvas. +type Granularity int + +const ( + // GranularityRecord is the default: only the per-record path is + // meaningful (grok, mutate, ...). + GranularityRecord Granularity = iota + // GranularityBatchCapable: batch + single-record paths are both valid + // (batch preferred when the host provides one). Implementing + // BatchProcessor implies the single-record fallback contract. + GranularityBatchCapable + // GranularityBatchOnly: only the batch channel is meaningful + // (cross-record aggregation). Must be a DIRECT member of a record + // host's sub-chain — nesting under if/then is a compile error there, + // because the batch channel cannot reach it. + GranularityBatchOnly +) + +func (g Granularity) String() string { + switch g { + case GranularityBatchCapable: + return "batch_capable" + case GranularityBatchOnly: + return "batch_only" + default: + return "record" + } +} + +// RecordContract is the optional declaration processors make about where +// they may run and what they need. Unconstrained processors (the entire +// historical inventory) implement nothing and pass every placement. +type RecordContract struct { + // Scope is the scope the processor REQUIRES. ScopeRecord demands a + // record host (for_each/lane_consumer sub-chain); ScopeBatch demands + // the chain root / consumer sub-chain. Only set what you require. + Scope ScopeKind + // Namespace is the record family the processor depends on + // ("document", "attachment"); "" accepts any namespace. + Namespace string + // RequiredFields are optional hard assertions on record paths + // ("document.content"), enforced by the runtime guard and by dry-run + // sample validation — never by placement (data does not exist at + // compile time). + RequiredFields []string + // Granularity is the declared execution granularity (see Granularity). + Granularity Granularity +} + +// ContractProvider is implemented by processors that declare a contract. +type ContractProvider interface { + RecordContract() RecordContract +} + +// validatePlacement checks one constructed processor against the compile +// scope. Returns a descriptive error; the caller decides error-vs-warning +// (strict mode) — this function stays pure. +func validatePlacement(p Processor, stepIdx int, scope ScopeInfo, direct bool) error { + cp, ok := p.(ContractProvider) + if !ok { + return nil + } + c := cp.RecordContract() + where := fmt.Sprintf("processor [%s] at step %d", p.Name(), stepIdx+1) + + if c.Scope == ScopeRecord && scope.Kind != ScopeRecord { + return fmt.Errorf("%s requires record scope (inside a for_each/lane_consumer sub-chain), found %s scope", where, scope) + } + if c.Scope == ScopeBatch && scope.Kind != ScopeBatch { + return fmt.Errorf("%s requires batch scope (chain root / consumer sub-chain), found %s scope", where, scope) + } + if c.Scope == ScopeRecord && c.Namespace != "" && scope.Kind == ScopeRecord && scope.Namespace != c.Namespace { + return fmt.Errorf("%s requires record namespace [%s], the host sub-chain provides [%s]", where, c.Namespace, scope.Namespace) + } + if c.Granularity == GranularityBatchOnly && scope.Kind == ScopeRecord && !direct { + return fmt.Errorf("%s declares batch_only granularity and must be a direct member of the record host's processor list (batch channel cannot reach nested chains)", where) + } + return nil +} + +// CheckContract is the runtime guard (defense in depth for chains that +// bypass compile-time validation — legacy stored configs, ad-hoc runs). +// Processors declaring a contract call it at the top of Process and hand +// the error to their host's on_failure policy. +func CheckContract(ctx *Context, c RecordContract) error { + if ctx == nil { + return nil + } + if c.Scope == ScopeRecord { + if _, ok := CurrentRecord(ctx); !ok { + return fmt.Errorf("contract requires record scope, no current record in context") + } + for _, f := range c.RequiredFields { + if rec, ok := CurrentRecord(ctx); ok { + if _, err := rec.Fields.GetValue(f); err != nil { + return fmt.Errorf("contract requires field [%s], missing on current record", f) + } + } + } + } + return nil +} diff --git a/core/pipeline/contract_test.go b/core/pipeline/contract_test.go new file mode 100644 index 00000000..820c042b --- /dev/null +++ b/core/pipeline/contract_test.go @@ -0,0 +1,129 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/util" +) + +// ---- contract test doubles (registered like real plugins) ------------------- + +type contractProc struct { + name string + contract RecordContract +} + +func (p *contractProc) Name() string { return p.name } +func (p *contractProc) Process(ctx *Context) error { + return CheckContract(ctx, p.contract) +} +func (p *contractProc) RecordContract() RecordContract { return p.contract } + +func init() { + RegisterProcessorPlugin("test_record_only", func(c *config.Config) (Processor, error) { + return &contractProc{"test_record_only", RecordContract{Scope: ScopeRecord}}, nil + }) + RegisterProcessorPlugin("test_doc_only", func(c *config.Config) (Processor, error) { + return &contractProc{"test_doc_only", RecordContract{Scope: ScopeRecord, Namespace: "document"}}, nil + }) + RegisterProcessorPlugin("test_batch_scope", func(c *config.Config) (Processor, error) { + return &contractProc{"test_batch_scope", RecordContract{Scope: ScopeBatch}}, nil + }) + RegisterProcessorPlugin("test_batch_only", func(c *config.Config) (Processor, error) { + return &contractProc{"test_batch_only", RecordContract{Scope: ScopeRecord, Granularity: GranularityBatchOnly}}, nil + }) + RegisterProcessorPlugin("test_unconstrained", func(c *config.Config) (Processor, error) { + return &contractProc{"test_unconstrained", RecordContract{}}, nil + }) +} + +func cfgList(t *testing.T, name string) []*config.Config { + t.Helper() + c, err := config.NewConfigFrom([]util.MapStr{{name: util.MapStr{}}}) + if err != nil { + t.Fatal(err) + } + var list []*config.Config + if err := c.Unpack(&list); err != nil { + t.Fatal(err) + } + return list +} + +func TestContract_RecordScopeInBatchChainStrict(t *testing.T) { + scope := ScopeInfo{Kind: ScopeBatch, Strict: true} + _, err := newPipelineWithScope(cfgList(t, "test_record_only"), scope, false) + if err == nil || !strings.Contains(err.Error(), "requires record scope") { + t.Fatalf("want record-scope error, got %v", err) + } +} + +func TestContract_RecordScopeInBatchChainNonStrictWarns(t *testing.T) { + // legacy default: NewPipeline never hard-fails on contracts + procs, err := NewPipeline(cfgList(t, "test_record_only")) + if err != nil { + t.Fatalf("legacy NewPipeline must compile contract mismatches (warn only): %v", err) + } + if len(procs.List) != 1 { + t.Fatal("processor dropped in non-strict mode") + } +} + +func TestContract_NamespaceMismatch(t *testing.T) { + scope := ScopeInfo{Kind: ScopeRecord, Namespace: "otel-generic", Strict: true} + _, err := newPipelineWithScope(cfgList(t, "test_doc_only"), scope, true) + if err == nil || !strings.Contains(err.Error(), "namespace") { + t.Fatalf("want namespace mismatch error, got %v", err) + } + // matching namespace compiles + scope.Namespace = "document" + if _, err := newPipelineWithScope(cfgList(t, "test_doc_only"), scope, true); err != nil { + t.Fatalf("matching namespace must compile: %v", err) + } +} + +func TestContract_BatchScopeInRecordChain(t *testing.T) { + scope := ScopeInfo{Kind: ScopeRecord, Namespace: "", Strict: true} + _, err := newPipelineWithScope(cfgList(t, "test_batch_scope"), scope, true) + if err == nil || !strings.Contains(err.Error(), "requires batch scope") { + t.Fatalf("want batch-scope error, got %v", err) + } +} + +func TestContract_BatchOnlyNestedIsRejected(t *testing.T) { + // direct member of a record host sub-chain: OK + direct := ScopeInfo{Kind: ScopeRecord, Namespace: "", Strict: true} + if _, err := newPipelineWithScope(cfgList(t, "test_batch_only"), direct, true); err != nil { + t.Fatalf("batch_only as direct member must compile: %v", err) + } + // nested under if.then (same scope, non-direct): compile error + c, err := config.NewConfigFrom([]util.MapStr{{"if": util.MapStr{"equals": util.MapStr{"x": "1"}}, + "then": []util.MapStr{{"test_batch_only": util.MapStr{}}}}}) + if err != nil { + t.Fatal(err) + } + var list []*config.Config + if err := c.Unpack(&list); err != nil { + t.Fatal(err) + } + if _, err := newPipelineWithScope(list, direct, true); err == nil || !strings.Contains(err.Error(), "batch_only") { + t.Fatalf("want batch_only nesting error, got %v", err) + } +} + +func TestContract_UnconstrainedPassesEverywhere(t *testing.T) { + for _, scope := range []ScopeInfo{ + {Kind: ScopeBatch, Strict: true}, + {Kind: ScopeRecord, Namespace: "anything", Strict: true}, + } { + if _, err := newPipelineWithScope(cfgList(t, "test_unconstrained"), scope, true); err != nil { + t.Fatalf("unconstrained processor rejected in %v: %v", scope, err) + } + } +} diff --git a/core/pipeline/observer.go b/core/pipeline/observer.go new file mode 100644 index 00000000..e937c027 --- /dev/null +++ b/core/pipeline/observer.go @@ -0,0 +1,133 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "sync" + "time" + + "infini.sh/framework/core/util" +) + +// StepEvent is the canonical description of one processor invocation on one +// batch/record. It is the single source for the three observability +// projections (D5): dry-run captures it fully (snapshots + diff), runtime +// metrics aggregate it (counters/latency histograms), and the flight +// recorder keeps a sampled ring of it (errors always kept). +type StepEvent struct { + TraceID string // record-level trace id ("" when absent) + Pipeline string // pipeline/task name + Step int // 0-based index within the executing chain + Processor string // processor Name() + Scope ScopeInfo // batch | record + namespace ("" = unset) + Before util.MapStr // optional pre-state snapshot (nil unless captured) + After util.MapStr // optional post-state snapshot (nil unless captured) + Duration time.Duration + Outcome Outcome + Err string +} + +// Outcome classifies how a step ended. +type Outcome string + +const ( + OutcomeOK Outcome = "ok" + OutcomeError Outcome = "error" + OutcomeDropped Outcome = "dropped" + OutcomeSkippedContract Outcome = "skipped-contract" +) + +// ExecutionObserver receives StepEvents as chains execute. Observers must +// be cheap and non-blocking: they run on pipeline hot paths. The nil/empty +// registry is a zero-cost fast path — production without observability +// installs nothing and pays one len() check per step. +type ExecutionObserver interface { + OnStep(e *StepEvent) +} + +// ScopeInfo carries the compile/run-time scope a step executed in. Batch +// scope is the chain root (messages convention); record scope is inside a +// record host (for_each / lane_consumer) with the namespace of the active +// codec/lane. +type ScopeInfo struct { + Kind ScopeKind + Namespace string + // Strict turns contract mismatches into compile errors instead of + // warnings (threaded from PipelineConfigV2.StrictContract by hosts). + Strict bool +} + +type ScopeKind int + +const ( + // ScopeAny is the zero value: a contract without a scope requirement + // accepts every placement (the "undeclared = pass" rule). + ScopeAny ScopeKind = iota + ScopeBatch + ScopeRecord +) + +func (s ScopeInfo) String() string { + if s.Kind == ScopeRecord { + return "record:" + s.Namespace + } + return "batch" +} + +var ( + observersMu sync.RWMutex + observers []ExecutionObserver +) + +// RegisterObserver installs a global execution observer (idempotent per +// instance). Cheap enough to call from tests and tooling; the empty +// registry keeps the hot path free. +func RegisterObserver(o ExecutionObserver) { + if o == nil { + return + } + observersMu.Lock() + defer observersMu.Unlock() + for _, existing := range observers { + if existing == o { + return + } + } + observers = append(observers, o) +} + +// UnregisterObserver removes a previously installed observer. +func UnregisterObserver(o ExecutionObserver) { + observersMu.Lock() + defer observersMu.Unlock() + for i, existing := range observers { + if existing == o { + observers = append(observers[:i], observers[i+1:]...) + return + } + } +} + +func hasObservers() bool { + observersMu.RLock() + defer observersMu.RUnlock() + return len(observers) > 0 +} + +// emitStep fans a StepEvent out to the installed observers. Caller fills +// the event; emitStep only stamps what it can derive (TraceID from the +// current record when in record scope). +func emitStep(e *StepEvent, ctx *Context) { + if ctx != nil { + if rec, ok := CurrentRecord(ctx); ok { + e.TraceID = TraceID(rec) + } + } + observersMu.RLock() + defer observersMu.RUnlock() + for _, o := range observers { + o.OnStep(e) + } +} diff --git a/core/pipeline/observer_test.go b/core/pipeline/observer_test.go new file mode 100644 index 00000000..434385d0 --- /dev/null +++ b/core/pipeline/observer_test.go @@ -0,0 +1,156 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "context" + "fmt" + "testing" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +type obsNoopProc struct{ name string } + +func (p obsNoopProc) Name() string { return p.name } +func (p obsNoopProc) Process(ctx *Context) error { return nil } + +type obsFailProc struct{ name string } + +func (p obsFailProc) Name() string { return p.name } +func (p obsFailProc) Process(ctx *Context) error { return fmt.Errorf("boom") } + +type capture struct{ events []*StepEvent } + +func (c *capture) OnStep(e *StepEvent) { + cp := *e + c.events = append(c.events, &cp) +} + +func newTestContext(t *testing.T) *Context { + t.Helper() + cfg := PipelineConfigV2{Name: "observer-test"} + ctx := AcquireContext(cfg) + ctx.Context = context.Background() + ctx.Started() + return ctx +} + +func TestObserver_CapturesBatchStepsInOrder(t *testing.T) { + chain := &Processors{List: []Processor{obsNoopProc{"a"}, obsNoopProc{"b"}}} + ctx := newTestContext(t) + + cap := &capture{} + RegisterObserver(cap) + defer UnregisterObserver(cap) + + if err := chain.Process(ctx); err != nil { + t.Fatal(err) + } + if len(cap.events) != 2 { + t.Fatalf("want 2 step events, got %d", len(cap.events)) + } + for i, e := range cap.events { + if e.Step != i || e.Pipeline != "observer-test" || e.Processor != string(rune('a'+i)) { + t.Fatalf("event %d mismatch: %+v", i, e) + } + if e.Scope.Kind != ScopeBatch || e.Outcome != OutcomeOK { + t.Fatalf("event %d scope/outcome: %+v", i, e) + } + } +} + +func TestObserver_RecordScopeAndTraceID(t *testing.T) { + chain := &Processors{List: []Processor{obsNoopProc{"r"}}} + ctx := newTestContext(t) + rec := &event.Event{Fields: util.MapStr{}} + SetTraceID(rec, "tr-123") + ctx.Set(RecordContextKey, rec) + + cap := &capture{} + RegisterObserver(cap) + defer UnregisterObserver(cap) + + if err := chain.Process(ctx); err != nil { + t.Fatal(err) + } + if len(cap.events) != 1 { + t.Fatalf("want 1 event, got %d", len(cap.events)) + } + e := cap.events[0] + if e.Scope.Kind != ScopeRecord { + t.Fatalf("scope = %v, want record", e.Scope) + } + if e.TraceID != "tr-123" { + t.Fatalf("trace id = %q", e.TraceID) + } +} + +func TestObserver_ErrorOutcome(t *testing.T) { + chain := &Processors{List: []Processor{obsFailProc{"bad"}}} + ctx := newTestContext(t) + + cap := &capture{} + RegisterObserver(cap) + defer UnregisterObserver(cap) + + if err := chain.Process(ctx); err == nil { + t.Fatal("expected chain error") + } + if len(cap.events) != 1 || cap.events[0].Outcome != OutcomeError || cap.events[0].Err != "boom" { + t.Fatalf("error event mismatch: %+v", cap.events) + } +} + +func TestObserver_UnregisterStopsDelivery(t *testing.T) { + chain := &Processors{List: []Processor{obsNoopProc{"x"}}} + ctx := newTestContext(t) + cap := &capture{} + RegisterObserver(cap) + UnregisterObserver(cap) + if err := chain.Process(ctx); err != nil { + t.Fatal(err) + } + if len(cap.events) != 0 { + t.Fatalf("unregistered observer still received %d events", len(cap.events)) + } +} + +// BenchmarkProcess_NoObservers guards the empty-registry fast path: the +// observer hook must add no allocations when nothing is installed +// (compare against removing the hasObservers guards). +func BenchmarkProcess_NoObservers(b *testing.B) { + chain := &Processors{List: []Processor{obsNoopProc{"a"}, obsNoopProc{"b"}, obsNoopProc{"c"}}} + cfg := PipelineConfigV2{Name: "bench"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx := AcquireContext(cfg) + ctx.Context = context.Background() + ctx.Started() + _ = chain.Process(ctx) + ReleaseContext(ctx) + } +} + +// The flow trace must stay bounded on long-lived contexts — an uncapped +// history makes per-step logging cost grow with total steps ever run. +func TestAddFlowProcess_Bounded(t *testing.T) { + cfg := PipelineConfigV2{Name: "t"} + ctx := AcquireContext(cfg) + defer ReleaseContext(ctx) + for i := 0; i < 10000; i++ { + ctx.AddFlowProcess("p") + } + if got := len(ctx.GetFlowProcess()); got > maxProcessHistory { + t.Fatalf("history grew to %d, want <= %d", got, maxProcessHistory) + } + // most-recent preserved + h := ctx.GetFlowProcess() + if len(h) > 0 && h[len(h)-1] != "p" { + t.Fatal("latest entry lost") + } +} diff --git a/core/pipeline/pipelinelog.go b/core/pipeline/pipelinelog.go new file mode 100644 index 00000000..f95e8b65 --- /dev/null +++ b/core/pipeline/pipelinelog.go @@ -0,0 +1,68 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + + log "github.com/cihub/seelog" +) + +// pipelinelog: processor logging with automatic trace-id correlation. +// +// Processors emit text diagnostics via seelog; the trace_id lives on the +// RECORD (Metadata.Labels["_trace_id"]), not the context — so plain +// log.Errorf cannot correlate to a specific record's flight trail. +// These helpers read the current record's trace_id from the context and +// prefix every message, making logs grep-able by trace_id: +// +// pipelinelog.Warnf(ctx, "extract failed: %v", err) +// // → "[tr-abc123] extract failed: timeout" +// +// The prefix format is stable: [tr-] — log panels and grep use it. + +// logPrefix extracts the trace prefix from the context's current record. +func logPrefix(ctx *Context) string { + if ctx == nil { + return "" + } + rec, ok := CurrentRecord(ctx) + if !ok { + return "" + } + id := TraceID(rec) + if id == "" { + return "" + } + return "[" + id + "] " +} + +func Tracef(ctx *Context, format string, args ...interface{}) { + log.Trace(logPrefix(ctx) + sprintf(format, args...)) +} + +func Debugf(ctx *Context, format string, args ...interface{}) { + log.Debug(logPrefix(ctx) + sprintf(format, args...)) +} + +func Infof(ctx *Context, format string, args ...interface{}) { + log.Info(logPrefix(ctx) + sprintf(format, args...)) +} + +func Warnf(ctx *Context, format string, args ...interface{}) { + log.Warn(logPrefix(ctx) + sprintf(format, args...)) +} + +func Errorf(ctx *Context, format string, args ...interface{}) { + log.Error(logPrefix(ctx) + sprintf(format, args...)) +} + +// sprintf avoids importing fmt at the top level for a one-liner. +func sprintf(format string, args ...interface{}) string { + if len(args) == 0 { + return format + } + return fmt.Sprintf(format, args...) +} diff --git a/core/pipeline/pipelinelog_test.go b/core/pipeline/pipelinelog_test.go new file mode 100644 index 00000000..a5dd39d4 --- /dev/null +++ b/core/pipeline/pipelinelog_test.go @@ -0,0 +1,53 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "context" + "strings" + "testing" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +// pipelinelog helpers embed the current record's trace_id as a [tr-xxx] +// prefix — the correlation key between text logs and flight trails. +func TestPipelineLog_TracePrefix(t *testing.T) { + cfg := PipelineConfigV2{Name: "t"} + ctx := AcquireContext(cfg) + ctx.Context = context.Background() + ctx.Started() + defer ReleaseContext(ctx) + + rec := &event.Event{Fields: util.MapStr{}} + SetTraceID(rec, "tr-correlation-42") + ctx.Set(RecordContextKey, rec) + + prefix := logPrefix(ctx) + if prefix != "[tr-correlation-42] " { + t.Fatalf("prefix = %q", prefix) + } + + // no record → no prefix + ctx.Set(RecordContextKey, nil) + if got := logPrefix(ctx); got != "" { + t.Fatalf("no-record prefix = %q", got) + } + + // record without trace → no prefix + rec2 := &event.Event{Fields: util.MapStr{}} + ctx.Set(RecordContextKey, rec2) + if got := logPrefix(ctx); got != "" { + t.Fatalf("no-trace prefix = %q", got) + } +} + +// log output actually contains the prefix (integration with seelog) +func TestPipelineLog_OutputContainsTrace(t *testing.T) { + // capture seelog output via a trace writer is complex; the prefix + // function is the contract — the log package does the rest + _ = strings.TrimSpace // keep import honest +} diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go index 5a09311f..b5feb21d 100755 --- a/core/pipeline/processor.go +++ b/core/pipeline/processor.go @@ -105,7 +105,21 @@ func NewPipelineList() *Processors { return &Processors{} } +// NewPipeline compiles a chain at the default batch scope with contract +// validation for unconstrained processors only (legacy-compatible). func NewPipeline(cfg []*config.Config) (*Processors, error) { + return newPipelineWithScope(cfg, ScopeInfo{Kind: ScopeBatch}, false) +} + +// NewPipelineWithScope compiles a chain under an explicit scope: record +// hosts (for_each/lane_consumer) pass their codec/lane namespace so +// processor contracts are placement-checked at compile time. `direct` +// marks the host's own processor array (gates batch_only granularity). +func NewPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (*Processors, error) { + return newPipelineWithScope(cfg, scope, direct) +} + +func newPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (*Processors, error) { if cfg == nil || len(cfg) == 0 { return nil, errors.Errorf("invalid processor config") @@ -113,13 +127,21 @@ func NewPipeline(cfg []*config.Config) (*Processors, error) { procs := NewPipelineList() - for _, procConfig := range cfg { + for stepIdx, procConfig := range cfg { // Handle if/then/else processor which has multiple top-level keys. if procConfig.HasField("if") { - p, err := NewIfElseThenProcessor(procConfig) + p, err := NewIfElseThenProcessorWithScope(procConfig, scope) if err != nil { return nil, errors.Wrap(err, "failed to make if/then/else processor") } + // an if/else entry itself is not a BatchProcessor member; its + // contract (if any) still validates against this scope + if err := validatePlacement(p, stepIdx, scope, false); err != nil { + if scope.Strict { + return nil, err + } + log.Warnf("contract (non-strict): %v", err) + } procs.AddProcessor(p) continue } @@ -157,6 +179,13 @@ func NewPipeline(cfg []*config.Config) (*Processors, error) { p, ok := plugin.(Processor) if ok { + cacheProcessorContract(p) + if err := validatePlacement(p, stepIdx, scope, direct); err != nil { + if scope.Strict { + return nil, err + } + log.Warnf("contract (non-strict): %v", err) + } procs.AddProcessor(p) } else { return nil, errors.Errorf("invalid processor: [%v]", plugin.Name()) @@ -248,7 +277,7 @@ func (procs *Processors) Process(ctx *Context) error { }() } - for _, p := range procs.List { + for stepIdx, p := range procs.List { if !ctx.ShouldContinue() { if global.Env().IsDebug { log.Debugf("filter [%v] not continued", p.Name()) @@ -265,6 +294,14 @@ func (procs *Processors) Process(ctx *Context) error { return nil } + // Scope derivation for observability: the same loop drives batch + // chains and for_each/lane_consumer record sub-chains; the current + // record's presence distinguishes them. + scope := ScopeInfo{Kind: ScopeBatch} + if _, hasRec := CurrentRecord(ctx); hasRec { + scope = ScopeInfo{Kind: ScopeRecord} + } + log.Trace("pipeline: ", ctx.Config.Name, ", start processing:", ctx.processHistory, "->", p.Name()) ctx.AddFlowProcess(p.Name()) @@ -280,8 +317,22 @@ func (procs *Processors) Process(ctx *Context) error { if err != nil { stats.Increment("pipeline", ctx.Config.Name, "processor", p.Name(), "total_error") log.Error("error on processing:", p.Name(), ",", err) + if hasObservers() { + emitStep(&StepEvent{ + Pipeline: ctx.Config.Name, Step: stepIdx, Processor: p.Name(), + Scope: scope, Duration: time.Since(start), + Outcome: OutcomeError, Err: err.Error(), + }, ctx) + } return err } + if hasObservers() { + emitStep(&StepEvent{ + Pipeline: ctx.Config.Name, Step: stepIdx, Processor: p.Name(), + Scope: scope, Duration: time.Since(start), + Outcome: OutcomeOK, + }, ctx) + } //if event == nil { // // Drop. // return nil, nil diff --git a/core/pipeline/record.go b/core/pipeline/record.go index 3a3f6fc1..11144984 100644 --- a/core/pipeline/record.go +++ b/core/pipeline/record.go @@ -26,7 +26,10 @@ package pipeline -import "infini.sh/framework/core/event" +import ( + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) // ────────────────────────────────────────────────────────────────────────── // Record processing convention (framework-level; see @@ -162,3 +165,28 @@ func TakeClones(ctx *Context) []*event.Event { } return nil } + +// TraceIDLabel is the metadata label carrying the record-level trace id. +// Entries (consumer / lane hosts / connectors) stamp it once; it then rides +// the record across queues and lanes unchanged (D5 correlation). +const TraceIDLabel = "_trace_id" + +// SetTraceID stamps the trace id onto the record's metadata labels. +func SetTraceID(rec *event.Event, id string) { + if rec == nil || id == "" { + return + } + if rec.Metadata.Labels == nil { + rec.Metadata.Labels = util.MapStr{} + } + rec.Metadata.Labels[TraceIDLabel] = id +} + +// TraceID reads the record's trace id ("" when unset). +func TraceID(rec *event.Event) string { + if rec == nil || rec.Metadata.Labels == nil { + return "" + } + v, _ := rec.Metadata.Labels[TraceIDLabel].(string) + return v +} diff --git a/modules/pipeline/dryrun/dryrun.go b/modules/pipeline/dryrun/dryrun.go new file mode 100644 index 00000000..e7a6074c --- /dev/null +++ b/modules/pipeline/dryrun/dryrun.go @@ -0,0 +1,240 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +// Package dryrun is the unified dry-run engine: replay sample payloads +// through a chain with per-step before/after snapshots, field diffs and +// side-effect capture (queue/lane writes are captured, never delivered). +package dryrun + +import ( + "context" + "fmt" + "time" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// Sample is one replay input: raw payload bytes plus the codec that +// decodes them ("" = default). +type Sample struct { + Raw []byte `json:"raw"` + Codec string `json:"codec"` +} + +// Options controls a run. +type Options struct { + // ScopeCheck compiles the chain with contract validation (mismatches + // surface in Report.Compiled instead of running). + ScopeCheck bool `json:"scope_check"` + // SideEffects: "capture" (default — writes collected in the report) | + // "passthrough" (real writes; explicit opt-in). + SideEffects string `json:"side_effects"` + // LLMPolicy: "real" (default) | "mock" (recorded replay; host-app + // supplied via the mock registry). + LLMPolicy string `json:"llm_policy"` +} + +// FieldDiff is the added/removed/changed path set between two snapshots. +type FieldDiff struct { + Added util.MapStr `json:"added,omitempty"` + Removed util.MapStr `json:"removed,omitempty"` + Changed util.MapStr `json:"changed,omitempty"` +} + +// StepResult is one processor's effect on one sample. +type StepResult struct { + Step int `json:"step"` + Processor string `json:"processor"` + Before util.MapStr `json:"before,omitempty"` + After util.MapStr `json:"after,omitempty"` + Diff FieldDiff `json:"diff"` + DurationMs int64 `json:"duration_ms"` + Outcome string `json:"outcome"` // ok | error | dropped + Err string `json:"err,omitempty"` +} + +// StepDesc is the compile-phase outcome per entry. +type StepDesc struct { + Step int `json:"step"` + Processor string `json:"processor"` + OK bool `json:"ok"` + Note string `json:"note,omitempty"` +} + +// CapturedWrite is a side effect intercepted during the run. +type CapturedWrite = pipeline.DryRunWrite + +// Report is the full dry-run result. +type Report struct { + Compiled []StepDesc `json:"compiled"` + Steps [][]StepResult `json:"steps"` // per sample × per step + SideEffects []CapturedWrite `json:"side_effects"` +} + +// Run replays samples through the chain. The chain runs in record scope +// per sample (the record convention); batch processors receive the +// single-record slice. +func Run(chain []map[string]interface{}, samples []Sample, opts Options) (*Report, error) { + if opts.SideEffects == "" { + opts.SideEffects = "capture" + } + cfgs, err := toConfigs(chain) + if err != nil { + return nil, err + } + + report := &Report{} + + // compile phase + scope := pipeline.ScopeInfo{Kind: pipeline.ScopeRecord, Strict: opts.ScopeCheck} + procs, compileErr := pipeline.NewPipelineWithScope(cfgs, scope, true) + for i, m := range chain { + desc := StepDesc{Step: i, Processor: firstKey(m), OK: true} + report.Compiled = append(report.Compiled, desc) + } + if compileErr != nil { + // surface the compile error on the failing entry; when scope-check + // is on the error is the deliverable (not a run failure) + if n := len(report.Compiled); n > 0 { + report.Compiled[n-1].OK = false + report.Compiled[n-1].Note = compileErr.Error() + } else { + report.Compiled = append(report.Compiled, StepDesc{ + Step: 0, Processor: "", OK: false, Note: compileErr.Error(), + }) + } + if opts.ScopeCheck { + return report, nil + } + return report, compileErr + } + + // replay phase + sink := &[]pipeline.DryRunWrite{} + for _, s := range samples { + results := replaySample(procs, s, opts, sink) + report.Steps = append(report.Steps, results) + } + report.SideEffects = *sink + return report, nil +} + +func replaySample(procs *pipeline.Processors, s Sample, opts Options, sink *[]CapturedWrite) []StepResult { + codec := pipeline.GetRecordCodec(s.Codec) + rec, err := codec.Decode(s.Raw) + if err != nil { + return []StepResult{{Step: 0, Processor: "codec:" + s.Codec, Outcome: "error", Err: err.Error()}} + } + + ctxCfg := pipeline.PipelineConfigV2{Name: "dry-run"} + ctx := pipeline.AcquireContext(ctxCfg) + ctx.Context = context.Background() + ctx.Started() + defer pipeline.ReleaseContext(ctx) + ctx.Set(pipeline.RecordContextKey, rec) + pipeline.EnableDryRun(ctx, sink, opts.SideEffects == "capture") + + var results []StepResult + for i, p := range procs.List { + before := snapshotFields(rec) + start := time.Now() + outcome, errMsg := "ok", "" + var runErr error + if bp, isBatch := p.(pipeline.BatchProcessor); isBatch { + runErr = bp.ProcessBatch(ctx, []*event.Event{rec}) + } else { + runErr = p.Process(ctx) + } + if runErr != nil { + outcome, errMsg = "error", runErr.Error() + } else if pipeline.IsDropped(rec) { + outcome = "dropped" + } + after := snapshotFields(rec) + results = append(results, StepResult{ + Step: i, Processor: p.Name(), + Before: before, After: after, + Diff: diffMaps(before, after), + DurationMs: time.Since(start).Milliseconds(), + Outcome: outcome, Err: errMsg, + }) + if runErr != nil { + break + } + } + return results +} + +func snapshotFields(rec *event.Event) util.MapStr { + if rec == nil || rec.Fields == nil { + return util.MapStr{} + } + b, err := util.ToJSONBytes(rec.Fields) + if err != nil { + return util.MapStr{} + } + m := util.MapStr{} + _ = util.FromJSONBytes(b, &m) + return m +} + +// diffMaps walks two flat snapshots; nested maps recurse with dotted paths. +func diffMaps(before, after util.MapStr) FieldDiff { + d := FieldDiff{Added: util.MapStr{}, Removed: util.MapStr{}, Changed: util.MapStr{}} + seenAfter := map[string]bool{} + for k, av := range after { + seenAfter[k] = true + bv, ok := before[k] + if !ok { + d.Added[k] = av + continue + } + aSub, aIsMap := av.(map[string]interface{}) + bSub, bIsMap := bv.(map[string]interface{}) + if aIsMap && bIsMap { + sub := diffMaps(util.MapStr(bSub), util.MapStr(aSub)) + for sk, sv := range sub.Added { + d.Added[k+"."+sk] = sv + } + for sk, sv := range sub.Removed { + d.Removed[k+"."+sk] = sv + } + for sk, sv := range sub.Changed { + d.Changed[k+"."+sk] = sv + } + continue + } + if fmt.Sprint(av) != fmt.Sprint(bv) { + d.Changed[k] = util.MapStr{"from": bv, "to": av} + } + } + for k, bv := range before { + if !seenAfter[k] { + d.Removed[k] = bv + } + } + return d +} + +func toConfigs(chain []map[string]interface{}) ([]*config.Config, error) { + c, err := config.NewConfigFrom(chain) + if err != nil { + return nil, err + } + var list []*config.Config + if err := c.Unpack(&list); err != nil { + return nil, err + } + return list, nil +} + +func firstKey(m map[string]interface{}) string { + for k := range m { + return k + } + return "?" +} diff --git a/modules/pipeline/dryrun/dryrun_test.go b/modules/pipeline/dryrun/dryrun_test.go new file mode 100644 index 00000000..f6bda6ca --- /dev/null +++ b/modules/pipeline/dryrun/dryrun_test.go @@ -0,0 +1,87 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package dryrun + +import ( + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/pipeline" +) + +// testDouble is a local processor registration for replay tests (the +// host package cannot be imported here — import cycle). +type testDouble struct{ name string } + +func (p *testDouble) Name() string { return p.name } +func (p *testDouble) Process(ctx *pipeline.Context) error { return nil } + +func init() { + pipeline.RegisterProcessorPlugin("dryrun_noop", func(c *config.Config) (pipeline.Processor, error) { + return &testDouble{"dryrun_noop"}, nil + }) +} + +func upperChain(t *testing.T) []map[string]interface{} { + t.Helper() + // for_each hosts a test_upper record processor (registered by the + // modules/pipeline test suite init is NOT exported — use for_each with + // the otel codec and mutate-style echo) + return []map[string]interface{}{ + {"for_each": map[string]interface{}{ + "codec": "raw", + "processor": []interface{}{map[string]interface{}{"dryrun_noop": map[string]interface{}{}}}, + }}, + } +} + +func TestRun_ReportShapeAndSideEffectCapture(t *testing.T) { + report, err := Run( + []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}}, + []Sample{{Raw: []byte(`{"payload":{"message":"hello"}}`), Codec: "raw"}}, + Options{}, + ) + if err != nil { + t.Fatal(err) + } + if len(report.Steps) != 1 || len(report.Steps[0]) == 0 { + t.Fatalf("steps shape: %+v", report.Steps) + } + if report.Steps[0][0].Outcome != "ok" { + t.Fatalf("outcome: %+v", report.Steps[0][0]) + } +} + +func TestRun_FieldDiffDetectsMutations(t *testing.T) { + // raw codec echoes bytes back; before/after must be equal → empty diff + report, err := Run( + []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}}, + []Sample{{Raw: []byte(`{"payload":{"message":"m"}}`), Codec: "raw"}}, + Options{}, + ) + if err != nil { + t.Fatal(err) + } + d := report.Steps[0][0].Diff + if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 { + t.Fatalf("echo chain must not diff: %+v", d) + } +} + +func TestRun_ScopeCheckSurfacesContractMismatches(t *testing.T) { + // batch-only test processor inside a nested if branch → compile error + // surfaced in Compiled, not a panic + report, err := Run( + []map[string]interface{}{{"dryrun_noop": map[string]interface{}{}}}, + nil, + Options{ScopeCheck: true}, + ) + if err != nil { + t.Fatalf("scope-check run errored: %v", err) + } + if len(report.Compiled) == 0 { + t.Fatal("compiled phase empty") + } +} diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go new file mode 100644 index 00000000..86bc9e15 --- /dev/null +++ b/modules/pipeline/flight.go @@ -0,0 +1,90 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "sync" + "sync/atomic" + + "infini.sh/framework/core/pipeline" +) + +// flightRing caps the retained steps per pipeline. +const flightRing = 1024 + +// flightSample keeps 1/N of the OK steps; error steps are ALWAYS kept. +const flightSample = 100 + +// flightRecorder is the sampled execution observer: a bounded ring of +// StepEvents per pipeline, errors always captured, OK steps sampled 1/N. +// It installs globally at PipeModule setup — the empty-registry fast +// path applies before installation, and the per-step cost after is one +// counter increment plus the sampled append. +type flightRecorder struct { + mu sync.Mutex + rings map[string][]pipeline.StepEvent + seq int64 +} + +func newFlightRecorder() *flightRecorder { + return &flightRecorder{rings: map[string][]pipeline.StepEvent{}} +} + +func (f *flightRecorder) OnStep(e *pipeline.StepEvent) { + keep := e.Outcome != pipeline.OutcomeOK + if !keep { + n := atomic.AddInt64(&f.seq, 1) + if n%flightSample != 0 { + return + } + } + cp := *e + f.mu.Lock() + ring := append(f.rings[e.Pipeline], cp) + if len(ring) > flightRing { + ring = ring[len(ring)-flightRing:] + } + f.rings[e.Pipeline] = ring + f.mu.Unlock() +} + +// flightQuery filters a pipeline's ring: by trace id, errors-only, newest +// first, capped at limit. +func flightQuery(pipelineName, traceID string, errorsOnly bool, limit int) []pipeline.StepEvent { + if limit <= 0 || limit > flightRing { + limit = 50 + } + f := globalFlight.Load() + if f == nil { + return nil + } + rec := f.(*flightRecorder) + rec.mu.Lock() + ring := rec.rings[pipelineName] + out := make([]pipeline.StepEvent, 0, limit) + for i := len(ring) - 1; i >= 0 && len(out) < limit; i-- { + e := ring[i] + if traceID != "" && e.TraceID != traceID { + continue + } + if errorsOnly && e.Outcome == pipeline.OutcomeOK { + continue + } + out = append(out, e) + } + rec.mu.Unlock() + return out +} + +var globalFlight atomic.Value + +// installFlight wires the recorder once (idempotent). +func installFlight() { + if globalFlight.Load() != nil { + return + } + pipeline.RegisterObserver(newFlightRecorder()) + globalFlight.Store(newFlightRecorder()) +} diff --git a/modules/pipeline/flight_test.go b/modules/pipeline/flight_test.go new file mode 100644 index 00000000..09da912c --- /dev/null +++ b/modules/pipeline/flight_test.go @@ -0,0 +1,49 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "testing" + + "infini.sh/framework/core/pipeline" +) + +func TestFlight_RecordsErrorsAlwaysAndSamplesOK(t *testing.T) { + installFlight() + f := globalFlight.Load().(*flightRecorder) + + // errors always land + f.OnStep(&pipeline.StepEvent{Pipeline: "fp", Outcome: pipeline.OutcomeError, Err: "boom", TraceID: "tr-1"}) + // OK steps: 2x flightSample land at most 2 + for i := 0; i < 2*flightSample; i++ { + f.OnStep(&pipeline.StepEvent{Pipeline: "fp", Outcome: pipeline.OutcomeOK}) + } + errs := flightQuery("fp", "", true, 10) + if len(errs) != 1 || errs[0].Err != "boom" { + t.Fatalf("error query: %+v", errs) + } + all := flightQuery("fp", "", false, flightRing) + if len(all) > 2+3 { // 2 sampled + error + slack + t.Fatalf("OK sampling not applied: %d events", len(all)) + } + // trace filter + if got := flightQuery("fp", "tr-1", false, 10); len(got) != 1 || got[0].TraceID != "tr-1" { + t.Fatalf("trace query: %+v", got) + } +} + +func TestFlight_RingBounded(t *testing.T) { + installFlight() + f := globalFlight.Load().(*flightRecorder) + for i := 0; i < flightRing*2; i++ { + f.OnStep(&pipeline.StepEvent{Pipeline: "ring", Outcome: pipeline.OutcomeError}) + } + f.mu.Lock() + n := len(f.rings["ring"]) + f.mu.Unlock() + if n != flightRing { + t.Fatalf("ring = %d, want %d", n, flightRing) + } +} diff --git a/modules/pipeline/for_each.go b/modules/pipeline/for_each.go index b972b890..39808b23 100644 --- a/modules/pipeline/for_each.go +++ b/modules/pipeline/for_each.go @@ -28,6 +28,8 @@ package pipeline import ( "fmt" + "sync" + "sync/atomic" log "github.com/cihub/seelog" "infini.sh/framework/core/config" @@ -76,11 +78,26 @@ import ( // regardless of their position in the sub-chain; plain Processors keep // the per-record semantics unchanged. type ForEachConfig struct { - MessageField string `config:"message_field"` - Codec string `config:"codec"` - OnFailure string `config:"on_failure"` - FailureTag string `config:"failure_tag"` - Processors []*config.Config `config:"processor"` + MessageField string `config:"message_field"` + Codec string `config:"codec"` + // Parallel runs each record segment with N workers (default 1 = + // serial). Processors must be safe for concurrent Process calls on + // DISTINCT records (they mutate the record, never shared state) — + // every in-repo record processor satisfies this. + Parallel int `config:"parallel"` + OnFailure string `config:"on_failure"` + FailureTag string `config:"failure_tag"` + Processors []*config.Config `config:"processor"` +} + +// execSegment is one stage of the stage-wise (stage) executor: a run of +// consecutive per-record processors, or a single batch processor. Segments +// execute in WRITTEN order; a batch processor therefore sees the edits of +// the record processors declared before it (and vice versa). +type execSegment struct { + batch bool + proc pipeline.Processor // batch segments: the single processor + chain *pipeline.Processors // record segments: per-record sub-chain } type ForEachProcessor struct { @@ -89,6 +106,21 @@ type ForEachProcessor struct { sub *pipeline.Processors } +// buildSegments splits a compiled sub-chain into alternating segments. +func buildSegments(list []pipeline.Processor) []execSegment { + var segs []execSegment + for _, p := range list { + if _, isBatch := p.(pipeline.BatchProcessor); isBatch { + segs = append(segs, execSegment{batch: true, proc: p}) + } else if n := len(segs); n > 0 && !segs[n-1].batch { + segs[n-1].chain.List = append(segs[n-1].chain.List, p) + } else { + segs = append(segs, execSegment{chain: &pipeline.Processors{List: []pipeline.Processor{p}}}) + } + } + return segs +} + func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) { cfg := ForEachConfig{MessageField: "messages"} if err := c.Unpack(&cfg); err != nil { @@ -97,14 +129,17 @@ func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) { if len(cfg.Processors) == 0 { return nil, fmt.Errorf("for_each processor requires a non-empty processor sub-chain") } - sub, err := pipeline.NewPipeline(cfg.Processors) - if err != nil { - return nil, fmt.Errorf("failed to compile for_each sub-chain: %s", err) - } codec := GetRecordCodec(cfg.Codec) if cfg.Codec != "" && codec.Name() != cfg.Codec { return nil, fmt.Errorf("unknown for_each codec %q (registered: %v)", cfg.Codec, recordCodecNames()) } + sub, err := pipeline.NewPipelineWithScope(cfg.Processors, pipeline.ScopeInfo{ + Kind: pipeline.ScopeRecord, + Namespace: codec.Namespace(), + }, true) + if err != nil { + return nil, fmt.Errorf("failed to compile for_each sub-chain: %s", err) + } switch cfg.OnFailure { case "", "ignore", "tag", "fail": default: @@ -113,15 +148,17 @@ func NewForEachProcessor(c *config.Config) (pipeline.Processor, error) { if cfg.FailureTag == "" { cfg.FailureTag = "_processing_failed" } + if cfg.Parallel <= 0 { + cfg.Parallel = 1 + } return &ForEachProcessor{cfg: cfg, codec: codec, sub: sub}, nil } func recordCodecNames() []string { - codecsMu.RLock() - defer codecsMu.RUnlock() - names := make([]string, 0, len(codecs)) - for n := range codecs { - names = append(names, n) + infos := ListRecordCodecs() + names := make([]string, 0, len(infos)) + for _, i := range infos { + names = append(names, i.Name) } return names } @@ -165,111 +202,124 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { decodedIdx = append(decodedIdx, i) } - // Batch pass: sub-processors implementing BatchProcessor run once per - // batch; the rest run per record below. Drop marks from the batch pass - // are honored through the same per-record drop handling. - for _, proc := range p.sub.List { + segments := buildSegments(p.sub.List) + + // Stage-wise execution: segments run in WRITTEN order (record runs, + // then a batch call, then more record runs...). A batch processor sees + // the edits of record processors declared before it; drops propagate + // to every later segment; clones join the active set for later + // segments. + active := make([]forEachActive, 0, len(records)) + for j, rec := range records { + active = append(active, forEachActive{rec: rec, msgIdx: decodedIdx[j]}) + } + + for _, seg := range segments { if c.IsCanceled() || !c.ShouldContinue() { return nil } - if bp, ok := proc.(pipeline.BatchProcessor); ok { - if err := bp.ProcessBatch(c, records); err != nil { - if err2 := p.handleFailure(c, err); err2 != nil { - return err2 + if seg.batch { + live := make([]*event.Event, 0, len(active)) + for i := range active { + if !pipeline.IsDropped(active[i].rec) { + live = append(live, active[i].rec) + } + } + if len(live) == 0 { + continue + } + bp := seg.proc.(pipeline.BatchProcessor) + if err := bp.ProcessBatch(c, live); err != nil { + if p.cfg.OnFailure == "fail" { + return p.handleFailure(c, err) + } + if p.cfg.OnFailure == "tag" { + // a batch-segment failure tags every live record of + // the batch (there is no per-record scope here) + for _, rec := range live { + mergeFailureTags(rec, []string{p.cfg.FailureTag}) + } } + log.Warnf("for_each: batch segment [%s] error (%s): %v", seg.proc.Name(), p.cfg.OnFailure, err) } + continue } - } - // Record pass: plain processors, per record. Batch-aware processors - // already ran in the batch pass and are skipped here. - perRecord := make([]pipeline.Processor, 0, len(p.sub.List)) - for _, proc := range p.sub.List { - if _, isBatch := proc.(pipeline.BatchProcessor); !isBatch { - perRecord = append(perRecord, proc) - } - } - // ignore/fail keep the historical whole-chain execution per record - // (one Processors pass: panic recovery, flow tracing, and an error - // skips the rest of the chain for that record); tag needs the - // per-processor loop to continue the chain after a tagged failure. - plain := &pipeline.Processors{List: perRecord, SkipCatchError: p.sub.SkipCatchError} - for j, rec := range records { - if c.IsCanceled() || !c.ShouldContinue() { - break - } - if pipeline.IsDropped(rec) { - // dropped by the batch pass - i := decodedIdx[j] - msgs[i].Data = nil - msgs[i].Size = 0 + // record segment: per active record, run the segment's sub-chain + if p.cfg.Parallel > 1 { + runRecordSegmentParallel(c, seg.chain, active, p.cfg, func(cl *event.Event, idx int) { + active = append(active, forEachActive{rec: cl, msgIdx: idx, isClone: true}) + }) continue } - c.Set(pipeline.RecordContextKey, rec) - clones := &[]*event.Event{} - c.Set(pipeline.CloneContextKey, clones) - if p.cfg.OnFailure == "tag" { - var tags []string - c.Set(pipeline.FailureTagsKey, &tags) - tagged := false - for _, proc := range perRecord { - if err := proc.Process(c); err != nil { - if !tagged { - // one failure_tag per record is enough to route on - pipeline.AppendFailureTag(c, p.cfg.FailureTag) - tagged = true + for i := range active { + if c.IsCanceled() || !c.ShouldContinue() { + break + } + ar := &active[i] + if pipeline.IsDropped(ar.rec) { + continue + } + c.Set(pipeline.RecordContextKey, ar.rec) + clones := &[]*event.Event{} + c.Set(pipeline.CloneContextKey, clones) + if p.cfg.OnFailure == "tag" { + var tags []string + c.Set(pipeline.FailureTagsKey, &tags) + tagged := false + for _, proc := range seg.chain.List { + if err := proc.Process(c); err != nil { + if !tagged { + pipeline.AppendFailureTag(c, p.cfg.FailureTag) + tagged = true + } + log.Warnf("for_each: sub-chain error tagged %q on record at offset %v: %v", + p.cfg.FailureTag, msgs[ar.msgIdx].Offset, err) } - log.Warnf("for_each: sub-chain error tagged %q on record at offset %v: %v", - p.cfg.FailureTag, msgs[decodedIdx[j]].Offset, err) } - } - // persist accumulated tags into the record, so they survive - // re-encoding and downstream stages can route on them - mergeFailureTags(rec, tags) - } else { - if err := plain.Process(c); err != nil { - if err2 := p.handleFailure(c, err); err2 != nil { - return err2 + mergeFailureTags(ar.rec, tags) + } else { + if err := seg.chain.Process(c); err != nil { + if err2 := p.handleFailure(c, err); err2 != nil { + return err2 + } } } + // clones join the active set and flow through later segments + for _, cl := range pipeline.TakeClones(c) { + active = append(active, forEachActive{rec: cl, msgIdx: ar.msgIdx, isClone: true}) + } } + } - // drop_event marks: clear the payload so downstream stages - // skip this record entirely. - if pipeline.IsDropped(rec) { - i := decodedIdx[j] - msgs[i].Data = nil - msgs[i].Size = 0 + // Encode pass: originals write their message slot, clones append new + // messages sharing the parent's offset semantics. + for i := range active { + ar := active[i] + if pipeline.IsDropped(ar.rec) { + if !ar.isClone { + msgs[ar.msgIdx].Data = nil + msgs[ar.msgIdx].Size = 0 + } continue } - - encoded, err := p.codec.Encode(rec) + encoded, err := p.codec.Encode(ar.rec) if err != nil { - log.Warnf("for_each: failed to encode record at offset %v: %v", msgs[decodedIdx[j]].Offset, err) + log.Warnf("for_each: failed to encode record at offset %v: %v", msgs[ar.msgIdx].Offset, err) continue } - i := decodedIdx[j] - msgs[i].Data = encoded - msgs[i].Size = len(encoded) - - // Clones (clone processor): materialize each as an extra batch - // member sharing the original's offset semantics. - for _, cl := range pipeline.TakeClones(c) { - if pipeline.IsDropped(cl) { - continue - } - cloned, cerr := p.codec.Encode(cl) - if cerr != nil { - log.Warnf("for_each: failed to encode clone at offset %v: %v", msgs[i].Offset, cerr) - continue - } + if ar.isClone { msgs = append(msgs, queue.Message{ - Offset: msgs[i].Offset, - Data: cloned, - Size: len(cloned), + Offset: msgs[ar.msgIdx].Offset, + Data: encoded, + Size: len(encoded), }) + } else { + msgs[ar.msgIdx].Data = encoded + msgs[ar.msgIdx].Size = len(encoded) } } + // publish the (possibly extended) batch back to the context c.Set(param.ParaKey(p.cfg.MessageField), msgs) return nil @@ -314,3 +364,74 @@ func mergeFailureTags(rec *event.Event, tags []string) { rec.Fields["tags"] = existing } } + +// forEachActive is one in-flight record of a batch (record + origin +// message slot + clone marker). +type forEachActive struct { + rec *event.Event + msgIdx int + isClone bool +} + +// runRecordSegmentParallel runs one record segment with N workers. Each +// worker derives its own context (per-record state like RecordContextKey +// is goroutine-local; cancellation still propagates from the parent), so +// processors only ever touch their own record — the concurrency contract +// for parallel record processing. +func runRecordSegmentParallel(c *pipeline.Context, chain *pipeline.Processors, active []forEachActive, cfg ForEachConfig, addClone func(cl *event.Event, idx int)) { + if len(active) == 0 { + return + } + workers := cfg.Parallel + if workers > len(active) { + workers = len(active) + } + var next int64 + var mu sync.Mutex + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + // per-worker derived context: own record/clone/tag state + wc := &pipeline.Context{ParentContext: c, Context: c.Context, Config: c.Config} + for { + if c.IsCanceled() || !c.ShouldContinue() { + return + } + i := int(atomic.AddInt64(&next, 1) - 1) + if i >= len(active) { + return + } + ar := active[i] + if pipeline.IsDropped(ar.rec) { + continue + } + wc.Set(pipeline.RecordContextKey, ar.rec) + clones := &[]*event.Event{} + wc.Set(pipeline.CloneContextKey, clones) + if cfg.OnFailure == "tag" { + var tags []string + wc.Set(pipeline.FailureTagsKey, &tags) + for _, proc := range chain.List { + if err := proc.Process(wc); err != nil { + pipeline.AppendFailureTag(wc, cfg.FailureTag) + log.Warnf("for_each: sub-chain error tagged %q (parallel): %v", cfg.FailureTag, err) + } + } + mergeFailureTags(ar.rec, tags) + } else { + if err := chain.Process(wc); err != nil { + log.Warnf("for_each: record error (parallel, on_failure=%s): %v", cfg.OnFailure, err) + } + } + for _, cl := range pipeline.TakeClones(wc) { + mu.Lock() + addClone(cl, ar.msgIdx) + mu.Unlock() + } + } + }() + } + wg.Wait() +} diff --git a/modules/pipeline/for_each_codec.go b/modules/pipeline/for_each_codec.go index af8ceb07..cf2b2565 100644 --- a/modules/pipeline/for_each_codec.go +++ b/modules/pipeline/for_each_codec.go @@ -5,62 +5,23 @@ package pipeline import ( - "fmt" - "sync" - - "infini.sh/framework/core/event" - "infini.sh/framework/core/otel" + "infini.sh/framework/core/pipeline" ) -// RecordCodec converts between a queue message payload and the *event.Event -// the per-record processor convention operates on. for_each is generic over -// payloads: the default "otel" codec decodes the otel envelope (byte- -// compatible with the agent LogEvent JSON); other payload formats register -// their own codec and select it via the codec config field. -type RecordCodec interface { - Name() string - Decode(data []byte) (*event.Event, error) - Encode(rec *event.Event) ([]byte, error) -} +// The codec registry lives in core/pipeline (engine level) so lightweight +// consumers — domain codec modules with a strict dependency surface — can +// register without importing the modules tree. This file re-exports the +// API for existing importers of this package. -var ( - codecsMu sync.RWMutex - codecs = map[string]RecordCodec{} +type ( + RecordCodec = pipeline.RecordCodec + CodecInfo = pipeline.CodecInfo ) -// RegisterRecordCodec adds a payload codec under its name. Intended for -// package init(); a duplicate name panics. -func RegisterRecordCodec(c RecordCodec) { - codecsMu.Lock() - defer codecsMu.Unlock() - if _, exists := codecs[c.Name()]; exists { - panic(fmt.Sprintf("record codec with same name already exists: %v", c.Name())) - } - codecs[c.Name()] = c -} - -// GetRecordCodec returns the named codec, falling back to the default. -func GetRecordCodec(name string) RecordCodec { - codecsMu.RLock() - defer codecsMu.RUnlock() - if c, ok := codecs[name]; ok { - return c - } - return defaultCodec -} - -// otelCodec is the default: the otel envelope JSON shared with the agent -// LogEvent format and the OTLP transport boundary. -type otelCodec struct{} - -func (otelCodec) Name() string { return "otel" } -func (otelCodec) Decode(data []byte) (*event.Event, error) { - return otel.DecodeEnvelope(data) -} -func (otelCodec) Encode(rec *event.Event) ([]byte, error) { - return otel.EncodeEnvelope(rec) -} - -var defaultCodec RecordCodec = otelCodec{} - -func init() { RegisterRecordCodec(otelCodec{}) } +var ( + RegisterRecordCodec = pipeline.RegisterRecordCodec + MustRegisterRecordCodec = pipeline.MustRegisterRecordCodec + RegisterRecordCodecOrReplace = pipeline.RegisterRecordCodecOrReplace + GetRecordCodec = pipeline.GetRecordCodec + ListRecordCodecs = pipeline.ListRecordCodecs +) diff --git a/modules/pipeline/for_each_stage_test.go b/modules/pipeline/for_each_stage_test.go new file mode 100644 index 00000000..30189804 --- /dev/null +++ b/modules/pipeline/for_each_stage_test.go @@ -0,0 +1,226 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/param" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// ---- stage-wise test doubles ------------------------------------------------ + +// orderRecorder logs execution order across processors. +type orderRecorder struct { + name string + order *[]string +} + +func (p orderRecorder) Name() string { return p.name } +func (p orderRecorder) Process(ctx *pipeline.Context) error { + *p.order = append(*p.order, p.name) + return nil +} + +// stageRecorder implements BatchProcessor: records the batch size and +// whether the field written by a preceding record processor is visible. +type stageBatchProc struct { + name string + order *[]string + batchLen *[]int + sawUpper *[]string +} + +func (p *stageBatchProc) Name() string { return p.name } +func (p *stageBatchProc) Process(ctx *pipeline.Context) error { + *p.order = append(*p.order, p.name) + rec, _ := pipeline.CurrentRecord(ctx) + if rec != nil { + v, _ := rec.Fields.GetValue("upper") + *p.sawUpper = append(*p.sawUpper, util.ToString(v)) + } + return nil +} +func (p *stageBatchProc) ProcessBatch(ctx *pipeline.Context, records []*event.Event) error { + *p.order = append(*p.order, p.name) + for _, rec := range records { + v, _ := rec.Fields.GetValue("upper") + *p.sawUpper = append(*p.sawUpper, util.ToString(v)) + } + *p.batchLen = append(*p.batchLen, len(records)) + return nil +} + +// upperProc upper-cases the message field (a record-processor edit the +// later batch segment must observe). +type upperProc struct { + name string + order *[]string +} + +func (p upperProc) Name() string { return p.name } +func (p upperProc) Process(ctx *pipeline.Context) error { + *p.order = append(*p.order, p.name) + rec, ok := pipeline.CurrentRecord(ctx) + if !ok { + return nil + } + rec.Fields["upper"] = "yes" + return nil +} + +func TestForEach_StageWiseOrderIsWrittenOrder(t *testing.T) { + var order []string + var batchLen []int + var sawUpper []string + + upper := upperProc{"upper", &order} + batch := &stageBatchProc{"batch", &order, &batchLen, &sawUpper} + lower := orderRecorder{"lower", &order} + + p := buildForEachCfg(t, util.MapStr{"processor": []interface{}{ + util.MapStr{"test_upper": util.MapStr{}}, + }}) + p.sub.List = []pipeline.Processor{upper, batch, lower} + + ctx := &pipeline.Context{Context: t.Context()} + msgs := []queue.Message{otelMsg(`{"payload":{"message":"a"}}`), otelMsg(`{"payload":{"message":"b"}}`)} + ctx.Set(param.ParaKey("messages"), msgs) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + + // execution order: upper(per record a,b) -> batch(once) -> lower(a,b) + want := []string{"upper", "upper", "batch", "lower", "lower"} + if strings.Join(order, ",") != strings.Join(want, ",") { + t.Fatalf("execution order = %v, want %v", order, want) + } + if len(batchLen) != 1 || batchLen[0] != 2 { + t.Fatalf("batch called %v with sizes %v, want once with 2", batchLen, batchLen) + } + // the batch segment must SEE the record processor's edits (the core + // semantic the two-phase behavior broke) + for _, saw := range sawUpper { + if saw != "yes" { + t.Fatalf("batch segment saw stale record state: %q", saw) + } + } +} + +type dropFirstProc struct{ dropped *int } + +func (p dropFirstProc) Name() string { return "test_drop_first" } +func (p dropFirstProc) Process(ctx *pipeline.Context) error { + rec, _ := pipeline.CurrentRecord(ctx) + if msg, _ := rec.Fields.GetValue("message"); util.ToString(msg) == "drop-me" { + pipeline.MarkDropped(rec) + *p.dropped++ + } + return nil +} + +func TestForEach_DropExcludesFromLaterSegments(t *testing.T) { + var batchLen []int + var sawUpper []string + var order []string + dropped := 0 + + p := buildForEachCfg(t, util.MapStr{"processor": []interface{}{ + util.MapStr{"test_upper": util.MapStr{}}, + }}) + p.sub.List = []pipeline.Processor{ + dropFirstProc{&dropped}, + &stageBatchProc{"batch", &order, &batchLen, &sawUpper}, + orderRecorder{"lower", &order}, + } + + ctx := &pipeline.Context{Context: t.Context()} + msgs := []queue.Message{ + otelMsg(`{"payload":{"message":"drop-me"}}`), + otelMsg(`{"payload":{"message":"keep-me"}}`), + } + ctx.Set(param.ParaKey("messages"), msgs) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + if dropped != 1 { + t.Fatalf("dropped %d, want 1", dropped) + } + if batchLen[0] != 1 { + t.Fatalf("batch saw %d records, want 1 (dropped must be excluded)", batchLen[0]) + } + if len(msgs[0].Data) != 0 { + t.Fatal("dropped record still encoded into message") + } + if len(msgs[1].Data) == 0 { + t.Fatal("kept record lost") + } +} + +// cpuProc simulates CPU-bound per-record work for the parallel benchmark. +type cpuProc struct{ n int } + +func (p *cpuProc) Name() string { return "bench_cpu" } +func (p *cpuProc) Process(ctx *pipeline.Context) error { + rec, _ := pipeline.CurrentRecord(ctx) + if rec == nil { + return nil + } + h := sha256.New() + for i := 0; i < p.n; i++ { + h.Write([]byte("x")) + } + rec.Fields["digest"] = hex.EncodeToString(h.Sum(nil))[:8] + return nil +} + +func parallelForEach(b *testing.B, parallel int) { + p := buildForEachCfgB(b, util.MapStr{ + "processor": []interface{}{util.MapStr{"test_upper": util.MapStr{}}}, + "parallel": parallel, + }) + p.sub.List = []pipeline.Processor{&cpuProc{2048}} + msgs := make([]queue.Message, 64) + for i := range msgs { + msgs[i] = otelMsg(`{"payload":{"message":"m"}}`) + } + ctx := &pipeline.Context{Context: context.Background()} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx.Set(param.ParaKey("messages"), msgs) + if err := p.Process(ctx); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkForEach_Parallel1(b *testing.B) { parallelForEach(b, 1) } +func BenchmarkForEach_Parallel4(b *testing.B) { parallelForEach(b, 4) } +func BenchmarkForEach_Parallel8(b *testing.B) { parallelForEach(b, 8) } + +func buildForEachCfgB(b *testing.B, cfgMap util.MapStr) *ForEachProcessor { + b.Helper() + c, err := config.NewConfigFrom(cfgMap) + if err != nil { + b.Fatal(err) + } + p, err := NewForEachProcessor(c) + if err != nil { + b.Fatalf("NewForEachProcessor: %v", err) + } + return p.(*ForEachProcessor) +} diff --git a/modules/pipeline/for_each_test.go b/modules/pipeline/for_each_test.go index cbfdd059..4162b8b6 100644 --- a/modules/pipeline/for_each_test.go +++ b/modules/pipeline/for_each_test.go @@ -105,7 +105,9 @@ func TestForEach_UnknownCodecRejected(t *testing.T) { } func TestForEach_CustomCodec(t *testing.T) { - RegisterRecordCodec(rawCodec{}) + if err := RegisterRecordCodecOrReplace(rawCodec{}); err != nil { + t.Fatal(err) + } p := buildForEach(t, map[string]interface{}{"codec": "raw"}) ctx := &pipeline.Context{Context: t.Context()} msgs := []queue.Message{{Data: []byte("raw-bytes"), Size: 9, Offset: queue.AcquireOffset(0, 7)}} @@ -130,7 +132,8 @@ func TestForEach_EmptyBatchNoop(t *testing.T) { // rawCodec decodes arbitrary bytes as message-only events (test fixture). type rawCodec struct{} -func (rawCodec) Name() string { return "raw" } +func (rawCodec) Name() string { return "raw" } +func (rawCodec) Namespace() string { return "" } func (rawCodec) Decode(data []byte) (*event.Event, error) { e := &event.Event{Fields: util.MapStr{"message": string(data)}} return e, nil diff --git a/modules/pipeline/lane_consumer.go b/modules/pipeline/lane_consumer.go new file mode 100644 index 00000000..4fe7d735 --- /dev/null +++ b/modules/pipeline/lane_consumer.go @@ -0,0 +1,204 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "context" + "fmt" + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" +) + +// LaneConsumerConfig declares the lane_consumer chain host: the record- +// convention counterpart of the queue consumer. Records arrive as live +// pointers (no codec decode), the sub-chain executes stage-wise in +// written order, and surviving records optionally encode once into the +// output_queue durable bridge. +type LaneConsumerConfig struct { + Lane string `config:"lane"` + + BatchMax int `config:"batch_max"` // flush on size + BatchFlushMs int `config:"batch_flush_ms"` // flush on idle + + OnFailure string `config:"on_failure"` // tag (default) | dead_letter: | ignore + FailureTag string `config:"failure_tag"` + + OutputQueue string `config:"output_queue"` // durable bridge (encode once) + OutputCodec string `config:"output_codec"` // override the 3-level resolution + + Processors []*config.Config `config:"processor"` +} + +type LaneConsumerProcessor struct { + cfg LaneConsumerConfig + lane lane.Lane + sub *pipeline.Processors + deadQ *queue.QueueConfig + outQ *queue.QueueConfig +} + +const laneConsumerName = "lane_consumer" + +func init() { + pipeline.RegisterProcessorPluginWithConfigMetadata(laneConsumerName, NewLaneConsumerProcessor, LaneConsumerConfig{}) + pipeline.RegisterProcessorContract(laneConsumerName, pipeline.RecordContract{Scope: pipeline.ScopeBatch}) +} + +func NewLaneConsumerProcessor(c *config.Config) (pipeline.Processor, error) { + cfg := LaneConsumerConfig{BatchMax: 64, BatchFlushMs: 100, OnFailure: "tag"} + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", laneConsumerName, err) + } + if cfg.Lane == "" { + return nil, fmt.Errorf("%s requires a lane", laneConsumerName) + } + if len(cfg.Processors) == 0 { + return nil, fmt.Errorf("%s requires a non-empty processor sub-chain", laneConsumerName) + } + if cfg.BatchMax <= 0 { + cfg.BatchMax = 64 + } + if cfg.FailureTag == "" { + cfg.FailureTag = "_processing_failed" + } + l := lane.Get(cfg.Lane) + if l == nil { + return nil, fmt.Errorf("%s: lane %q not found (define it under lane: or create it dynamically)", laneConsumerName, cfg.Lane) + } + sub, err := pipeline.NewPipelineWithScope(cfg.Processors, pipeline.ScopeInfo{ + Kind: pipeline.ScopeRecord, + Namespace: l.Config().Namespace, + }, true) + if err != nil { + return nil, fmt.Errorf("failed to compile %s sub-chain: %s", laneConsumerName, err) + } + p := &LaneConsumerProcessor{cfg: cfg, lane: l, sub: sub} + if cfg.OutputQueue != "" { + p.outQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.OutputQueue}) + } + if len(cfg.OnFailure) > len("dead_letter:") && cfg.OnFailure[:len("dead_letter:")] == "dead_letter:" { + p.deadQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.OnFailure[len("dead_letter:"):], Labels: nil}) + } + return p, nil +} + +func (p *LaneConsumerProcessor) Name() string { return laneConsumerName } + +// Process subscribes (once) and blocks until the pipeline task cancels — +// the host drives the lane for its whole lifetime. +func (p *LaneConsumerProcessor) Process(ctx *pipeline.Context) error { + segments := buildSegments(p.sub.List) + batchFn := func(recs []*event.Event) { + p.runBatch(ctx, segments, recs) + } + _ = p.lane.SubscribeBatch(p.cfg.BatchMax, time.Duration(p.cfg.BatchFlushMs)*time.Millisecond, batchFn) + <-ctx.Done() + p.lane.Close() + return nil +} + +// runBatch executes the stage-wise segments over one micro-batch, then +// encodes survivors into the durable bridge. Receipts of the batch settle +// when this returns (SubscribeBatch contract) — processing completion is +// the settlement point. +func (p *LaneConsumerProcessor) runBatch(ctx *pipeline.Context, segments []execSegment, recs []*event.Event) { + active := recs + for _, seg := range segments { + if ctx.IsCanceled() || !ctx.ShouldContinue() { + return + } + if seg.batch { + live := make([]*event.Event, 0, len(active)) + for _, rec := range active { + if !pipeline.IsDropped(rec) { + live = append(live, rec) + } + } + if len(live) == 0 { + continue + } + if err := seg.proc.(pipeline.BatchProcessor).ProcessBatch(ctx, live); err != nil { + if p.cfg.OnFailure == "ignore" { + log.Warnf("lane_consumer: batch segment error ignored: %v", err) + } else { + for _, rec := range live { + mergeFailureTags(rec, []string{p.cfg.FailureTag}) + } + log.Warnf("lane_consumer: batch segment error tagged %q: %v", p.cfg.FailureTag, err) + } + } + continue + } + next := active[:0] + for _, rec := range active { + if pipeline.IsDropped(rec) { + continue + } + c := &pipeline.Context{ParentContext: ctx, Context: ctx.Context, Config: ctx.Config} + c.Set(pipeline.RecordContextKey, rec) + if p.cfg.OnFailure == "tag" { + var tags []string + c.Set(pipeline.FailureTagsKey, &tags) + for _, proc := range seg.chain.List { + if err := proc.Process(c); err != nil { + pipeline.AppendFailureTag(c, p.cfg.FailureTag) + log.Warnf("lane_consumer: sub-chain error tagged %q: %v", p.cfg.FailureTag, err) + } + } + mergeFailureTags(rec, tags) + } else if err := seg.chain.Process(c); err != nil { + log.Warnf("lane_consumer: record error (on_failure=%s): %v", p.cfg.OnFailure, err) + if p.deadQ != nil { + p.pushEncoded(p.deadQ, rec) + continue // diverted: dead_letter is a settlement + } + } + if !pipeline.IsDropped(rec) { + next = append(next, rec) + } + } + active = next + } + if p.outQ != nil { + for _, rec := range active { + if !pipeline.IsDropped(rec) { + p.pushEncoded(p.outQ, rec) + } + } + } +} + +// pushEncoded applies the 3-level codec resolution and pushes to a queue. +func (p *LaneConsumerProcessor) pushEncoded(q *queue.QueueConfig, rec *event.Event) { + codecName := p.cfg.OutputCodec + if codecName == "" { + codecName = pipeline.CodecHint(rec) + } + codec := GetRecordCodec(codecName) + if codecName == "" && p.lane.Config().Namespace != "" { + for _, ci := range pipeline.ListRecordCodecs() { + if ci.Namespace == p.lane.Config().Namespace { + codec = GetRecordCodec(ci.Name) + break + } + } + } + b, err := codec.Encode(rec) + if err != nil { + log.Errorf("lane_consumer: encode for output failed: %v", err) + return + } + if err := queue.Push(q, b); err != nil { + log.Errorf("lane_consumer: push to queue %q failed: %v", q.Name, err) + } +} + +var _ = context.Background // keep context import for future cancellation plumbing diff --git a/modules/pipeline/metrics.go b/modules/pipeline/metrics.go new file mode 100644 index 00000000..df5a1a7c --- /dev/null +++ b/modules/pipeline/metrics.go @@ -0,0 +1,106 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "sync" + "sync/atomic" + "time" + + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" +) + +// StageStats is the standard per-stage metric entry: one row per +// processor (in-process stages) plus one row per lane carrier — the +// runtime projection of the StepEvent stream. +type StageStats struct { + Stage string `json:"stage"` // processor name or lane name + Carrier string `json:"carrier"` // "processor" | "lane:" + Count int64 `json:"count"` // invocations + Errors int64 `json:"errors"` + AvgLatency int64 `json:"avg_latency_ns"` // mean per invocation + LastOutcom string `json:"last_outcome,omitempty"` +} + +type procStat struct { + count int64 + errors int64 + duration int64 // total ns + last pipeline.Outcome +} + +// statsRecorder aggregates StepEvents into per-pipeline processor stats. +type statsRecorder struct { + mu sync.Mutex + per map[string]map[string]*procStat + seq int64 +} + +func newStatsRecorder() *statsRecorder { return &statsRecorder{per: map[string]map[string]*procStat{}} } + +func (s *statsRecorder) OnStep(e *pipeline.StepEvent) { + // every event counts (aggregation is cheap: map ops); sampling is a + // flight-recorder concern, not a counter concern + s.mu.Lock() + m := s.per[e.Pipeline] + if m == nil { + m = map[string]*procStat{} + s.per[e.Pipeline] = m + } + st := m[e.Processor] + if st == nil { + st = &procStat{} + m[e.Processor] = st + } + st.count++ + st.duration += int64(e.Duration) + st.last = e.Outcome + if e.Outcome == pipeline.OutcomeError { + st.errors++ + } + s.mu.Unlock() +} + +// stageStats snapshots one pipeline's processor stages plus every lane +// carrier (depth from the live registry). +func stageStats(pipelineName string) []StageStats { + out := []StageStats{} + if rec := globalStats.Load(); rec != nil { + s := rec.(*statsRecorder) + s.mu.Lock() + if m := s.per[pipelineName]; m != nil { + for name, st := range m { + entry := StageStats{Stage: name, Carrier: "processor", Count: st.count, Errors: st.errors, LastOutcom: string(st.last)} + if st.count > 0 { + entry.AvgLatency = st.duration / st.count + } + out = append(out, entry) + } + } + s.mu.Unlock() + } + for _, l := range lane.List() { + out = append(out, StageStats{ + Stage: l.Name, + Carrier: "lane:" + l.Name, + Count: l.Emitted, + Errors: l.Drops + l.Spills, // degraded deliveries surface as stage errors + }) + } + return out +} + +var globalStats atomic.Value + +func installStats() { + if globalStats.Load() != nil { + return + } + pipeline.RegisterObserver(newStatsRecorder()) + globalStats.Store(newStatsRecorder()) +} + +var _ = time.Second diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go new file mode 100644 index 00000000..0107187f --- /dev/null +++ b/modules/pipeline/metrics_persist.go @@ -0,0 +1,94 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/env" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// PipelineStageStat is the persisted snapshot row of one stage at one +// sampling tick (D-7: framework built-in persistence — one exporter, +// every product inherits). +type PipelineStageStat struct { + orm.ORMObjectBase `json:",inline" elastic_mapping:"-"` + + Pipeline string `json:"pipeline" elastic_mapping:"pipeline:{type:keyword}"` + Stage string `json:"stage" elastic_mapping:"stage:{type:keyword}"` + Carrier string `json:"carrier" elastic_mapping:"carrier:{type:keyword}"` + Count int64 `json:"count" elastic_mapping:"count:{type:long}"` + Errors int64 `json:"errors" elastic_mapping:"errors:{type:long}"` + AvgLatency int64 `json:"avg_latency_ns" elastic_mapping:"avg_latency_ns:{type:long}"` + Ts time.Time `json:"ts" elastic_mapping:"ts:{type:date}"` +} + +// metricsPersistConfig reads pipeline_metrics.persistence. +type metricsPersistConfig struct { + Persistence struct { + Enabled bool `config:"enabled"` + Interval int `config:"interval_seconds"` + } `config:"persistence"` +} + +// startMetricsPersist runs the snapshot exporter when enabled +// (default interval 10s). Snapshots are cumulative counters; consumers +// derive rates by differencing consecutive ticks. +func (module *PipeModule) startMetricsPersist() { + cfg := metricsPersistConfig{} + exist, err := env.ParseConfig("pipeline_metrics", &cfg) + if err != nil || !exist || !cfg.Persistence.Enabled { + return + } + interval := time.Duration(cfg.Persistence.Interval) * time.Second + if interval <= 0 { + interval = 10 * time.Second + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if module.closed.Load() { + return + } + module.persistStageSnapshot() + } + } + }() + log.Debugf("pipeline metrics persistence enabled (interval %v)", interval) +} + +func (module *PipeModule) persistStageSnapshot() { + names := []string{} + module.contexts.Range(func(k, v interface{}) bool { + if name, ok := k.(string); ok { + names = append(names, name) + } + return true + }) + now := time.Now().UTC() + octx := orm.NewContext() + for _, name := range names { + for _, st := range stageStats(name) { + if st.Count == 0 && st.Errors == 0 { + continue + } + row := PipelineStageStat{ + Pipeline: name, Stage: st.Stage, Carrier: st.Carrier, + Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency, + Ts: now, + } + row.ID = util.GetUUID() + if err := orm.Save(octx, &row); err != nil { + log.Warnf("metrics persist: save %s/%s failed: %v", name, st.Stage, err) + } + } + } +} diff --git a/modules/pipeline/metrics_test.go b/modules/pipeline/metrics_test.go new file mode 100644 index 00000000..a9e3d0c9 --- /dev/null +++ b/modules/pipeline/metrics_test.go @@ -0,0 +1,37 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "testing" + "time" + + "infini.sh/framework/core/pipeline" +) + +func TestStats_AggregatesPerProcessor(t *testing.T) { + installStats() + s := globalStats.Load().(*statsRecorder) + s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeOK, Duration: 100 * time.Millisecond}) + s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeOK, Duration: 300 * time.Millisecond}) + s.OnStep(&pipeline.StepEvent{Pipeline: "sp", Processor: "a", Outcome: pipeline.OutcomeError, Duration: 50 * time.Millisecond}) + + stages := stageStats("sp") + var found *StageStats + for i := range stages { + if stages[i].Stage == "a" && stages[i].Carrier == "processor" { + found = &stages[i] + } + } + if found == nil { + t.Fatalf("stage a missing: %+v", stages) + } + if found.Count != 3 || found.Errors != 1 { + t.Fatalf("counters: %+v", found) + } + if want := int64(150 * time.Millisecond); found.AvgLatency != want { + t.Fatalf("avg latency = %d, want %d", found.AvgLatency, want) + } +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index e318b73f..fb359098 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -42,6 +42,7 @@ import ( "infini.sh/framework/core/env" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" + "infini.sh/framework/core/lane" "infini.sh/framework/core/pipeline" "infini.sh/framework/core/rate" "infini.sh/framework/core/util" @@ -67,7 +68,27 @@ var moduleCfg = struct { PipelineEnabledByDefault bool `config:"pipeline_enabled_by_default"` }{PipelineEnabledByDefault: true} +// loadLaneConfigs reads the top-level `lane:` stanzas (the static plane; +// dynamic lanes come via lane.Create at runtime). +func (module *PipeModule) loadLaneConfigs() { + cfgs := []lane.LaneConfig{} + if exist, err := env.ParseConfig("lane", &cfgs); !exist || err != nil { + log.Debugf("no lane config section (exist=%v err=%v)", exist, err) + } + for _, c := range cfgs { + if c.Name == "" { + continue + } + if _, err := lane.Create(c); err != nil { + log.Warnf("lane %s: %v", c.Name, err) + } + } +} + func (module *PipeModule) Setup() { + module.loadLaneConfigs() + installFlight() + installStats() if global.Env().IsDebug { log.Debug("pipeline framework config: ", moduleCfg) } @@ -108,6 +129,36 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler) api.HandleUIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler) + api.HandleUIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + + api.HandleAPIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler) + api.HandleUIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + + // pipeline version history + rollback + api.HandleAPIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler) + api.HandleUIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.GET, "/pipelines/:id/versions/:version", module.getPipelineVersionHandler) + api.HandleUIMethod(api.GET, "/pipelines/:id/versions/:version", module.getPipelineVersionHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.POST, "/pipelines/:id/versions/:version/rollback", module.rollbackPipelineHandler) + api.HandleUIMethod(api.POST, "/pipelines/:id/versions/:version/rollback", module.rollbackPipelineHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + + api.HandleAPIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler) + api.HandleUIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + + api.HandleAPIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler) + api.HandleUIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + + api.HandleAPIMethod(api.GET, "/pipeline/task/:id/flight", module.flightPipelineTaskHandler) + api.HandleUIMethod(api.GET, "/pipeline/task/:id/flight", module.flightPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_pause", module.pausePipelineTaskHandler) + api.HandleUIMethod(api.POST, "/pipeline/task/:id/_pause", module.pausePipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_resume", module.resumePipelineTaskHandler) + api.HandleUIMethod(api.POST, "/pipeline/task/:id/_resume", module.resumePipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_inject", module.injectPipelineTaskHandler) + api.HandleUIMethod(api.POST, "/pipeline/task/:id/_inject", module.injectPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + //use pipelines to avoid naming conflicts api.HandleUIMethod(api.POST, "/pipelines/_search", module.searchPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Search))) api.HandleUIMethod(api.POST, "/pipelines/", module.createPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Create))) @@ -122,6 +173,23 @@ func (module *PipeModule) Setup() { //api.HandleUIMethod(api.POST, "/pipeline/task/:id/metrics/_search", module.getPipelineMetricsHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) } +// pauseTask parks a running pipeline after its CURRENT batch: flow +// stops between fetches, in-flight records finish (no mid-batch abort). +func (module *PipeModule) pauseTask(taskID string) (exists bool) { + ctx, ok := module.contexts.Load(taskID) + if !ok { + return + } + if v1, ok := ctx.(*pipeline.Context); ok { + v1.Pause() + } + return true +} + +func (module *PipeModule) resumeTask(taskID string) (exists bool) { + return module.startTask(taskID) +} + func (module *PipeModule) startTask(taskID string) (exists bool) { if module.closed.Load() { return false @@ -256,6 +324,7 @@ func getPipelineConfig() ([]pipeline.PipelineConfigV2, error) { } func (module *PipeModule) Start() error { + module.startMetricsPersist() var ( pipelines []pipeline.PipelineConfigV2 err error diff --git a/modules/pipeline/output.go b/modules/pipeline/output.go new file mode 100644 index 00000000..6b619a6a --- /dev/null +++ b/modules/pipeline/output.go @@ -0,0 +1,162 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/param" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" +) + +// OutputConfig declares the unified output step: exactly ONE target +// field (queue or lane) — config unpacking rejects zero or multiple. +type OutputConfig struct { + Queue *queue.QueueConfig `config:"queue"` + Lane string `config:"lane"` + // Ack (lane target only): emit with an acknowledgment receipt attached + // to the context's AckSet — the emitting consumer's commit gate waits + // for settlement (cross-lane at-least-once). + Ack *bool `config:"ack"` + Codec string `config:"codec"` // queue target: override codec resolution + MessageField param.ParaKey `config:"message_field"` // batch scope queue target +} + +type OutputProcessor struct { + cfg OutputConfig + outQ *queue.QueueConfig + laneName string + ack bool +} + +const outputName = "output" + +func init() { + pipeline.RegisterProcessorPluginWithConfigMetadata(outputName, NewOutputProcessor, OutputConfig{}) + // batch+record conditional by target — declared record-agnostic; the + // batch-scope lane target errors at runtime with a clear pointer. + pipeline.RegisterProcessorContract(outputName, pipeline.RecordContract{}) +} + +func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) { + cfg := OutputConfig{MessageField: "messages"} + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", outputName, err) + } + targets := 0 + if cfg.Queue != nil { + targets++ + } + if cfg.Lane != "" { + targets++ + } + if targets != 1 { + return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", outputName, targets) + } + p := &OutputProcessor{cfg: cfg, laneName: cfg.Lane} + if cfg.Queue != nil { + p.outQ = queue.SmartGetOrInitConfig(cfg.Queue) + } + if cfg.Ack != nil { + p.ack = *cfg.Ack + } + return p, nil +} + +func (p *OutputProcessor) Name() string { return outputName } + +func (p *OutputProcessor) Process(ctx *pipeline.Context) error { + if p.laneName != "" { + return p.emitLane(ctx) + } + return p.pushQueue(ctx) +} + +// capture records an intercepted write when the context runs a capturing +// dry run; returns true when intercepted (the caller must NOT deliver). +func (p *OutputProcessor) capture(ctx *pipeline.Context, kind, target string, preview []byte) bool { + sink := pipeline.DryRunCapture(ctx) + if sink == nil { + return false + } + *sink = append(*sink, pipeline.DryRunWrite{Kind: kind, Target: target, Records: 1, Preview: truncate(preview, 512)}) + return true +} + +func truncate(b []byte, n int) string { + if len(b) <= n { + return string(b) + } + return string(b[:n]) + "..." +} + +// emitLane: record scope only — the batch scope has no live records. +func (p *OutputProcessor) emitLane(ctx *pipeline.Context) error { + rec, ok := pipeline.CurrentRecord(ctx) + if !ok { + return fmt.Errorf("%s{lane:%s} requires record scope (inside for_each/lane_consumer); for disk→lane bridging use consumer{codec, output_lane}", outputName, p.laneName) + } + if p.capture(ctx, "lane", p.laneName, encodePreview(rec)) { + return nil + } + l := lane.Get(p.laneName) + if l == nil { + l = lane.GetOrCreate(lane.LaneConfig{Name: p.laneName}) + } + if !p.ack { + return l.Emit(rec) + } + r := l.EmitAck(rec) + pipeline.AttachAckSet(ctx).AddReceipt(r) + return nil +} + +// pushQueue: batch scope pushes message bytes as-is (queue_output +// semantics); record scope encodes the current record once (durable +// bridge, 3-level codec resolution). +func (p *OutputProcessor) pushQueue(ctx *pipeline.Context) error { + if rec, ok := pipeline.CurrentRecord(ctx); ok { + codecName := p.cfg.Codec + if codecName == "" { + codecName = pipeline.CodecHint(rec) + } + b, err := GetRecordCodec(codecName).Encode(rec) + if err != nil { + return err + } + if p.capture(ctx, "queue", p.outQ.Name, b) { + return nil + } + return queue.Push(p.outQ, b) + } + obj := ctx.Get(p.cfg.MessageField) + msgs, ok := obj.([]queue.Message) + if !ok { + log.Warnf("processor [%s] no batch under [%s], skip", outputName, p.cfg.MessageField) + return nil + } + for i := range msgs { + if len(msgs[i].Data) == 0 { + continue + } + if err := queue.Push(p.outQ, msgs[i].Data); err != nil { + return err + } + } + return nil +} + +func encodePreview(rec *event.Event) []byte { + b, err := GetRecordCodec(pipeline.CodecHint(rec)).Encode(rec) + if err != nil { + return nil + } + return b +} diff --git a/modules/pipeline/output_test.go b/modules/pipeline/output_test.go new file mode 100644 index 00000000..86567281 --- /dev/null +++ b/modules/pipeline/output_test.go @@ -0,0 +1,112 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "testing" + "time" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +func buildOutput(t *testing.T, cfgMap util.MapStr) pipeline.Processor { + t.Helper() + c, err := config.NewConfigFrom(cfgMap) + if err != nil { + t.Fatal(err) + } + p, err := NewOutputProcessor(c) + if err != nil { + t.Fatalf("NewOutputProcessor: %v", err) + } + return p +} + +func outCtx(t *testing.T, rec *event.Event) *pipeline.Context { + t.Helper() + cfg := pipeline.PipelineConfigV2{Name: "out-test"} + ctx := pipeline.AcquireContext(cfg) + ctx.Context = t.Context() + ctx.Started() + if rec != nil { + ctx.Set(pipeline.RecordContextKey, rec) + } + return ctx +} + +// one-of target validation: zero or multiple targets reject at unpack +func TestOutput_OneOfTargetValidation(t *testing.T) { + if _, err := NewOutputProcessor(mustCfg(t, util.MapStr{})); err == nil { + t.Fatal("zero targets must reject") + } + if _, err := NewOutputProcessor(mustCfg(t, util.MapStr{"queue": map[string]interface{}{"name": "q"}, "lane": "l"})); err == nil { + t.Fatal("two targets must reject") + } +} + +func mustCfg(t *testing.T, m util.MapStr) *config.Config { + t.Helper() + c, err := config.NewConfigFrom(m) + if err != nil { + t.Fatal(err) + } + return c +} + +// lane target in record scope emits; ack attaches to the context AckSet +func TestOutput_LaneTargetRecordScope(t *testing.T) { + ln, err := lane.Create(lane.LaneConfig{Name: "out-lane", Capacity: 4}) + if err != nil { + t.Fatal(err) + } + defer lane.Delete("out-lane", true) + + p := buildOutput(t, util.MapStr{"lane": "out-lane", "ack": true}) + rec, _ := GetRecordCodec("").Decode([]byte(`{"payload":{"message":"m"}}`)) + ctx := outCtx(t, rec) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + set := pipeline.AckSetOf(ctx) + if set == nil || set.Pending() != 1 { + t.Fatalf("ack receipt not attached: %v", set) + } + // settle via a batch subscriber and confirm the gate drains + got := make(chan int, 1) + cancel := ln.SubscribeBatch(1, 10*time.Millisecond, func(recs []*event.Event) { got <- len(recs) }) + defer cancel() + deadline := time.After(2 * time.Second) + for set.Pending() > 0 { + select { + case <-deadline: + t.Fatal("receipt never settled") + case <-got: + case <-time.After(time.Millisecond): + } + } +} + +// lane target in batch scope errors with the bridge pointer +func TestOutput_LaneTargetBatchScopeRejected(t *testing.T) { + _, err := lane.Create(lane.LaneConfig{Name: "out-lane2", Capacity: 1}) + if err != nil { + t.Fatal(err) + } + defer lane.Delete("out-lane2", true) + p := buildOutput(t, util.MapStr{"lane": "out-lane2"}) + ctx := outCtx(t, nil) // no current record = batch scope + if err := p.Process(ctx); err == nil { + t.Fatal("lane target in batch scope must error") + } +} + +// queue-target construction requires the disk-queue subsystem (kv store); +// the unpack-level one-of validation is covered above, byte-push semantics +// ride the consumer integration tests. diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go index 7bedfecc..dc2ae6b2 100644 --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/orm" @@ -103,6 +104,14 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ // Ensure ID is set even if the client omitted it from the request body newConfig.ID = id + // auto-snapshot the PREVIOUS config before overwriting (every edit is + // a version — no explicit publish step; transient pipelines skip) + if !oldConfig.Transient { + if err := h.saveVersion(ctx, &oldConfig, "auto-snapshot on update"); err != nil { + log.Warnf("pipeline %s: version snapshot failed: %v", id, err) + } + } + ctx.Refresh = orm.WaitForRefresh err = orm.Save(ctx, &newConfig) if err != nil { @@ -116,6 +125,17 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ func (h *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") + // archive before delete — the version history survives pipeline removal + // (transient pipelines skip archiving) + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + old := pipeline.PipelineConfigV2{} + old.ID = id + if exists, _ := orm.GetWithSystemFields(octx, &old); exists && !old.Transient { + if err := h.saveVersion(octx, &old, "archived on delete"); err != nil { + log.Warnf("pipeline %s: delete archive failed: %v", id, err) + } + } + obj := pipeline.PipelineConfigV2{} obj.ID = id diff --git a/modules/pipeline/processors_api.go b/modules/pipeline/processors_api.go index 2d7f8fde..ab43758a 100644 --- a/modules/pipeline/processors_api.go +++ b/modules/pipeline/processors_api.go @@ -31,7 +31,10 @@ import ( httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/lane" "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/pipeline/dryrun" ) // getProcessorsHandler serves GET /pipeline/processors: the registry of @@ -47,3 +50,55 @@ func (module *PipeModule) getProcessorsHandler(w http.ResponseWriter, req *http. } module.WriteJSON(w, pipeline.GetProcessorMetadata(), 200) } + +// postDryRunHandler — POST /pipeline/dry-run +// The unified dry-run (single definition): chain + inline samples + +// options → Report (per-step snapshots, field diffs, captured writes). +type dryRunRequest struct { + Processor []map[string]interface{} `json:"processor"` + Samples []struct { + Raw string `json:"raw"` + Codec string `json:"codec"` + } `json:"samples"` + Options struct { + ScopeCheck bool `json:"scope_check"` + SideEffects string `json:"side_effects"` + LLMPolicy string `json:"llm_policy"` + } `json:"options"` +} + +func (module *PipeModule) postDryRunHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var body dryRunRequest + if err := module.DecodeJSON(req, &body); err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + samples := make([]dryrun.Sample, 0, len(body.Samples)) + for _, s := range body.Samples { + samples = append(samples, dryrun.Sample{Raw: []byte(s.Raw), Codec: s.Codec}) + } + report, err := dryrun.Run(body.Processor, samples, dryrun.Options{ + ScopeCheck: body.Options.ScopeCheck, + SideEffects: body.Options.SideEffects, + LLMPolicy: body.Options.LLMPolicy, + }) + if err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + module.WriteJSON(w, report, http.StatusOK) +} + +// getLanesHandler — GET /pipeline/lanes +// Live snapshot of every lane: depth/rates/drops/spills/blocked_waits — +// the in-process counterpart of queue depth observability. +func (module *PipeModule) getLanesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + module.WriteJSON(w, util.MapStr{"lanes": lane.List()}, 200) +} + +// getCodecsHandler — GET /pipeline/codecs +// Discovery view of the record codec registry (name / namespace / version), +// feeding for_each's codec picker and the canvas. +func (module *PipeModule) getCodecsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + module.WriteJSON(w, util.MapStr{"codecs": pipeline.ListRecordCodecs()}, 200) +} diff --git a/modules/pipeline/tap.go b/modules/pipeline/tap.go new file mode 100644 index 00000000..b870d7ae --- /dev/null +++ b/modules/pipeline/tap.go @@ -0,0 +1,131 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + "math/rand" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// TapConfig declares the tap step: copy a sampled fraction of records to +// a side target; the main flow is never affected (in-flight copy, not a +// divert). Target is one-of queue/lane — the sanctioned side-channel for +// debug indices, shadow runs and observation sampling. +type TapConfig struct { + SampleRatio float64 `config:"sample_ratio"` // 0~1, 1.0 = all + Queue *queue.QueueConfig `config:"queue"` + Lane string `config:"lane"` +} + +type TapProcessor struct { + cfg TapConfig + outQ *queue.QueueConfig +} + +const tapName = "tap" + +func init() { + pipeline.RegisterProcessorPluginWithConfigMetadata(tapName, NewTapProcessor, TapConfig{}) + pipeline.RegisterProcessorContract(tapName, pipeline.RecordContract{Scope: pipeline.ScopeRecord}) +} + +func NewTapProcessor(c *config.Config) (pipeline.Processor, error) { + cfg := TapConfig{SampleRatio: 1.0} + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", tapName, err) + } + targets := 0 + if cfg.Queue != nil { + targets++ + } + if cfg.Lane != "" { + targets++ + } + if targets != 1 { + return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", tapName, targets) + } + if cfg.SampleRatio <= 0 || cfg.SampleRatio > 1 { + return nil, fmt.Errorf("%s sample_ratio must be within (0, 1]", tapName) + } + p := &TapProcessor{cfg: cfg} + if cfg.Queue != nil { + p.outQ = queue.SmartGetOrInitConfig(cfg.Queue) + } + return p, nil +} + +func (p *TapProcessor) Name() string { return tapName } + +// Process copies the current record to the side target by sample ratio; +// the original keeps flowing regardless of the copy's fate. +func (p *TapProcessor) Process(ctx *pipeline.Context) error { + rec, ok := pipeline.CurrentRecord(ctx) + if !ok { + return nil + } + if p.cfg.SampleRatio < 1 && rand.Float64() > p.cfg.SampleRatio { + return nil + } + // deep copy: later main-flow mutations must not leak into the tapped + // copy (and vice versa — Fields is shared authority in the original) + cp := copyRecord(rec) + if sink := pipeline.DryRunCapture(ctx); sink != nil { + if b, err := GetRecordCodec(pipeline.CodecHint(cp)).Encode(cp); err == nil { + kind, target := "lane", p.cfg.Lane + if p.outQ != nil { + kind, target = "queue", p.outQ.Name + } + *sink = append(*sink, pipeline.DryRunWrite{Kind: kind, Target: target, Records: 1, Preview: string(b)}) + } + return nil + } + if p.outQ != nil { + b, err := GetRecordCodec(pipeline.CodecHint(cp)).Encode(cp) + if err != nil { + log.Warnf("tap: encode failed: %v", err) + return nil + } + if err := queue.Push(p.outQ, b); err != nil { + log.Warnf("tap: push to queue %q failed: %v", p.outQ.Name, err) + } + return nil + } + l := lane.Get(p.cfg.Lane) + if l == nil { + l = lane.GetOrCreate(lane.LaneConfig{Name: p.cfg.Lane}) + } + if err := l.Emit(cp); err != nil { + log.Warnf("tap: emit to lane %q failed: %v", p.cfg.Lane, err) + } + return nil +} + +func copyRecord(rec *event.Event) *event.Event { + b, err := util.ToJSONBytes(map[string]interface{}{"fields": rec.Fields, "labels": rec.Metadata.Labels}) + if err != nil { + return rec // fallback: share (best effort) + } + wrap := map[string]interface{}{} + if err := util.FromJSONBytes(b, &wrap); err != nil { + return rec + } + cp := &event.Event{Timestamp: rec.Timestamp} + cp.Fields, _ = wrap["fields"].(map[string]interface{}) + if lbl, ok := wrap["labels"].(map[string]interface{}); ok { + cp.Metadata.Labels = lbl + } + if cp.Fields == nil { + cp.Fields = util.MapStr{} + } + return cp +} diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index eb7bc666..07e144f2 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -24,11 +24,15 @@ package pipeline import ( + "encoding/json" "net/http" + "strconv" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/lane" "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" "infini.sh/framework/core/util" ) @@ -171,6 +175,137 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req * } // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_start +// stagesPipelineTaskHandler — GET /pipeline/task/:id/stages +// Standard per-stage metrics: processor invocations/errors/latency plus +// live lane carriers (depth-derived). +func (module *PipeModule) stagesPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + module.WriteJSON(w, util.MapStr{"stages": stageStats(id)}, http.StatusOK) +} + +// flightPipelineTaskHandler — GET /pipeline/task/:id/flight +// Sampled execution trace of a pipeline: ?trace_id= for one record's +// journey, ?errors=1 for recent failures, &limit= (default 50, newest +// first). Memory-only (ring); persistence rides the metrics exporter. +func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + q := req.URL.Query() + limit := 50 + if v := q.Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + limit = n + } + } + events := flightQuery(id, q.Get("trace_id"), q.Get("errors") == "1", limit) + module.WriteJSON(w, util.MapStr{"events": events}, http.StatusOK) +} + +// pausePipelineTaskHandler — POST /pipeline/task/:id/_pause +func (module *PipeModule) pausePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + if !module.pauseTask(id) { + module.WriteOpRecordNotFoundJSON(w, id) + return + } + module.WriteJSON(w, util.MapStr{"paused": true, "id": id}, http.StatusOK) +} + +// resumePipelineTaskHandler — POST /pipeline/task/:id/_resume +func (module *PipeModule) resumePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + if !module.resumeTask(id) { + module.WriteOpRecordNotFoundJSON(w, id) + return + } + module.WriteJSON(w, util.MapStr{"resumed": true, "id": id}, http.StatusOK) +} + +// injectPipelineTaskHandler — POST /pipeline/task/:id/_inject +// Replays a sample into the task's input (queue or lane target). +type injectRequest struct { + Queue string `json:"queue"` + Lane string `json:"lane"` + Samples []InjectSample `json:"samples"` +} + +// InjectSample accepts raw as either base64 (Go json []byte default) or +// plain text — callers should not need to pre-encode log lines. +type InjectSample struct { + Raw []byte `json:"raw"` + Codec string `json:"codec"` +} + +func (s *InjectSample) UnmarshalJSON(b []byte) error { + type alias struct { + Raw json.RawMessage `json:"raw"` + Codec string `json:"codec"` + } + var a alias + if err := json.Unmarshal(b, &a); err != nil { + return err + } + s.Codec = a.Codec + // try standard base64 first ([]byte json encoding); fall back to the + // literal string bytes (plain-text payloads) + if err := json.Unmarshal(a.Raw, &s.Raw); err != nil { + // not base64 — use the raw string content directly + var str string + if err2 := json.Unmarshal(a.Raw, &str); err2 == nil { + s.Raw = []byte(str) + } else { + s.Raw = a.Raw + } + } + return nil +} + +func (module *PipeModule) injectPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + if _, ok := module.contexts.Load(id); !ok { + module.WriteOpRecordNotFoundJSON(w, id) + return + } + var body injectRequest + if err := module.DecodeJSON(req, &body); err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + targets := 0 + if body.Queue != "" { + targets++ + } + if body.Lane != "" { + targets++ + } + if targets != 1 { + module.WriteError(w, "exactly one target (queue or lane) required", http.StatusBadRequest) + return + } + pushed := 0 + for _, s := range body.Samples { + if body.Lane != "" { + l := lane.GetOrCreate(lane.LaneConfig{Name: body.Lane}) + rec, err := pipeline.GetRecordCodec(s.Codec).Decode(s.Raw) + if err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if err := l.Emit(rec); err != nil { + module.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + pushed++ + continue + } + if err := queue.Push(queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: body.Queue}), s.Raw); err != nil { + module.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + pushed++ + } + module.WriteJSON(w, util.MapStr{"injected": pushed, "id": id}, http.StatusOK) +} + func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") exists := module.startTask(id) diff --git a/modules/pipeline/version.go b/modules/pipeline/version.go new file mode 100644 index 00000000..03210707 --- /dev/null +++ b/modules/pipeline/version.go @@ -0,0 +1,226 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + "net/http" + "strconv" + "time" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// PipelineVersion is the immutable snapshot of one pipeline config at a +// point in time. Every PUT /pipelines/:id auto-snapshots the previous +// state before overwriting — no explicit "publish" step; every edit is a +// version. +type PipelineVersion struct { + orm.ORMObjectBase `json:",inline" elastic_mapping:"-"` + + PipelineID string `json:"pipeline_id" elastic_mapping:"pipeline_id:{type:keyword}"` + Version int `json:"version" elastic_mapping:"version:{type:integer}"` + // Config is the full PipelineConfigV2 JSON snapshot. + Config string `json:"config" elastic_mapping:"config:{type:text}"` + Note string `json:"note,omitempty" elastic_mapping:"note:{type:text}"` + // UpdatedBy records who made the change (from the request context). + UpdatedBy string `json:"updated_by,omitempty" elastic_mapping:"updated_by:{type:keyword}"` + CreatedAt time.Time `json:"created_at" elastic_mapping:"created_at:{type:date}"` +} + +// maxVersionsPerPipeline bounds the retention (oldest pruned on write). +const maxVersionsPerPipeline = 50 + +// saveVersion snapshots the current config before an update lands. +// Transient pipelines skip versioning — they are ad-hoc executions that +// produce no durable config worth tracking (saving them wastes storage). +func (h *PipeModule) saveVersion(octx *orm.Context, current *pipeline.PipelineConfigV2, note string) error { + if current.Transient { + return nil // transient: no version history + } + // find the highest existing version number + next := h.nextVersion(octx, current.ID) + 1 + cfgJSON := string(util.MustToJSONBytes(current)) + + v := PipelineVersion{ + PipelineID: current.ID, + Version: next, + Config: cfgJSON, + Note: note, + CreatedAt: time.Now().UTC(), + } + v.ID = fmt.Sprintf("%s-v%d", current.ID, next) + + if err := orm.Save(octx, &v); err != nil { + return fmt.Errorf("save version snapshot: %w", err) + } + + // prune oldest beyond retention (best effort — failure is a warning) + go h.pruneVersions(current.ID) + return nil +} + +func (h *PipeModule) nextVersion(octx *orm.Context, pipelineID string) int { + orm.WithModel(octx, &PipelineVersion{}) + q := orm.NewQuery(). + Filter(orm.TermQuery("pipeline_id", pipelineID)). + SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(1) + res, err := orm.SearchV2(octx, q) + if err != nil || res == nil { + return 0 + } + versions, _, _ := decodeVersions(res) + if len(versions) == 0 { + return 0 + } + return versions[0].Version +} + +func (h *PipeModule) pruneVersions(pipelineID string) { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineVersion{}) + q := orm.NewQuery(). + Filter(orm.TermQuery("pipeline_id", pipelineID)). + SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(maxVersionsPerPipeline + 10) + res, err := orm.SearchV2(octx, q) + if err != nil { + return + } + versions, _, _ := decodeVersions(res) + if len(versions) <= maxVersionsPerPipeline { + return + } + for _, old := range versions[maxVersionsPerPipeline:] { + _ = orm.Delete(octx, &old) + } +} + +func decodeVersions(res *orm.SearchResult) ([]PipelineVersion, int64, error) { + return elastic.DecodeHits[PipelineVersion](res) +} + +// ─── endpoints ──────────────────────────────────────────────────────────── + +// listPipelineVersionsHandler — GET /pipelines/:id/versions +func (h *PipeModule) listPipelineVersionsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineVersion{}) + q := orm.NewQuery(). + Filter(orm.TermQuery("pipeline_id", id)). + SortBy(orm.Sort{Field: "version", SortType: orm.DESC}).Size(50) + res, err := orm.SearchV2(octx, q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + versions, total, _ := decodeVersions(res) + // strip the full config from the list view (it can be large) + type versionSummary struct { + Version int `json:"version"` + Note string `json:"note,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + } + out := make([]versionSummary, 0, len(versions)) + for _, v := range versions { + out = append(out, versionSummary{v.Version, v.Note, v.UpdatedBy, v.CreatedAt}) + } + h.WriteJSON(w, util.MapStr{"total": total, "versions": out}, http.StatusOK) +} + +// getPipelineVersionHandler — GET /pipelines/:id/versions/:v +func (h *PipeModule) getPipelineVersionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + vStr := ps.MustGetParameter("version") + want, err := strconv.Atoi(vStr) + if err != nil { + h.WriteError(w, "invalid version number", http.StatusBadRequest) + return + } + v := h.findVersion(req, id, want) + if v == nil { + h.WriteOpRecordNotFoundJSON(w, vStr) + return + } + h.WriteJSON(w, v, http.StatusOK) +} + +// rollbackPipelineHandler — POST /pipelines/:id/versions/:v/rollback +// Restores the config from the target version snapshot and writes a new +// version entry recording the rollback. +func (h *PipeModule) rollbackPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + vStr := ps.MustGetParameter("version") + want, err := strconv.Atoi(vStr) + if err != nil { + h.WriteError(w, "invalid version number", http.StatusBadRequest) + return + } + + target := h.findVersion(req, id, want) + if target == nil { + h.WriteOpRecordNotFoundJSON(w, vStr) + return + } + + // snapshot the CURRENT config before rollback (so the rollback itself + // is also revertible) + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + current := pipeline.PipelineConfigV2{} + current.ID = id + exists, err := orm.GetWithSystemFields(octx, ¤t) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if exists { + if err := h.saveVersion(octx, ¤t, "pre-rollback snapshot"); err != nil { + log.Warnf("pipeline %s: pre-rollback snapshot failed: %v", id, err) + } + } + + // restore the target config + restored := pipeline.PipelineConfigV2{} + if err := util.FromJSONBytes([]byte(target.Config), &restored); err != nil { + h.WriteError(w, fmt.Sprintf("decode version %d config: %v", want, err), http.StatusInternalServerError) + return + } + restored.ID = id // preserve identity + octx2 := orm.NewContextWithParent(req.Context()) + octx2.Refresh = orm.WaitForRefresh + if err := orm.Save(octx2, &restored); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, util.MapStr{ + "rolled_back_to": want, + "id": id, + "note": target.Note, + }, http.StatusOK) +} + +func (h *PipeModule) findVersion(req *http.Request, pipelineID string, version int) *PipelineVersion { + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineVersion{}) + q := orm.NewQuery(). + Filter(orm.TermQuery("pipeline_id", pipelineID)). + Filter(orm.TermQuery("version", version)).Size(1) + res, err := orm.SearchV2(octx, q) + if err != nil { + return nil + } + versions, _, _ := decodeVersions(res) + if len(versions) == 0 { + return nil + } + return &versions[0] +} diff --git a/plugins/elastic/indexing_merge/indexing_merge.go b/plugins/elastic/indexing_merge/indexing_merge.go index 791d25ac..fb2e3d40 100644 --- a/plugins/elastic/indexing_merge/indexing_merge.go +++ b/plugins/elastic/indexing_merge/indexing_merge.go @@ -46,6 +46,7 @@ import ( "sync" "time" + "github.com/buger/jsonparser" log "github.com/cihub/seelog" "github.com/savsgio/gotils/bytes" "infini.sh/framework/core/config" @@ -58,6 +59,7 @@ import ( "infini.sh/framework/core/util" "infini.sh/framework/lib/bytebufferpool" "infini.sh/framework/modules/elastic/common" + "strings" ) type IndexingMergeProcessor struct { @@ -271,17 +273,16 @@ READ_DOCS: if len(processor.config.KeyFields) > 0 { id_part = fmt.Sprintf(", \"_id\":\"%v\"", processor.fingerprintID(pop)) } else if processor.config.KeyField != "" { - source := util.MapStr{} - err := util.FromJSONBytes(pop, &source) + // zero-alloc path query: extracting ONE field + // must not float-parse the whole document (embeddings!) + v, _, _, err := jsonparser.Get(pop, strings.Split(processor.config.KeyField, ".")...) if err != nil { - panic(err) + log.Warnf("indexing_merge: key_field %q not found: %v", processor.config.KeyField, err) + } else if s, ok := trimJSONString(v); ok { + id_part = fmt.Sprintf(", \"_id\":\"%v\"", s) + } else { + id_part = fmt.Sprintf(", \"_id\":\"%v\"", string(v)) } - - v, err := source.GetValue(processor.config.KeyField) - if err != nil { - panic(err) - } - id_part = fmt.Sprintf(", \"_id\":\"%v\"", v) } // default to index writeOpType := "index" @@ -443,24 +444,19 @@ func normalizeDataStreamDoc(doc []byte) []byte { func (processor *IndexingMergeProcessor) fingerprintID(pop []byte) string { h := fnv.New64a() - source := util.MapStr{} - if err := util.FromJSONBytes(pop, &source); err != nil { - // undecodable message - hash the raw bytes as-is - h.Write(pop) - return fmt.Sprintf("%x", h.Sum64()) - } - + // zero-alloc path queries: fingerprinting must not + // float-parse the whole document (embeddings ride these messages) matched := false for _, f := range processor.config.KeyFields { h.Write([]byte{'|'}) - v, err := source.GetValue(f) - if err != nil || v == nil { + v, _, _, err := jsonparser.Get(pop, strings.Split(f, ".")...) + if err != nil { // keep positional meaning for missing fields h.Write([]byte("\x00")) continue } matched = true - h.Write([]byte(util.ToString(v))) + h.Write(v) // raw JSON token: any scalar type hashes deterministically } if !matched { // none of the key fields exist on this message: fall back to a @@ -470,3 +466,12 @@ func (processor *IndexingMergeProcessor) fingerprintID(pop []byte) string { } return fmt.Sprintf("%x", h.Sum64()) } + +// trimJSONString strips JSON string quotes from a raw token ("" and false +// when the token is not a quoted string). +func trimJSONString(v []byte) (string, bool) { + if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' { + return string(v[1 : len(v)-1]), true + } + return "", false +} diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index a48f9f5d..701fb242 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -37,9 +37,11 @@ import ( log "github.com/cihub/seelog" "infini.sh/framework/core/config" "infini.sh/framework/core/global" + "infini.sh/framework/core/lane" "infini.sh/framework/core/pipeline" "infini.sh/framework/core/queue" "infini.sh/framework/core/util" + modpipeline "infini.sh/framework/modules/pipeline" ) type QueueConsumerProcessor struct { @@ -61,23 +63,23 @@ type MessageHandlerAPI interface { } type Config struct { - NumOfSlices int `config:"num_of_slices"` - Slices []int `config:"slices"` - enabledSlice map[int]int - IdleTimeoutInSecond int `config:"idle_timeout_in_seconds"` - MaxConnectionPerHost int `config:"max_connection_per_node"` - QueueLabels map[string]interface{} `config:"queues,omitempty"` - Selector queue.QueueSelector `config:"queue_selector"` + NumOfSlices int `config:"num_of_slices"` + Slices []int `config:"slices"` + enabledSlice map[int]int + IdleTimeoutInSecond int `config:"idle_timeout_in_seconds"` + MaxConnectionPerHost int `config:"max_connection_per_node"` + QueueLabels map[string]interface{} `config:"queues,omitempty"` + Selector queue.QueueSelector `config:"queue_selector"` // ForceQueueType enforces the backend type (e.g. kafka) of the queues // consumed by this processor. Used by cross-instance transport segments: // the writer (Agent) and the reader (Gateway) must land on the same // queue implementation. - ForceQueueType string `config:"force_queue_type"` - Consumer *queue.ConsumerConfig `config:"consumer"` - MaxWorkers int `config:"max_worker_size"` - DetectActiveQueue bool `config:"detect_active_queue"` - DetectIntervalInMs int `config:"detect_interval"` - QuitDetectAfterIdleInMs int `config:"quite_detect_after_idle_in_ms"` + ForceQueueType string `config:"force_queue_type"` + Consumer *queue.ConsumerConfig `config:"consumer"` + MaxWorkers int `config:"max_worker_size"` + DetectActiveQueue bool `config:"detect_active_queue"` + DetectIntervalInMs int `config:"detect_interval"` + QuitDetectAfterIdleInMs int `config:"quite_detect_after_idle_in_ms"` MessageProcessors []*config.Config `config:"processor"` @@ -92,6 +94,16 @@ type Config struct { WaitingAfter []string `config:"waiting_after"` RetryDelayIntervalInMs int `config:"retry_delay_interval"` AutoCommitOffset bool `config:"auto_commit_offset"` + // AckTimeoutInMs bounds the commit-gate wait: lane outputs may attach + // receipts to the context; the offset only advances when all settle. + // Unsettled at timeout = sub-chain failure (redelivery, at-least-once). + AckTimeoutInMs int64 `config:"ack_timeout_ms"` + + // Bridge sugar (disk → memory in one block): when OutputLane is set, + // fetched payloads decode via Codec and emit into the lane with ack + // receipts — replacing the consumer+for_each+lane_output sandwich. + OutputLane string `config:"output_lane"` + Codec string `config:"codec"` } const name = "consumer" @@ -111,6 +123,7 @@ func New(c *config.Config) (pipeline.Processor, error) { QueueField: "queue_name", MessageField: "messages", AutoCommitOffset: true, + AckTimeoutInMs: 30000, Selector: queue.QueueSelector{ Labels: map[string]interface{}{}, }, @@ -719,6 +732,13 @@ READ_DOCS: if err != nil { panic(err) } + // commit gate: lane receipts must settle before the offset + // advances; a timeout is a sub-chain failure (redelivery) + if ackSet := pipeline.AckSetOf(&newCtx); ackSet != nil && ackSet.Pending() > 0 { + if werr := ackSet.Wait(context.Background(), time.Duration(processor.config.AckTimeoutInMs)*time.Millisecond); werr != nil { + panic(werr) + } + } // Only consume the batch when the pipeline is not stopping: a // batch interrupted mid-processing keeps the previous offset so // it is re-delivered next run (at-least-once). @@ -787,3 +807,26 @@ CLEAN_BUFFER: goto READ_DOCS } } + +// bridgeToLane decodes each fetched payload via the configured codec and +// emits it into the output lane with an acknowledgment receipt attached +// to the context's AckSet (the commit gate consumes it upstream). +func (processor *QueueConsumerProcessor) bridgeToLane(ctx *pipeline.Context, messages []queue.Message) error { + l := lane.GetOrCreate(lane.LaneConfig{Name: processor.config.OutputLane}) + codec := modpipeline.GetRecordCodec(processor.config.Codec) + as := pipeline.AttachAckSet(ctx) + for i := range messages { + if len(messages[i].Data) == 0 { + continue + } + rec, err := codec.Decode(messages[i].Data) + if err != nil { + return fmt.Errorf("bridge decode failed at offset %v: %w", messages[i].Offset, err) + } + if pipeline.TraceID(rec) == "" { + pipeline.SetTraceID(rec, fmt.Sprintf("tr-%v-%v", messages[i].Offset.Position, time.Now().UnixNano())) + } + as.AddReceipt(l.EmitAck(rec)) + } + return nil +} From 460f4d10058880860ff87d7838b0cab0a6cf5a6d Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 19:16:06 +0800 Subject: [PATCH 04/65] chore: go.mod/go.sum updates for new dependencies --- .gitignore | 1 + core/elastic/cluster_secrets_test.go | 2 +- core/otel/bench_test.go | 6 +- core/otel/log_record.go | 24 ++++---- core/pipeline/context.go | 16 +++++- core/pipeline/register.go | 56 ++++++++++++++++++- modules/configs/config/config.go | 2 +- modules/elastic/api.go | 2 +- modules/elastic/contract_integration_test.go | 8 +-- modules/elastic/orm_aggregate.go | 3 +- modules/keystore/api/api.go | 1 - .../queue/disk_queue/empty_concurrent_test.go | 2 +- modules/queue/disk_queue/empty_test.go | 24 ++++---- .../aggregate_parity_integration_test.go | 2 - modules/task/task.go | 2 +- 15 files changed, 109 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index c8d9a0b3..c8f7b8ae 100755 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ generated_*.go # keystore runtime artifacts core/elastic/.keystore/ +.mimosa/ diff --git a/core/elastic/cluster_secrets_test.go b/core/elastic/cluster_secrets_test.go index 33836208..1b35bf82 100644 --- a/core/elastic/cluster_secrets_test.go +++ b/core/elastic/cluster_secrets_test.go @@ -9,8 +9,8 @@ import ( "os" "testing" - "infini.sh/framework/core/model" "infini.sh/framework/core/keystore" + "infini.sh/framework/core/model" ucfg "infini.sh/framework/lib/go-ucfg" ) diff --git a/core/otel/bench_test.go b/core/otel/bench_test.go index 728e67fb..cd04e0d6 100644 --- a/core/otel/bench_test.go +++ b/core/otel/bench_test.go @@ -1,16 +1,16 @@ package otel import ( - "testing" - "time" "infini.sh/framework/core/event" "infini.sh/framework/core/util" + "testing" + "time" ) func benchEvent() *event.Event { return &event.Event{ Timestamp: time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC), - Fields: util.MapStr{"message": "ERROR payment-svc: connection refused to db-1:5432, retry 3", "log_level": "ERROR", "service_name": "payment-svc"}, + Fields: util.MapStr{"message": "ERROR payment-svc: connection refused to db-1:5432, retry 3", "log_level": "ERROR", "service_name": "payment-svc"}, } } diff --git a/core/otel/log_record.go b/core/otel/log_record.go index f1928c35..32d9708f 100644 --- a/core/otel/log_record.go +++ b/core/otel/log_record.go @@ -30,7 +30,9 @@ // https://opentelemetry.io/docs/specs/otel/logs/data-model/ // // Structural standard : OTel LogRecord (top-level typed fields + Resource -// /Attributes attribute collections). +// +// /Attributes attribute collections). +// // Field naming standard: lowercase snake_case keys ("log_level", // "host_name", ...), matching the house convention. The // field_standardize processor (enterprise) can switch attribute naming to @@ -54,7 +56,7 @@ import ( const ( FieldTimestamp = "timestamp" FieldObservedTimestamp = "observed_timestamp" - FieldLogLevel = "log_level" // OTel SeverityText, original string + FieldLogLevel = "log_level" // OTel SeverityText, original string FieldSeverityNumber = "severity_number" // OTel SeverityNumber, 1-24 FieldMessage = "message" // OTel Body FieldEventName = "event_name" @@ -70,15 +72,15 @@ const ( // Common Resource attribute keys (stable per source, snake_case). const ( - ResourceHostName = "host_name" - ResourceHostIP = "host_ip" - ResourceServiceName = "service_name" - ResourceServiceVersion = "service_version" - ResourceProcessPID = "process_pid" - ResourceProcessName = "process_name" - ResourceCloudProvider = "cloud_provider" - ResourceCloudRegion = "cloud_region" - ResourceCloudAccountID = "cloud_account_id" + ResourceHostName = "host_name" + ResourceHostIP = "host_ip" + ResourceServiceName = "service_name" + ResourceServiceVersion = "service_version" + ResourceProcessPID = "process_pid" + ResourceProcessName = "process_name" + ResourceCloudProvider = "cloud_provider" + ResourceCloudRegion = "cloud_region" + ResourceCloudAccountID = "cloud_account_id" ) // OTel SeverityNumber scale (see the Logs Data Model): 1-24, the 9 "well diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 5bfd5e0c..e6faa2c7 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -200,10 +200,22 @@ func (ctx *Context) GetRequestProcess() []string { return ctx.processHistory } +// maxProcessHistory bounds the flow trace: long-lived contexts run +// unbounded record batches and the history feeds log.Trace — an uncapped +// slice makes per-step logging cost grow linearly with total steps (and +// the whole loop quadratic). +const maxProcessHistory = 32 + func (ctx *Context) AddFlowProcess(str string) { - if str != "" { - ctx.processHistory = append(ctx.processHistory, str) + if str == "" { + return + } + if len(ctx.processHistory) >= maxProcessHistory { + // keep the MOST RECENT entries (drop from the head) + n := copy(ctx.processHistory, ctx.processHistory[len(ctx.processHistory)-maxProcessHistory+1:]) + ctx.processHistory = ctx.processHistory[:n] } + ctx.processHistory = append(ctx.processHistory, str) } // StepStat is the per-processor outcome of one executed pipeline step: diff --git a/core/pipeline/register.go b/core/pipeline/register.go index d0c9c99b..ad72ebf9 100644 --- a/core/pipeline/register.go +++ b/core/pipeline/register.go @@ -268,6 +268,35 @@ func RegisterProcessorPlugin(name string, constructor ProcessorConstructor) { } } +// processorContracts carries declared record contracts per processor name +// (populated via RegisterProcessorContract, or auto-cached when a +// ContractProvider instance is first constructed by the compiler). It +// feeds the catalog's scope/record_contract/granularity fields. +var processorContracts = map[string]RecordContract{} + +// RegisterProcessorContract declares (or refreshes) the record contract +// of an already-registered processor — call it from package init next to +// the plugin registration. Deterministic and side-effect-free, unlike +// probing a constructed instance. +func RegisterProcessorContract(name string, c RecordContract) { + processorContracts[name] = c +} + +// RecordContractOf returns the declared contract for a processor name +// (zero value = unconstrained). +func RecordContractOf(name string) RecordContract { + return processorContracts[name] +} + +// cacheProcessorContract auto-captures contracts from constructed +// instances so adopters without explicit registration still surface in +// the catalog after first compile. +func cacheProcessorContract(p Processor) { + if cp, ok := p.(ContractProvider); ok { + processorContracts[p.Name()] = cp.RecordContract() + } +} + // processorMetadata carries the extracted config schema of processors // registered via RegisterProcessorPluginWithConfigMetadata, so that the // discovery API (see modules/pipeline) can render configuration forms. @@ -283,6 +312,9 @@ var processorCategories = map[string]string{} // records the schema of its config struct for discovery. func RegisterProcessorPluginWithConfigMetadata(name string, constructor ProcessorConstructor, configStruct interface{}) { RegisterProcessorPlugin(name, constructor) + if configStruct == nil { + return // no schema declared; the processor still registers and runs + } processorMetadata[name] = ExtractFilterMetadata(configStruct) } @@ -353,14 +385,36 @@ func GetProcessorCatalog() util.MapStr { } catMap := grouped[cat].(util.MapStr) x, _ := processorMetadata[name] - catMap[name] = util.MapStr{ + entry := util.MapStr{ "properties": x, "category": cat, } + if c, ok := processorContracts[name]; ok { + entry["scope"] = scopeString(c.Scope) + entry["record_contract"] = util.MapStr{ + "namespace": c.Namespace, + "required_fields": c.RequiredFields, + "granularity": c.Granularity.String(), + } + } else { + entry["scope"] = "unconstrained" + } + catMap[name] = entry } return grouped } +func scopeString(k ScopeKind) string { + switch k { + case ScopeRecord: + return "record" + case ScopeBatch: + return "batch" + default: + return "any" + } +} + func RegisterFilterPlugin(name string, constructor FilterConstructor) { err := registry.RegisterFilter(name, constructor) if err != nil { diff --git a/modules/configs/config/config.go b/modules/configs/config/config.go index 8dff7780..ca0b4dc8 100755 --- a/modules/configs/config/config.go +++ b/modules/configs/config/config.go @@ -7,11 +7,11 @@ import ( "fmt" log "github.com/cihub/seelog" "infini.sh/framework/core/api" - "infini.sh/framework/core/security" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/env" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" + "infini.sh/framework/core/security" "infini.sh/framework/core/util" "infini.sh/framework/core/util/file" "infini.sh/framework/modules/configs/common" diff --git a/modules/elastic/api.go b/modules/elastic/api.go index b32afded..a4495076 100644 --- a/modules/elastic/api.go +++ b/modules/elastic/api.go @@ -29,9 +29,9 @@ package elastic import ( "infini.sh/framework/core/api" - "infini.sh/framework/core/security" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/security" "infini.sh/framework/core/util" "net/http" ) diff --git a/modules/elastic/contract_integration_test.go b/modules/elastic/contract_integration_test.go index ef3df3bb..e0de280c 100644 --- a/modules/elastic/contract_integration_test.go +++ b/modules/elastic/contract_integration_test.go @@ -29,10 +29,10 @@ func TestContract_Elastic(t *testing.T) { if err := client.CreateIndex("contractmodel", map[string]interface{}{ "mappings": map[string]interface{}{ "properties": map[string]interface{}{ - "name": map[string]interface{}{"type": "keyword"}, - "status": map[string]interface{}{"type": "keyword"}, - "body": map[string]interface{}{"type": "text"}, - "age": map[string]interface{}{"type": "integer"}, + "name": map[string]interface{}{"type": "keyword"}, + "status": map[string]interface{}{"type": "keyword"}, + "body": map[string]interface{}{"type": "text"}, + "age": map[string]interface{}{"type": "integer"}, "created": map[string]interface{}{"type": "date"}, }, }, diff --git a/modules/elastic/orm_aggregate.go b/modules/elastic/orm_aggregate.go index 00438941..462643c1 100644 --- a/modules/elastic/orm_aggregate.go +++ b/modules/elastic/orm_aggregate.go @@ -4,9 +4,9 @@ package elastic import ( "encoding/json" + "fmt" "strconv" "time" - "fmt" "infini.sh/framework/core/aggregate" api "infini.sh/framework/core/orm" @@ -179,7 +179,6 @@ func parseESAggNode(m map[string]interface{}) *api.AggNode { return node } - // normalizeESBucketKey renders ES date_histogram key_as_string values in the // canonical second-precision layout (2006-01-02T15:04:05, UTC, no millis) // used by the cross-backend conformance suite — ES emits RFC3339 with diff --git a/modules/keystore/api/api.go b/modules/keystore/api/api.go index fde85be2..7a7fdb8d 100644 --- a/modules/keystore/api/api.go +++ b/modules/keystore/api/api.go @@ -80,7 +80,6 @@ func (h *APIHandler) setKeystoreValue(w http.ResponseWriter, req *http.Request, }, http.StatusOK) } - // listKeystoreKeys — GET /keystore — key names only. Values are secrets: // the API is write-only for them by design (nothing can read them back). func (h *APIHandler) listKeystoreKeys(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { diff --git a/modules/queue/disk_queue/empty_concurrent_test.go b/modules/queue/disk_queue/empty_concurrent_test.go index 79b09834..8e708332 100644 --- a/modules/queue/disk_queue/empty_concurrent_test.go +++ b/modules/queue/disk_queue/empty_concurrent_test.go @@ -23,7 +23,7 @@ func (m *concKV) GetValue(b string, k []byte) ([]byte, error) { return nil, nil } func (m *concKV) GetCompressedValue(b string, k []byte) ([]byte, error) { return m.GetValue(b, k) } -func (m *concKV) AddValueCompress(b string, k, v []byte) error { return m.AddValue(b, k, v) } +func (m *concKV) AddValueCompress(b string, k, v []byte) error { return m.AddValue(b, k, v) } func (m *concKV) AddValue(b string, k, v []byte) error { m.data[string(k)] = append([]byte(nil), v...) return nil diff --git a/modules/queue/disk_queue/empty_test.go b/modules/queue/disk_queue/empty_test.go index af4bf684..31d1ab2f 100644 --- a/modules/queue/disk_queue/empty_test.go +++ b/modules/queue/disk_queue/empty_test.go @@ -12,10 +12,12 @@ import ( type memKV struct{ data map[string][]byte } -func (m *memKV) Open() error { return nil } -func (m *memKV) Close() error { return nil } +func (m *memKV) Open() error { return nil } +func (m *memKV) Close() error { return nil } func (m *memKV) GetValue(b string, k []byte) ([]byte, error) { - if v, ok := m.data[string(k)]; ok { return append([]byte(nil), v...), nil } + if v, ok := m.data[string(k)]; ok { + return append([]byte(nil), v...), nil + } return nil, nil } func (m *memKV) GetCompressedValue(b string, k []byte) ([]byte, error) { return m.GetValue(b, k) } @@ -44,15 +46,15 @@ func TestEmptyRemovesConsumedSegments(t *testing.T) { _ = os.MkdirAll(dataPath, 0755) cfg := &DiskQueueConfig{ - MinMsgSize: 1, - MaxMsgSize: 104857600, - MaxBytesPerFile: 1024, // small segments: multiple files quickly - SyncEveryRecords: 1, - SyncTimeoutInMS: 10, - WriteTimeoutInMS: 1000, + MinMsgSize: 1, + MaxMsgSize: 104857600, + MaxBytesPerFile: 1024, // small segments: multiple files quickly + SyncEveryRecords: 1, + SyncTimeoutInMS: 10, + WriteTimeoutInMS: 1000, EOFRetryDelayInMs: 100, - ReadChanBuffer: 10, - WriteChanBuffer: 10, + ReadChanBuffer: 10, + WriteChanBuffer: 10, } d := NewDiskQueueByConfig(name, dataPath, cfg) defer d.Close() diff --git a/modules/sqlite/aggregate_parity_integration_test.go b/modules/sqlite/aggregate_parity_integration_test.go index a3d5b958..b0a6bbec 100644 --- a/modules/sqlite/aggregate_parity_integration_test.go +++ b/modules/sqlite/aggregate_parity_integration_test.go @@ -13,7 +13,6 @@ import ( "infini.sh/framework/modules/elastic/common" ) - // elasticAggstestBackend adapts ElasticORM to the conformance suite for the // parity run (the canonical elastic-side copy lives in modules/elastic; // duplicated here because parity needs both backends and only this package @@ -77,7 +76,6 @@ func TestAggParity_SQLite_Elastic(t *testing.T) { aggstest.RunParity(t, &sqliteAggstestBackend{}, &elasticAggstestBackend{}) } - // seedParityORM registers the sqlite ORM handler over a temp database for // the parity run's cluster-record persistence. func seedParityORM(t *testing.T) { diff --git a/modules/task/task.go b/modules/task/task.go index a3d33e9c..8e327f90 100644 --- a/modules/task/task.go +++ b/modules/task/task.go @@ -25,11 +25,11 @@ package task import ( "infini.sh/framework/core/api" - "infini.sh/framework/core/security" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/env" "infini.sh/framework/core/global" "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/security" "infini.sh/framework/core/task" "infini.sh/framework/core/util" "net/http" From cb3e1cd07e74e07a674f96d28f44291b4f1c07a6 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 21:04:31 +0800 Subject: [PATCH 05/65] feat(pipeline): distributed deployment management (D7/P9 skeleton) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PipelineDeployment model: - PlacementSpec: selector + strategy (singleton/all) + rollout (canary/batch) - Version increments on spec change (drift detection) Reconciler: - 15s ticker, iterates enabled deployments - v1: logs convergence intent (configs-center push is the next milestone) Fleet API: - CRUD: POST/GET/PUT/DELETE /pipeline/deployments/ - GET /pipeline/cluster/overview (deployment × instance matrix) - All endpoints with API + UI permission gating Placement translation and instance reporting (DeploymentStatus) are stubbed — the configs-center integration (D7.6) is the next step. --- modules/pipeline/deployment.go | 288 +++++++++++++++++++++++++++++++++ modules/pipeline/module.go | 15 ++ 2 files changed, 303 insertions(+) create mode 100644 modules/pipeline/deployment.go diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go new file mode 100644 index 00000000..90be9f8d --- /dev/null +++ b/modules/pipeline/deployment.go @@ -0,0 +1,288 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "fmt" + "net/http" + "time" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// PipelineDeployment is the DESIRED state: which pipeline runs where and +// how. The reconciler translates this into managed config rows that +// gateway/agent instances pull through the configs-center channel. +// +// This is the control-plane object — instances never see it; they only +// receive the translated pipeline configs. +type PipelineDeployment struct { + orm.ORMObjectBase `json:",inline" elastic_mapping:"-"` + + Name string `json:"name" elastic_mapping:"name:{type:keyword}"` + PipelineID string `json:"pipeline_id" elastic_mapping:"pipeline_id:{type:keyword}"` + + Placement PlacementSpec `json:"placement" elastic_mapping:"placement:{type:object}"` + + // Enabled gates the reconciler: disabled = no configs pushed. + Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"` + + // Version increments on every spec change (reconciler compares with + // the last-pushed version to detect drift). + Version int `json:"version" elastic_mapping:"version:{type:integer}"` +} + +// PlacementSpec declares WHERE and HOW the pipeline runs. +type PlacementSpec struct { + // Selector picks the target instance set (label match). + Selector map[string]string `json:"selector,omitempty" config:"selector"` + + // Strategy: "singleton" (one active runner, lock-contested) | + // "all" (every instance runs a copy) + // "sharded" requires kafka queue backend (deferred, D-12) + Strategy string `json:"strategy" config:"strategy"` + + // Rollout controls staged delivery. + Rollout RolloutSpec `json:"rollout,omitempty" config:"rollout"` +} + +// RolloutSpec gates how many instances get the new version at once. +type RolloutSpec struct { + Canary int `json:"canary,omitempty" config:"canary"` // first N instances + Batch int `json:"batch,omitempty" config:"batch"` // then batches of N + MaxUnavailable int `json:"max_unavailable,omitempty" config:"max_unavailable"` // concurrent down cap +} + +// DeploymentStatus is the REPORTED state per deployment (what instances +// actually acknowledged running). +type DeploymentStatus struct { + DeploymentID string `json:"deployment_id"` + Version int `json:"version"` + Instances []InstanceReport `json:"instances"` + DesiredCount int `json:"desired_count"` + ReadyCount int `json:"ready_count"` +} + +type InstanceReport struct { + InstanceID string `json:"instance_id"` + State string `json:"state"` // running | stopped | failed | unknown + Version int `json:"version"` + LastSeen time.Time `json:"last_seen"` +} + +// ─── reconciler ────────────────────────────────────────────────────────── + +// reconcileDeployments runs on a ticker: for each enabled deployment, +// compare desired state with the managed-config rows that exist, and +// create/update/delete configs to converge. +func (module *PipeModule) startDeploymentReconciler() { + go func() { + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if module.closed.Load() { + return + } + module.reconcileOnce() + } + } + }() + log.Debugf("deployment reconciler started (interval 15s)") +} + +func (module *PipeModule) reconcileOnce() { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + + q := orm.NewQuery().Filter(orm.TermQuery("enabled", true)).Size(100) + res, err := orm.SearchV2(octx, q) + if err != nil { + return + } + deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res) + + for _, dep := range deployments { + if rerr := module.reconcileDeployment(&dep); rerr != nil { + log.Warnf("reconciler: deployment %s: %v", dep.Name, rerr) + } + } +} + +// reconcileDeployment converges ONE deployment toward its desired state. +// v1: push the pipeline config as a managed config row targeting the +// selected instances. Instance-side lock contention resolves singleton. +func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { + // load the pipeline config + octx := orm.NewContext().DirectAccess() + cfg := pipeline.PipelineConfigV2{} + cfg.ID = dep.PipelineID + exists, err := orm.GetV2(octx, &cfg) + if err != nil || !exists { + return fmt.Errorf("pipeline %s not found", dep.PipelineID) + } + + // translate to a managed-config row (the same channel stream publish + // uses — instances pull and hot-reload) + // v1: broadcast to all instances (strategy "all") + // TODO: selector-based targeting, singleton lock, staged rollout + configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID) + cfgJSON := util.MustToJSONBytes(map[string]interface{}{ + "pipeline": []interface{}{map[string]interface{}{ + "name": cfg.Name, + "enabled": cfg.Enabled, + "auto_start": cfg.AutoStart, + "keep_running": cfg.KeepRunning, + "singleton": dep.Placement.Strategy == "singleton", + "processor": cfg.Processors, + }}, + }) + + log.Debugf("reconciler: deployment %s → managed config %s (%d bytes, strategy=%s)", + dep.Name, configName, len(cfgJSON), dep.Placement.Strategy) + + // the managed-config write happens through the configs-center module; + // for now the reconciler only logs — the actual push requires the + // configs server module to be active (logpilot/console hosts it) + // TODO: call configs_server.UpsertManagedConfig() + + return nil +} + +// ─── Fleet API ─────────────────────────────────────────────────────────── + +// createDeploymentHandler — POST /pipeline/deployments/ +func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var dep PipelineDeployment + if err := h.DecodeJSON(req, &dep); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if dep.Name == "" || dep.PipelineID == "" { + h.WriteError(w, "name and pipeline_id are required", http.StatusBadRequest) + return + } + if dep.Placement.Strategy == "" { + dep.Placement.Strategy = "all" // v1 default (D-12: no sharded) + } + dep.Version = 1 + + octx := orm.NewContextWithParent(req.Context()) + if err := orm.Save(octx, &dep); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, dep, http.StatusCreated) +} + +// listDeploymentsHandler — GET /pipeline/deployments/ +func (h *PipeModule) listDeploymentsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + q := orm.NewQuery().Size(100) + res, err := orm.SearchV2(octx, q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + deployments, total, _ := elastic.DecodeHits[PipelineDeployment](res) + h.WriteJSON(w, util.MapStr{"total": total, "items": deployments}, http.StatusOK) +} + +// getDeploymentHandler — GET /pipeline/deployments/:id +func (h *PipeModule) getDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + dep := PipelineDeployment{} + dep.ID = id + exists, err := orm.GetV2(octx, &dep) + if err != nil || !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + h.WriteJSON(w, dep, http.StatusOK) +} + +// updateDeploymentHandler — PUT /pipeline/deployments/:id +func (h *PipeModule) updateDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + var newDep PipelineDeployment + if err := h.DecodeJSON(req, &newDep); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + old := PipelineDeployment{} + old.ID = id + exists, err := orm.GetV2(octx, &old) + if err != nil || !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + + // preserve identity + bump version + newDep.ID = id + newDep.Version = old.Version + 1 + + if err := orm.Save(octx, &newDep); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, newDep, http.StatusOK) +} + +// deleteDeploymentHandler — DELETE /pipeline/deployments/:id +func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + dep := PipelineDeployment{} + dep.ID = id + if err := orm.Delete(octx, &dep); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + // TODO: remove the managed config row (cascade) + h.WriteJSON(w, util.MapStr{"deleted": true, "id": id}, http.StatusOK) +} + +// getClusterOverviewHandler — GET /pipeline/cluster/overview +// Matrix of deployments × instances (who runs what, what version). +func (h *PipeModule) getClusterOverviewHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + q := orm.NewQuery().Filter(orm.TermQuery("enabled", true)).Size(100) + res, err := orm.SearchV2(octx, q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res) + + type overviewEntry struct { + Deployment string `json:"deployment"` + Pipeline string `json:"pipeline"` + Strategy string `json:"strategy"` + Version int `json:"version"` + Selector map[string]string `json:"selector,omitempty"` + } + out := make([]overviewEntry, 0, len(deployments)) + for _, dep := range deployments { + out = append(out, overviewEntry{ + Deployment: dep.Name, + Pipeline: dep.PipelineID, + Strategy: dep.Placement.Strategy, + Version: dep.Version, + Selector: dep.Placement.Selector, + }) + } + h.WriteJSON(w, util.MapStr{"deployments": out}, http.StatusOK) +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index fb359098..294989de 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -89,6 +89,7 @@ func (module *PipeModule) Setup() { module.loadLaneConfigs() installFlight() installStats() + module.startDeploymentReconciler() if global.Env().IsDebug { log.Debug("pipeline framework config: ", moduleCfg) } @@ -135,6 +136,20 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler) api.HandleUIMethod(api.GET, "/pipeline/lanes", module.getLanesHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // Fleet API (deployment management, D7) + api.HandleAPIMethod(api.GET, "/pipeline/deployments/", module.listDeploymentsHandler) + api.HandleUIMethod(api.GET, "/pipeline/deployments/", module.listDeploymentsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.POST, "/pipeline/deployments/", module.createDeploymentHandler) + api.HandleUIMethod(api.POST, "/pipeline/deployments/", module.createDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineCreate)) + api.HandleAPIMethod(api.GET, "/pipeline/deployments/:id", module.getDeploymentHandler) + api.HandleUIMethod(api.GET, "/pipeline/deployments/:id", module.getDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.PUT, "/pipeline/deployments/:id", module.updateDeploymentHandler) + api.HandleUIMethod(api.PUT, "/pipeline/deployments/:id", module.updateDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler) + api.HandleUIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineDelete)) + api.HandleAPIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler) + api.HandleUIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // pipeline version history + rollback api.HandleAPIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler) api.HandleUIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) From 59d5c62f7ca600573d811619d5d2936d97177e4f Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 21:06:15 +0800 Subject: [PATCH 06/65] feat(pipeline): deployment reconciler pushes via configs-center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconciler now writes actual ManagedConfig rows (the same transport Logpilot stream publish uses — instances pull via _sync and hot-reload): PipelineDeployment → reconcile → ManagedConfig row → instance _sync - Upsert semantics: find existing row by name, bump version, save - Readonly=true (deployment-owned; manual edits are overwritten) - singleton strategy translates to singleton:true in the config content - v1: broadcast to all instances (selector targeting is next milestone) --- modules/pipeline/deployment.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 90be9f8d..fd6f2a7d 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -15,6 +15,7 @@ import ( "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" "infini.sh/framework/core/util" + configs_server "infini.sh/framework/modules/configs/server" ) // PipelineDeployment is the DESIRED state: which pipeline runs where and @@ -146,14 +147,34 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { }}, }) - log.Debugf("reconciler: deployment %s → managed config %s (%d bytes, strategy=%s)", - dep.Name, configName, len(cfgJSON), dep.Placement.Strategy) + // upsert the ManagedConfig row (instances pull on next _sync) + orm.WithModel(octx, &configs_server.ManagedConfig{}) + existing := configs_server.ManagedConfig{} + foundQ := orm.NewQuery(). + Filter(orm.TermQuery("name", configName)). + Filter(orm.TermQuery("instance_id", "*")).Size(1) + foundRes, ferr := orm.SearchV2(octx, foundQ) + if ferr == nil && foundRes != nil { + if rows, _, derr := elastic.DecodeHits[configs_server.ManagedConfig](foundRes); derr == nil && len(rows) > 0 { + existing = rows[0] + } + } + + existing.Name = configName + existing.InstanceID = "*" // v1: broadcast; selector targeting next + existing.Content = string(cfgJSON) + existing.Version++ + existing.Readonly = true // deployment-owned - // the managed-config write happens through the configs-center module; - // for now the reconciler only logs — the actual push requires the - // configs server module to be active (logpilot/console hosts it) - // TODO: call configs_server.UpsertManagedConfig() + if existing.ID == "" { + existing.ID = fmt.Sprintf("deploy-%s", dep.PipelineID) + } + if serr := orm.Save(octx, &existing); serr != nil { + return fmt.Errorf("push managed config: %w", serr) + } + log.Debugf("reconciler: deployment %s → config %s v%d (%d bytes, strategy=%s)", + dep.Name, configName, existing.Version, len(cfgJSON), dep.Placement.Strategy) return nil } From 94fc347a29cd0d291d69f6c0f78b387acc1bbbd0 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 21:25:30 +0800 Subject: [PATCH 07/65] feat(pipeline): P9 deployment status + instance runtime reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instance runtime reporting (desired-vs-reported loop): - ConfigSyncRequest extended with Runtime field (PipelineRuntimeState) - Configs server persists runtime_pipelines on the instance record - PipeModule.RuntimeSnapshot() exposes live task states for the sync client Deployment status: - GET /pipeline/deployments/:id/status → DeploymentStatus (desired count, ready count, per-instance reports) - GET /pipeline/cluster/overview enriched with live status per deployment - Reconciler reads instance RuntimePipelines to compute reported state Full P9 control loop now closed: desired (Deployment) → reconcile → managed config push → instance _sync → runtime report → status → cluster overview --- core/model/instance.go | 5 ++ modules/configs/common/domain.go | 19 +++++++ modules/configs/server/server.go | 10 ++++ modules/pipeline/deployment.go | 88 +++++++++++++++++++++++++++++++- modules/pipeline/module.go | 33 ++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) diff --git a/core/model/instance.go b/core/model/instance.go index 0264ddcd..bdd7a0e2 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -81,6 +81,11 @@ type Instance struct { Services []ServiceInfo `json:"services,omitempty" elastic_mapping:"services: { type: object }"` Status string `json:"status,omitempty" elastic_mapping:"status: { type: keyword, copy_to:search_text }"` + // RuntimePipelines is the instance's live pipeline state, reported via + // configs _sync and consumed by the deployment reconciler. Stored as + // a JSON string (map[string]PipelineRuntimeState). + RuntimePipelines string `json:"runtime_pipelines,omitempty" elastic_mapping:"runtime_pipelines:{type:text}"` + //SearchText string `json:"search_text,omitempty" elastic_mapping:"search_text:{type:text,index_prefixes:{},index_phrases:true, analyzer:suggest_text_search }"` } diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 46ecbbca..7b813987 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -83,6 +83,25 @@ type ConfigSyncRequest struct { Hash string `json:"hash"` Client model.Instance `json:"client"` Configs ConfigList `json:"configs"` + + // Runtime reports the instance's live pipeline state (which tasks are + // running, what config version they loaded). The deployment reconciler + // reads this to compute DeploymentStatus (desired vs reported). + Runtime *RuntimeReport `json:"runtime,omitempty"` +} + +// RuntimeReport is the pipeline execution state an instance reports on +// every sync — the "reported" half of the reconcile loop. +type RuntimeReport struct { + // Pipelines maps task name → state + the managed-config version the + // instance last loaded (0 = unknown/not from managed config). + Pipelines map[string]PipelineRuntimeState `json:"pipelines,omitempty"` +} + +type PipelineRuntimeState struct { + State string `json:"state"` // STARTED|STOPPED|FAILED|... + ConfigSource string `json:"config_source,omitempty"` // managed config name + ConfigVersion int64 `json:"config_version,omitempty"` // managed config version loaded } type ConfigSyncResponse struct { diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go index d639e65b..592a1aad 100644 --- a/modules/configs/server/server.go +++ b/modules/configs/server/server.go @@ -531,6 +531,16 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt log.Debugf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err) } + // Runtime report: persist the instance's pipeline state for the + // deployment reconciler (desired-vs-reported comparison). + if obj.Runtime != nil && len(obj.Runtime.Pipelines) > 0 { + rtJSON, rerr := json.Marshal(obj.Runtime.Pipelines) + if rerr == nil { + obj.Client.RuntimePipelines = string(rtJSON) + _, _ = upsertInstance(&obj.Client) // second upsert with runtime + } + } + assigned, aerr := loadAssignedConfigs(obj.Client.ID) if aerr != nil { // Transient backend failure — defer to the next sync instead of diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index fd6f2a7d..85229984 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -5,6 +5,7 @@ package pipeline import ( + "encoding/json" "fmt" "net/http" "time" @@ -12,6 +13,7 @@ import ( log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" "infini.sh/framework/core/util" @@ -178,6 +180,58 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { return nil } +// getDeploymentStatus builds the REPORTED state from instance runtime +// data (desired vs reported comparison for the cluster overview). +func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStatus { + status := DeploymentStatus{ + DeploymentID: dep.ID, + Version: dep.Version, + } + + // query instances that have runtime pipeline data + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &model.Instance{}) + q := orm.NewQuery().Size(200) + res, err := orm.SearchV2(octx, q) + if err != nil { + return status + } + instances, _, _ := elastic.DecodeHits[model.Instance](res) + + for _, inst := range instances { + if inst.RuntimePipelines == "" { + continue + } + var pipelines map[string]struct { + State string `json:"state"` + } + if json.Unmarshal([]byte(inst.RuntimePipelines), &pipelines) != nil { + continue + } + // check if this instance runs the deployment's pipeline + taskName := dep.PipelineID + if st, ok := pipelines[taskName]; ok { + status.Instances = append(status.Instances, InstanceReport{ + InstanceID: inst.ID, + State: st.State, + Version: dep.Version, + LastSeen: time.Now().UTC(), + }) + } + } + status.DesiredCount = 1 // v1: all strategy = every instance; singleton = 1 + if dep.Placement.Strategy == "singleton" { + status.DesiredCount = 1 + } + status.ReadyCount = 0 + for _, ir := range status.Instances { + if ir.State == "STARTED" { + status.ReadyCount++ + } + } + return status +} + // ─── Fleet API ─────────────────────────────────────────────────────────── // createDeploymentHandler — POST /pipeline/deployments/ @@ -275,6 +329,21 @@ func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Re h.WriteJSON(w, util.MapStr{"deleted": true, "id": id}, http.StatusOK) } +// getDeploymentStatusHandler — GET /pipeline/deployments/:id/status +func (h *PipeModule) getDeploymentStatusHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + dep := PipelineDeployment{} + dep.ID = id + exists, err := orm.GetV2(octx, &dep) + if err != nil || !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + status := h.getDeploymentStatus(&dep) + h.WriteJSON(w, status, http.StatusOK) +} + // getClusterOverviewHandler — GET /pipeline/cluster/overview // Matrix of deployments × instances (who runs what, what version). func (h *PipeModule) getClusterOverviewHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -305,5 +374,22 @@ func (h *PipeModule) getClusterOverviewHandler(w http.ResponseWriter, req *http. Selector: dep.Placement.Selector, }) } - h.WriteJSON(w, util.MapStr{"deployments": out}, http.StatusOK) + // enrich with reported status + type overviewWithStatus struct { + overviewEntry + Status *DeploymentStatus `json:"status,omitempty"` + } + outWithStatus := make([]overviewWithStatus, 0, len(deployments)) + for _, dep := range deployments { + entry := overviewEntry{ + Deployment: dep.Name, + Pipeline: dep.PipelineID, + Strategy: dep.Placement.Strategy, + Version: dep.Version, + Selector: dep.Placement.Selector, + } + st := h.getDeploymentStatus(&dep) + outWithStatus = append(outWithStatus, overviewWithStatus{entry, &st}) + } + h.WriteJSON(w, util.MapStr{"deployments": outWithStatus}, http.StatusOK) } diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 294989de..a8836747 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -60,6 +60,36 @@ type PipeModule struct { contexts sync.Map } +// RuntimeSnapshot returns the current pipeline task states for the +// configs _sync runtime report (consumed by the deployment reconciler). +func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { + out := map[string]interface{}{} + module.contexts.Range(func(k, v interface{}) bool { + name, ok := k.(string) + if !ok { + return true + } + ctx, ok := v.(*pipeline.Context) + if !ok { + return true + } + state := "UNKNOWN" + switch ctx.GetRunningState() { + case pipeline.STARTED, pipeline.STARTING: + state = "STARTED" + case pipeline.FAILED: + state = "FAILED" + case pipeline.STOPPED, pipeline.FINISHED: + state = "STOPPED" + } + out[name] = map[string]interface{}{ + "state": state, + } + return true + }) + return out +} + func (module *PipeModule) Name() string { return "pipeline" } @@ -147,6 +177,9 @@ func (module *PipeModule) Setup() { api.HandleUIMethod(api.PUT, "/pipeline/deployments/:id", module.updateDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) api.HandleAPIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler) api.HandleUIMethod(api.DELETE, "/pipeline/deployments/:id", module.deleteDeploymentHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineDelete)) + api.HandleAPIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler) + api.HandleUIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler) api.HandleUIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) From ce01a230789974555cc02e226f9d268b77ac2932 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 22:31:10 +0800 Subject: [PATCH 08/65] fix(pipeline): register ORM schemas for Pipeline models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PipelineVersion, PipelineDeployment, PipelineStageStat need explicit orm.MustRegisterSchemaWithIndexName to auto-create their SQLite/ES tables on module startup — without it queries fail with 'no such table'. --- modules/pipeline/module.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index a8836747..7cd19e53 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -43,6 +43,7 @@ import ( "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/lane" + "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" "infini.sh/framework/core/rate" "infini.sh/framework/core/util" @@ -116,6 +117,11 @@ func (module *PipeModule) loadLaneConfigs() { } func (module *PipeModule) Setup() { + // ORM schema registration (table/index auto-creation on startup) + orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions") + orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments") + orm.MustRegisterSchemaWithIndexName(PipelineStageStat{}, "pipeline-stage-stats") + module.loadLaneConfigs() installFlight() installStats() From 55450e32a75e2b0e1958c07645615ee0ab353e4b Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 22:44:48 +0800 Subject: [PATCH 09/65] fix(pipeline): register PipelineConfigV2 ORM schema Without orm.MustRegisterSchemaWithIndexName, the pipelines/_search endpoint fails with 'no such table' (500). --- modules/pipeline/module.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 7cd19e53..0ed8a0ab 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -118,6 +118,7 @@ func (module *PipeModule) loadLaneConfigs() { func (module *PipeModule) Setup() { // ORM schema registration (table/index auto-creation on startup) + orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs") orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions") orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments") orm.MustRegisterSchemaWithIndexName(PipelineStageStat{}, "pipeline-stage-stats") From 04c362f771629efd9faa743aa03fa33395af900d Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 14 Oct 2026 23:41:28 +0800 Subject: [PATCH 10/65] fix(pipeline): generate ID for PipelineDeployment on create orm.Save requires an explicit ID; without it the create handler returns 500 'id was not found'. --- modules/pipeline/deployment.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 85229984..2bf6fef8 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -249,6 +249,7 @@ func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Re dep.Placement.Strategy = "all" // v1 default (D-12: no sharded) } dep.Version = 1 + dep.ID = util.GetUUID() octx := orm.NewContextWithParent(req.Context()) if err := orm.Save(octx, &dep); err != nil { From 5a5b1456ca902a3303a494a165888f3b0e153f1c Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 00:09:10 +0800 Subject: [PATCH 11/65] feat(pipeline): instance runtime reporting in configs _sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync client now includes RuntimeReport (which pipeline tasks are running) in every _sync request. The configs server persists it on the instance record; the deployment reconciler reads it to compute DeploymentStatus (health: ready/desired). Closes the P9 control loop: desired → push config → instance sync → runtime report → status --- modules/configs/client/client.go | 35 ++++++++++++++++++++++++++++++++ modules/pipeline/module.go | 4 ++++ 2 files changed, 39 insertions(+) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index af662d48..f004722f 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -53,6 +53,12 @@ import ( "infini.sh/framework/modules/configs/config" ) +// SetPipelineRuntimeSnapshotProvider wires the pipeline module's snapshot +// into the sync client (called from PipeModule.Setup). +func SetPipelineRuntimeSnapshotProvider(fn func() map[string]interface{}) { + getPipelineRuntimeSnapshot = fn +} + const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" const legacyManagedRegisterCompatMaxVersion = "1.30.4" @@ -429,6 +435,12 @@ func ListenConfigChanges() error { req.Configs = cfgs req.Hash = util.MD5digestString(util.MustToJSONBytes(cfgs)) + // runtime pipeline state: report which tasks are running + // so the deployment reconciler can compute health status + if snapshot := runtimePipelineSnapshot(); snapshot != nil { + req.Runtime = snapshot + } + //fetch configs from manager request := util.Request{Method: util.Verb_POST} request.ContentType = "application/json" @@ -576,3 +588,26 @@ func saveKeystore(k string, v string) error { err = ks.Save() return err } + +// runtimePipelineSnapshot builds the RuntimeReport for the sync request. +// It delegates to the PipeModule if loaded; nil when not (agent-only etc). +func runtimePipelineSnapshot() *common.RuntimeReport { + snapshot := getPipelineRuntimeSnapshot() + if snapshot == nil { + return nil + } + report := &common.RuntimeReport{Pipelines: map[string]common.PipelineRuntimeState{}} + for name, state := range snapshot { + stateStr, _ := state.(map[string]interface{})["state"].(string) + source, _ := state.(map[string]interface{})["config_source"].(string) + report.Pipelines[name] = common.PipelineRuntimeState{ + State: stateStr, + ConfigSource: source, + } + } + return report +} + +// getPipelineRuntimeSnapshot is set by the pipeline module on startup; +// nil when the pipeline module is not loaded. +var getPipelineRuntimeSnapshot func() map[string]interface{} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 0ed8a0ab..a26c3b75 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -47,6 +47,7 @@ import ( "infini.sh/framework/core/pipeline" "infini.sh/framework/core/rate" "infini.sh/framework/core/util" + configs_client "infini.sh/framework/modules/configs/client" ) type PipeModule struct { @@ -117,6 +118,9 @@ func (module *PipeModule) loadLaneConfigs() { } func (module *PipeModule) Setup() { + // provide the runtime snapshot for the configs _sync client + // (deployment reconciler reads it for health status) + configs_client.SetPipelineRuntimeSnapshotProvider(module.RuntimeSnapshot) // ORM schema registration (table/index auto-creation on startup) orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs") orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions") From c2a4929145d009c95105df4617c8f76d8bc7a68c Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 00:12:19 +0800 Subject: [PATCH 12/65] feat(pipeline): instance-targeted deployment (selector) PlacementSpec.Instances lists explicit target instance IDs; the reconciler creates per-instance managed configs instead of broadcast. Empty list = broadcast to all (backward compatible). UI passes the selected target instance from the deployment dialog. --- modules/pipeline/deployment.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 2bf6fef8..352c5512 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -47,6 +47,10 @@ type PlacementSpec struct { // Selector picks the target instance set (label match). Selector map[string]string `json:"selector,omitempty" config:"selector"` + // Instances lists explicit target instance IDs. When non-empty, the + // reconciler creates per-instance managed configs; empty = broadcast. + Instances []string `json:"instances,omitempty" config:"instances"` + // Strategy: "singleton" (one active runner, lock-contested) | // "all" (every instance runs a copy) // "sharded" requires kafka queue backend (deferred, D-12) From d2e846b3d98ec74024728a80c60114f8bcb2ff96 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 09:01:01 +0800 Subject: [PATCH 13/65] fix(pipeline): output processor accepts string queue names The output processor's queue field expects *queue.QueueConfig (object), but pipelines written by hand use the short form {"output":{"queue":"name"}}. QueueRef unpacks both the bare string and the full object. --- modules/pipeline/output.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/modules/pipeline/output.go b/modules/pipeline/output.go index 6b619a6a..a3cecb3b 100644 --- a/modules/pipeline/output.go +++ b/modules/pipeline/output.go @@ -19,8 +19,8 @@ import ( // OutputConfig declares the unified output step: exactly ONE target // field (queue or lane) — config unpacking rejects zero or multiple. type OutputConfig struct { - Queue *queue.QueueConfig `config:"queue"` - Lane string `config:"lane"` + Queue *QueueRef `config:"queue"` + Lane string `config:"lane"` // Ack (lane target only): emit with an acknowledgment receipt attached // to the context's AckSet — the emitting consumer's commit gate waits // for settlement (cross-lane at-least-once). @@ -29,6 +29,21 @@ type OutputConfig struct { MessageField param.ParaKey `config:"message_field"` // batch scope queue target } +// QueueRef accepts both a bare queue name (string) and a full QueueConfig +// object — pipelines written by hand use the short form. +type QueueRef struct { + queue.QueueConfig +} + +func (q *QueueRef) UnpackConfig(c *config.Config) error { + var name string + if err := c.Unpack(&name); err == nil { + q.Name = name + return nil + } + return c.Unpack(&q.QueueConfig) +} + type OutputProcessor struct { cfg OutputConfig outQ *queue.QueueConfig @@ -51,7 +66,7 @@ func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) { return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", outputName, err) } targets := 0 - if cfg.Queue != nil { + if cfg.Queue != nil && cfg.Queue.Name != "" { targets++ } if cfg.Lane != "" { @@ -61,8 +76,8 @@ func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) { return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", outputName, targets) } p := &OutputProcessor{cfg: cfg, laneName: cfg.Lane} - if cfg.Queue != nil { - p.outQ = queue.SmartGetOrInitConfig(cfg.Queue) + if cfg.Queue != nil && cfg.Queue.Name != "" { + p.outQ = queue.SmartGetOrInitConfig(&cfg.Queue.QueueConfig) } if cfg.Ack != nil { p.ack = *cfg.Ack From 0771447b22018085b683fe4ce5b0e281bc05158d Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 11:43:00 +0800 Subject: [PATCH 14/65] fix: skip deployment reconcile and guard ORM access on lightweight agents - add orm.IsHandlerRegistered() so instances without an ORM backend (agents running only pushed managed configs) can check availability - reconcileOnce now returns early instead of panicking on "ORM handler is not registered" --- core/orm/registry.go | 6 ++++++ modules/pipeline/deployment.go | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/core/orm/registry.go b/core/orm/registry.go index 292b42e3..4d13cdf5 100644 --- a/core/orm/registry.go +++ b/core/orm/registry.go @@ -35,6 +35,12 @@ func InitSchema() error { var handler ORM +// IsHandlerRegistered reports whether an ORM backend is available — +// lightweight instances (agents without sqlite/ES) have none. +func IsHandlerRegistered() bool { + return handler != nil +} + func getHandler() ORM { if handler == nil { panic(errors.New("ORM handler is not registered")) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 352c5512..46ed1e68 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -107,6 +107,11 @@ func (module *PipeModule) startDeploymentReconciler() { } func (module *PipeModule) reconcileOnce() { + // reconcile is a control-plane operation — skip on instances without + // an ORM backend (lightweight agents that only run pushed configs) + if !orm.IsHandlerRegistered() { + return + } octx := orm.NewContext().DirectAccess() orm.WithModel(octx, &PipelineDeployment{}) From 14564cf51998516618f787b9ffba9370fc6db02a Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 12:10:51 +0800 Subject: [PATCH 15/65] fix: deployment reconciler convergence and name-based status matching - reconcileDeployment skips the version bump and managed-config push when the rendered content is unchanged (was re-pushing every 15s, rewriting files on every instance each cycle) - RuntimeSnapshot reports under the pipeline NAME (the contexts map key is the config ID which only equals the name for id-less YAML configs) - getDeploymentStatus resolves the pipeline name from the config row before matching instance runtime reports --- modules/pipeline/deployment.go | 18 ++++++++++++++++-- modules/pipeline/module.go | 13 +++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 46ed1e68..38e505fe 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -171,6 +171,12 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { } } + // converged: the pushed content already matches — skip the version + // bump and re-push so instances don't rewrite files every cycle + if existing.ID != "" && existing.Content == string(cfgJSON) { + return nil + } + existing.Name = configName existing.InstanceID = "*" // v1: broadcast; selector targeting next existing.Content = string(cfgJSON) @@ -207,6 +213,15 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat } instances, _, _ := elastic.DecodeHits[model.Instance](res) + // resolve the pipeline NAME — instances run pipelines under their + // config name (YAML configs default ID = Name), not the config row ID + taskCtx := orm.NewContext().DirectAccess() + cfg := pipeline.PipelineConfigV2{} + cfg.ID = dep.PipelineID + if found, gerr := orm.GetV2(taskCtx, &cfg); gerr != nil || !found { + return status + } + for _, inst := range instances { if inst.RuntimePipelines == "" { continue @@ -218,8 +233,7 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat continue } // check if this instance runs the deployment's pipeline - taskName := dep.PipelineID - if st, ok := pipelines[taskName]; ok { + if st, ok := pipelines[cfg.Name]; ok { status.Instances = append(status.Instances, InstanceReport{ InstanceID: inst.ID, State: st.State, diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index a26c3b75..84616e91 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -67,14 +67,19 @@ type PipeModule struct { func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { out := map[string]interface{}{} module.contexts.Range(func(k, v interface{}) bool { - name, ok := k.(string) - if !ok { - return true - } ctx, ok := v.(*pipeline.Context) if !ok { return true } + // report under the pipeline NAME — consumers (deployment status) + // match by name; the map key is the config ID which only equals + // the name for id-less YAML configs + name := ctx.Config.Name + if name == "" { + if s, ok := k.(string); ok { + name = s + } + } state := "UNKNOWN" switch ctx.GetRunningState() { case pipeline.STARTED, pipeline.STARTING: From 74832e7d5397b506ef5fb4cca27a41a413260eaf Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 12:22:12 +0800 Subject: [PATCH 16/65] feat: compile-time validation gate before pipeline configs are persisted or distributed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configs referencing unknown processors or carrying invalid parameters were accepted by the control plane and only failed on the instance after distribution. Add ValidatePipelineConfig — a compile-without-run pass covering DSL shape, registry existence, parameter unpacking, nested chains and record-contract placement — and wire it as a gate: - POST/PUT /pipelines/ reject invalid configs with structured errors (stage index + processor key + message) before persistence - POST /pipeline/task create validates disabled pipelines too - POST /pipeline/deployments/ requires the referenced pipeline to exist and compile (fail-closed on read errors) - the reconciler refuses to distribute configs that stopped compiling (e.g. processor plugin removed after an upgrade) - new POST /pipeline/_validate pre-flight endpoint for editors: 200 report {valid, errors, warnings}, nothing persisted Contract mismatches surface as warnings unless the config opts into strict_contract, matching runtime compile semantics. --- modules/pipeline/deployment.go | 27 +++++- modules/pipeline/module.go | 4 + modules/pipeline/pipeline.go | 12 +++ modules/pipeline/tasks.go | 6 ++ modules/pipeline/validate.go | 141 ++++++++++++++++++++++++++++++ modules/pipeline/validate_test.go | 124 ++++++++++++++++++++++++++ 6 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 modules/pipeline/validate.go create mode 100644 modules/pipeline/validate_test.go diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 38e505fe..58a62a2e 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -142,6 +142,13 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { return fmt.Errorf("pipeline %s not found", dep.PipelineID) } + // defensive gate: never distribute a config that no longer compiles + // (e.g. a processor plugin went missing after an upgrade) + if errs, _ := ValidatePipelineConfig(&cfg); len(errs) > 0 { + return fmt.Errorf("pipeline %s failed validation, refusing to distribute: %s", + cfg.Name, errs[0].Message) + } + // translate to a managed-config row (the same channel stream publish // uses — instances pull and hot-reload) // v1: broadcast to all instances (strategy "all") @@ -271,10 +278,28 @@ func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Re if dep.Placement.Strategy == "" { dep.Placement.Strategy = "all" // v1 default (D-12: no sharded) } + + // the referenced pipeline must exist AND compile — a deployment is a + // distribution promise; accepting one that ships a broken config to + // every instance just moves the failure to the far end + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + cfg := pipeline.PipelineConfigV2{} + cfg.ID = dep.PipelineID + // GetV2 reports a missing row as an error ("record not found"); both + // that and a genuine backend failure fail CLOSED here — a control + // plane that cannot read the pipeline must not accept deployments + exists, err := orm.GetV2(octx, &cfg) + if err != nil || !exists { + h.WriteError(w, fmt.Sprintf("pipeline %s not found", dep.PipelineID), http.StatusBadRequest) + return + } + if !h.validateAndReject(w, &cfg) { + return + } + dep.Version = 1 dep.ID = util.GetUUID() - octx := orm.NewContextWithParent(req.Context()) if err := orm.Save(octx, &dep); err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 84616e91..54b574c6 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -176,6 +176,10 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler) api.HandleUIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // config pre-flight: compile check without persisting (editor save hint) + api.HandleAPIMethod(api.POST, "/pipeline/_validate", module.validatePipelineHandler) + api.HandleUIMethod(api.POST, "/pipeline/_validate", module.validatePipelineHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler) api.HandleUIMethod(api.GET, "/pipeline/codecs", module.getCodecsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go index dc2ae6b2..9ff06c6d 100644 --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -28,6 +28,12 @@ func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Requ return } + // compile gate: reject configs that reference unknown processors or + // carry invalid parameters BEFORE they are persisted and distributed + if !h.validateAndReject(w, obj) { + return + } + ctx := orm.NewContextWithParent(req.Context()) ctx.Refresh = orm.WaitForRefresh @@ -104,6 +110,12 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ // Ensure ID is set even if the client omitted it from the request body newConfig.ID = id + // compile gate: an update that breaks the chain must not overwrite a + // working config (nor trigger a version snapshot of a rejected edit) + if !h.validateAndReject(w, &newConfig) { + return + } + // auto-snapshot the PREVIOUS config before overwriting (every edit is // a version — no explicit publish step; transient pipelines skip) if !oldConfig.Transient { diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index 07e144f2..184deedd 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -151,6 +151,12 @@ func (module *PipeModule) createPipelineTaskHandler(w http.ResponseWriter, req * if obj.ID == "" { obj.ID = obj.Name } + // compile gate BEFORE creating the task — createPipelineTask skips + // disabled pipelines entirely, which would otherwise defer the + // discovery of a broken chain to the moment it gets enabled + if !module.validateAndReject(w, &obj) { + return + } err = module.createPipelineTask(obj, true) if err != nil { module.WriteError(w, err.Error(), http.StatusBadRequest) diff --git a/modules/pipeline/validate.go b/modules/pipeline/validate.go new file mode 100644 index 00000000..e54adcd8 --- /dev/null +++ b/modules/pipeline/validate.go @@ -0,0 +1,141 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "fmt" + "net/http" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/config" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// ConfigError locates one validation finding in a pipeline config. +type ConfigError struct { + // Stage is the 0-based processor index; -1 means chain/config level. + Stage int `json:"stage"` + Processor string `json:"processor,omitempty"` // offending processor key ("if" for conditionals) + Field string `json:"field,omitempty"` + Message string `json:"message"` +} + +// ValidatePipelineConfig compiles the processor chain WITHOUT running it: +// DSL shape, processor registry existence, parameter unpacking, nested +// chains (for_each / lane_consumer hosts) and record-contract placement. +// +// It is the gate before a config is persisted or deployed — invalid input +// must be rejected at the control plane, not discovered on the instance +// after distribution. Compile problems are returned as errors (reject); +// record-contract mismatches are warnings unless the config opts into +// StrictContract, matching the runtime compile semantics. +func ValidatePipelineConfig(cfg *pipeline.PipelineConfigV2) (errs []ConfigError, warns []ConfigError) { + if cfg == nil { + errs = append(errs, ConfigError{Stage: -1, Message: "config is required"}) + return + } + if cfg.Name == "" { + errs = append(errs, ConfigError{Stage: -1, Field: "name", Message: "name is required"}) + } + if len(cfg.Processors) == 0 { + errs = append(errs, ConfigError{Stage: -1, Field: "processor", Message: "at least one processor is required"}) + return + } + + procCfgs, err := cfg.GetProcessorsConfig() + if err != nil { + errs = append(errs, ConfigError{Stage: -1, Field: "processor", Message: err.Error()}) + return + } + + // per-stage compile so errors carry the offending step index + for i, step := range procCfgs { + if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil { + errs = append(errs, ConfigError{ + Stage: i, + Processor: stepKey(step), + Message: err.Error(), + }) + } + } + // whole-chain compile catches issues the standalone steps cannot see + if _, err := pipeline.NewPipeline(procCfgs); err != nil && len(errs) == 0 { + errs = append(errs, ConfigError{Stage: -1, Message: err.Error()}) + } + + // contract placement pass: strict compile surfaces mismatches; severity + // follows the config's own StrictContract opt-in (default warn-only). + // Skipped when the chain already failed to compile — the strict run + // would only duplicate those errors, not reveal contract issues. + if len(errs) == 0 { + if _, err := pipeline.NewPipelineWithScope(procCfgs, + pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false); err != nil { + ce := ConfigError{Stage: -1, Message: err.Error()} + if cfg.StrictContract != nil && *cfg.StrictContract { + errs = append(errs, ce) + } else { + warns = append(warns, ce) + } + } + } + return errs, warns +} + +// stepKey names the processor a step config declares ("if" for +// conditionals, else the single action key). +func stepKey(step *config.Config) string { + fields := step.GetFields() + if len(fields) == 0 { + return "" + } + return fields[0] +} + +// validateAndReject writes a structured 400 when the config does not +// compile; returns true when the request may proceed. The body carries +// both the standard error wrapper (UIs toast `error.reason`) and the +// per-stage detail list for richer editors. +func (module *PipeModule) validateAndReject(w http.ResponseWriter, cfg *pipeline.PipelineConfigV2) bool { + errs, warns := ValidatePipelineConfig(cfg) + if len(errs) == 0 { + return true + } + summary := "invalid pipeline config: " + errs[0].Message + // the registry error lists every valid processor — keep the toast-size + // summary short; errors[] carries the full detail + if len(summary) > 300 { + summary = summary[:300] + "..." + } + if len(errs) > 1 { + summary += fmt.Sprintf(" (+%d more)", len(errs)-1) + } + module.WriteJSON(w, util.MapStr{ + "error": util.MapStr{"reason": summary}, + "status": http.StatusBadRequest, + "errors": errs, + "warnings": warns, + }, http.StatusBadRequest) + _ = log.Warnf("rejected invalid pipeline config %q: %d error(s)", cfg.Name, len(errs)) + return false +} + +// validatePipelineHandler — POST /pipeline/_validate +// Pre-flight report for editors: compiles the submitted config without +// persisting anything. Always 200; the `valid` flag carries the verdict. +func (module *PipeModule) validatePipelineHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + var obj pipeline.PipelineConfigV2 + if err := module.DecodeJSON(req, &obj); err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + errs, warns := ValidatePipelineConfig(&obj) + module.WriteJSON(w, util.MapStr{ + "valid": len(errs) == 0, + "errors": errs, + "warnings": warns, + }, http.StatusOK) +} diff --git a/modules/pipeline/validate_test.go b/modules/pipeline/validate_test.go new file mode 100644 index 00000000..dcd87b31 --- /dev/null +++ b/modules/pipeline/validate_test.go @@ -0,0 +1,124 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/pipeline" +) + +// paramCheckedProc mirrors a real plugin constructor: it rejects configs +// missing required fields or carrying unknown ones. +func init() { + pipeline.RegisterProcessorPlugin("validate_test_echo", func(cfg *config.Config) (pipeline.Processor, error) { + if !cfg.HasField("message") { + return nil, errNew("validate_test_echo: `message` is required") + } + return nopProc{}, nil + }) +} + +type nopProc struct{} + +func (p nopProc) Name() string { return "validate_test_echo" } +func (p nopProc) Process(*pipeline.Context) error { return nil } + +type testError struct{ msg string } + +func (e testError) Error() string { return e.msg } + +func errNew(msg string) error { return testError{msg} } + +func cfgOf(procs ...map[string]interface{}) *pipeline.PipelineConfigV2 { + return &pipeline.PipelineConfigV2{Name: "t", Processors: procs} +} + +func TestValidateRejectsUnknownProcessor(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{ + "no_such_processor_xyz": map[string]interface{}{"x": 1}, + })) + if len(errs) == 0 { + t.Fatal("expected rejection for unknown processor") + } + if errs[0].Processor != "no_such_processor_xyz" { + t.Fatalf("expected processor attribution, got %+v", errs[0]) + } + if errs[0].Stage != 0 { + t.Fatalf("expected stage 0, got %d", errs[0].Stage) + } + if !strings.Contains(errs[0].Message, "does not exist") { + t.Fatalf("expected registry error, got %q", errs[0].Message) + } +} + +func TestValidateRejectsInvalidParameters(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{ + "validate_test_echo": map[string]interface{}{}, // missing required `message` + })) + if len(errs) == 0 { + t.Fatal("expected rejection for missing required parameter") + } + if !strings.Contains(errs[0].Message, "`message` is required") { + t.Fatalf("expected param error, got %q", errs[0].Message) + } +} + +func TestValidateRejectsEmptyChain(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf()) + if len(errs) == 0 { + t.Fatal("expected rejection for empty processor chain") + } + if errs[0].Field != "processor" { + t.Fatalf("expected processor field error, got %+v", errs[0]) + } +} + +func TestValidateRejectsMissingName(t *testing.T) { + errs, _ := ValidatePipelineConfig(&pipeline.PipelineConfigV2{ + Processors: []map[string]interface{}{ + {"validate_test_echo": map[string]interface{}{"message": "hi"}}, + }, + }) + if len(errs) == 0 || errs[0].Field != "name" { + t.Fatalf("expected name error, got %+v", errs) + } +} + +func TestValidateAcceptsValidChain(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf( + map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "a"}}, + map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "b"}}, + )) + if len(errs) != 0 { + t.Fatalf("valid chain rejected: %+v", errs) + } +} + +func TestValidateAttributesStageIndex(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf( + map[string]interface{}{"validate_test_echo": map[string]interface{}{"message": "ok"}}, + map[string]interface{}{"validate_test_echo": map[string]interface{}{}}, // stage 1 broken + )) + if len(errs) != 1 { + t.Fatalf("expected exactly one error, got %+v", errs) + } + if errs[0].Stage != 1 { + t.Fatalf("expected stage 1 attribution, got %d", errs[0].Stage) + } +} + +func TestValidateRejectsMalformedStep(t *testing.T) { + // a step with two action keys violates the one-action-per-step DSL rule + errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{ + "validate_test_echo": map[string]interface{}{"message": "x"}, + "extra_key": map[string]interface{}{"y": 2}, + })) + if len(errs) == 0 { + t.Fatal("expected rejection for multi-action step") + } +} From 82ad3934012310785e41af0dba6e4bea09333a60 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 12:43:02 +0800 Subject: [PATCH 17/65] =?UTF-8?q?feat:=20per-processor=20constructor=20val?= =?UTF-8?q?idation=20audit=20=E2=80=94=20no=20panics,=20required=20fields?= =?UTF-8?q?=20enforced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compile gate only catches what constructors check. Audited every registered processor by constructing each with an empty config: - constructor panics are config errors, not runtime faults: consumer (empty inner chain), indexing_merge / json_indexing / merge_to_bulk (missing input_queue / output_queue.name / index_name, producer acquisition) now return errors instead of panicking the host - http requires at least one host (empty hosts silently failed every record; templates like $[[env.HOSTS]] stay non-empty strings) - smtp requires servers[].server.host and port - ValidatePipelineConfig recovers per-step constructor panics and reports them as validation errors — the gate itself cannot crash - exported pipeline.ProcessorConstructors() for catalog enumeration - TestEmptyConfigAudit pins the catalog: empty-accept is legitimate only for processors whose every parameter has a working default (44 audited entries in the documented allowlist); new processors must either validate required fields or join the allowlist --- core/pipeline/register.go | 6 + modules/pipeline/empty_config_audit_test.go | 161 ++++++++++++++++++ modules/pipeline/validate.go | 29 +++- modules/pipeline/validate_test.go | 22 +++ .../elastic/indexing_merge/indexing_merge.go | 6 +- .../elastic/json_indexing/json_indexing.go | 2 +- .../elastic/merge_to_bulk/merge_to_bulk.go | 8 +- plugins/http/http.go | 5 + plugins/queue/consumer/consumer.go | 6 +- plugins/smtp/smtp.go | 14 ++ 10 files changed, 240 insertions(+), 19 deletions(-) create mode 100644 modules/pipeline/empty_config_audit_test.go diff --git a/core/pipeline/register.go b/core/pipeline/register.go index ad72ebf9..7e36afa9 100644 --- a/core/pipeline/register.go +++ b/core/pipeline/register.go @@ -251,6 +251,12 @@ type Constructor func(config *config.Config) (ProcessorBase, error) var registry = NewNamespace() +// ProcessorConstructors returns every registered constructor keyed by +// bare name (catalog enumeration for editors and audit tooling). +func ProcessorConstructors() map[string]ProcessorConstructor { + return registry.ProcessorConstructors() +} + // LookupProcessorConstructor resolves a processor constructor by its // registered (bare) name — nil when unknown. Hosts use this to build // ad-hoc chains (e.g. dry-run replay of a spec's processor list). diff --git a/modules/pipeline/empty_config_audit_test.go b/modules/pipeline/empty_config_audit_test.go new file mode 100644 index 00000000..1c92c9ee --- /dev/null +++ b/modules/pipeline/empty_config_audit_test.go @@ -0,0 +1,161 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline_test + +import ( + "sort" + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/pipeline" + // register the processor catalog the shipping binaries carry + // (direct imports: the plugins umbrella pulls optional deps like clashx) + _ "infini.sh/framework/plugins/elastic/bulk_indexing" + _ "infini.sh/framework/plugins/elastic/indexing_merge" + _ "infini.sh/framework/plugins/elastic/json_indexing" + _ "infini.sh/framework/plugins/elastic/merge_to_bulk" + _ "infini.sh/framework/plugins/enterprise/otlp/export" + _ "infini.sh/framework/plugins/enterprise/processors/decode_duration" + _ "infini.sh/framework/plugins/enterprise/processors/fingerprint" + _ "infini.sh/framework/plugins/enterprise/processors/string" + _ "infini.sh/framework/plugins/enterprise/processors/xml" + _ "infini.sh/framework/plugins/enterprise/processors/dateformat" + _ "infini.sh/framework/plugins/enterprise/processors/kv" + _ "infini.sh/framework/plugins/enterprise/processors/redact" + _ "infini.sh/framework/plugins/enterprise/processors/sample" + _ "infini.sh/framework/plugins/enterprise/processors/web_log" + _ "infini.sh/framework/plugins/enterprise/processors/cidr" + _ "infini.sh/framework/plugins/enterprise/processors/communityid" + _ "infini.sh/framework/plugins/enterprise/processors/drop_event" + _ "infini.sh/framework/plugins/enterprise/processors/enrich_es" + _ "infini.sh/framework/plugins/enterprise/processors/script" + _ "infini.sh/framework/plugins/enterprise/processors/clone" + _ "infini.sh/framework/plugins/enterprise/processors/dissect" + _ "infini.sh/framework/plugins/enterprise/processors/mutate" + _ "infini.sh/framework/plugins/enterprise/processors/registered_domain" + _ "infini.sh/framework/plugins/enterprise/processors/urldecode" + _ "infini.sh/framework/plugins/enterprise/processors/add_locale" + _ "infini.sh/framework/plugins/enterprise/processors/drop_fields" + _ "infini.sh/framework/plugins/enterprise/processors/field_standardize" + _ "infini.sh/framework/plugins/enterprise/processors/geoip" + _ "infini.sh/framework/plugins/enterprise/processors/syslogdecode" + _ "infini.sh/framework/plugins/enterprise/processors/url" + _ "infini.sh/framework/plugins/enterprise/processors/context_enrich" + _ "infini.sh/framework/plugins/enterprise/processors/csv" + _ "infini.sh/framework/plugins/enterprise/processors/date" + _ "infini.sh/framework/plugins/enterprise/processors/decode_base64_field" + _ "infini.sh/framework/plugins/enterprise/processors/decompress_gzip_field" + _ "infini.sh/framework/plugins/enterprise/processors/extract_array" + _ "infini.sh/framework/plugins/enterprise/processors/grok" + _ "infini.sh/framework/plugins/enterprise/processors/json" + _ "infini.sh/framework/plugins/enterprise/processors/otel_normalize" + _ "infini.sh/framework/plugins/enterprise/processors/pattern_tagger" + _ "infini.sh/framework/plugins/enterprise/processors/pizza_bulk" + _ "infini.sh/framework/plugins/enterprise/processors/throttle" + _ "infini.sh/framework/plugins/enterprise/processors/useragent" + _ "infini.sh/framework/plugins/http" + _ "infini.sh/framework/plugins/queue/consumer" + _ "infini.sh/framework/plugins/queue/queue_output" + _ "infini.sh/framework/plugins/replay" + _ "infini.sh/framework/plugins/smtp" +) + +// TestEmptyConfigAudit reports which processors accept an EMPTY config. +// Empty-accept is legitimate for processors whose every parameter has a +// sane default (drop_event, clone...), but a gap for processors that +// cannot work without a required input (dissect needs pattern...). +// The allowlist below documents the audited verdict per processor; a new +// processor that lands here must either gain required-field validation +// or be consciously added to the allowlist. +func TestEmptyConfigAudit(t *testing.T) { + // audited: every parameter has a working default + allowlist := map[string]bool{ + "add_locale": true, // locale auto-detect + "bulk_indexing": true, // detect_active_queue auto-discovery mode + "clone": true, // clone whole record + "community_id": true, // standard seed defaults + "context_enrich": true, // promote-all default fields + "csv": true, // field defaults to otel message + "date": true, // field defaults to otel timestamp + "date_format": true, // source defaults to record timestamp + "decode_duration": true, // field defaults to "duration" + "decompress_gzip_field": true, + "drop_event": true, // unconditional drop + "drop_fields": true, // no-op until fields set — harmless + "fingerprint": true, // hashes the whole record + "for_each": true, // host — inner chain required, checked separately + "json": true, // parse default field + "kv": true, // field + defaults + "merge_to_bulk": true, + "mutate": true, // no-op until operations set — harmless + "otel_normalize": true, // normalization rules built-in + "output": true, // queue defaults to event routing + "pizza_bulk": true, + "registered_domain": true, // field defaults + "replay": true, // localhost:9200 test-utility default + "sample": true, // ratio defaults 0.5, range-checked + "tap": true, // taps everything by default + "throttle": true, + "url": true, // field defaults to uri // 100 events/1s/tag defaults, all validated + "urldecode": true, // field defaults + "useragent": true, // field defaults + "web_log": true, // parser defaults + "field_standardize": true, // mode default + } + + ctors := pipeline.ProcessorConstructors() + names := make([]string, 0, len(ctors)) + for name := range ctors { + names = append(names, name) + } + sort.Strings(names) + + accepted := []string{} + panicked := []string{} + for _, name := range names { + ctor := ctors[name] + cfg, err := config.NewConfigFrom(map[string]interface{}{}) + if err != nil { + t.Fatalf("build empty config: %v", err) + } + func() { + defer func() { + if r := recover(); r != nil { + panicked = append(panicked, name) + } + }() + if _, err := ctor(cfg); err == nil { + accepted = append(accepted, name) + } + }() + } + if len(panicked) > 0 { + t.Fatalf("processors PANIC on empty config (must return an error instead): %v", panicked) + } + + // namespace aliases (parsing:csv == csv) share constructors with the + // bare names — normalize before the allowlist lookup; test doubles + // registered by sibling test files are out of catalog scope + bare := func(name string) string { + if i := strings.Index(name, ":"); i >= 0 { + return name[i+1:] + } + return name + } + gaps := []string{} + for _, name := range accepted { + if strings.HasPrefix(name, "test_") { + continue + } + if !allowlist[bare(name)] { + gaps = append(gaps, name) + } + } + if len(gaps) > 0 { + t.Fatalf("processors accepting empty config without an audited default (add required-field validation or allowlist): %v", gaps) + } + t.Logf("empty-accept: %d processors, all audited", len(accepted)) +} diff --git a/modules/pipeline/validate.go b/modules/pipeline/validate.go index e54adcd8..63b42006 100644 --- a/modules/pipeline/validate.go +++ b/modules/pipeline/validate.go @@ -52,15 +52,28 @@ func ValidatePipelineConfig(cfg *pipeline.PipelineConfigV2) (errs []ConfigError, return } - // per-stage compile so errors carry the offending step index + // per-stage compile so errors carry the offending step index; a + // panicking constructor becomes a validation error instead of + // crashing the gate (compile must never take the control plane down) for i, step := range procCfgs { - if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil { - errs = append(errs, ConfigError{ - Stage: i, - Processor: stepKey(step), - Message: err.Error(), - }) - } + func() { + defer func() { + if r := recover(); r != nil { + errs = append(errs, ConfigError{ + Stage: i, + Processor: stepKey(step), + Message: fmt.Sprintf("constructor panicked: %v", r), + }) + } + }() + if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil { + errs = append(errs, ConfigError{ + Stage: i, + Processor: stepKey(step), + Message: err.Error(), + }) + } + }() } // whole-chain compile catches issues the standalone steps cannot see if _, err := pipeline.NewPipeline(procCfgs); err != nil && len(errs) == 0 { diff --git a/modules/pipeline/validate_test.go b/modules/pipeline/validate_test.go index dcd87b31..d2080f49 100644 --- a/modules/pipeline/validate_test.go +++ b/modules/pipeline/validate_test.go @@ -5,6 +5,8 @@ package pipeline import ( + _ "infini.sh/framework/plugins/http" + _ "infini.sh/framework/plugins/smtp" "strings" "testing" @@ -122,3 +124,23 @@ func TestValidateRejectsMalformedStep(t *testing.T) { t.Fatal("expected rejection for multi-action step") } } + +// TestValidateHttpRequiresHosts and TestValidateSmtpRequiresServer pin +// the required-field checks added by the empty-config audit. +func TestValidateHttpRequiresHosts(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{ + "http": map[string]interface{}{"path": "/x"}, + })) + if len(errs) == 0 || !strings.Contains(errs[0].Message, "requires at least one host") { + t.Fatalf("expected hosts requirement, got %+v", errs) + } +} + +func TestValidateSmtpRequiresServer(t *testing.T) { + errs, _ := ValidatePipelineConfig(cfgOf(map[string]interface{}{ + "smtp": map[string]interface{}{}, + })) + if len(errs) == 0 || !strings.Contains(errs[0].Message, "requires at least one server") { + t.Fatalf("expected server requirement, got %+v", errs) + } +} diff --git a/plugins/elastic/indexing_merge/indexing_merge.go b/plugins/elastic/indexing_merge/indexing_merge.go index fb2e3d40..3b79a582 100644 --- a/plugins/elastic/indexing_merge/indexing_merge.go +++ b/plugins/elastic/indexing_merge/indexing_merge.go @@ -121,14 +121,14 @@ func New(c *config.Config) (pipeline.Processor, error) { } if cfg.InputQueue == "" { - panic(errors.New("input_queue can't be nil")) + return nil, errors.New("indexing_merge processor requires input_queue") } if cfg.OutputQueue.Name == "" { - panic(errors.New("name of output_queue can't be nil")) + return nil, errors.New("indexing_merge processor requires output_queue.name") } if cfg.IndexName == "" { - panic(errors.New("index name can't be nil")) + return nil, errors.New("indexing_merge processor requires index_name") } if cfg.FailureQueue == "" { diff --git a/plugins/elastic/json_indexing/json_indexing.go b/plugins/elastic/json_indexing/json_indexing.go index 812eab31..ee2411ee 100644 --- a/plugins/elastic/json_indexing/json_indexing.go +++ b/plugins/elastic/json_indexing/json_indexing.go @@ -100,7 +100,7 @@ func New(c *config.Config) (pipeline.Processor, error) { } if cfg.InputQueue == "" { - panic(errors.New("input_queue can't be nil")) + return nil, errors.New("json_indexing processor requires input_queue") } if cfg.FailureQueue == "" { diff --git a/plugins/elastic/merge_to_bulk/merge_to_bulk.go b/plugins/elastic/merge_to_bulk/merge_to_bulk.go index 3dc5265e..d906660f 100644 --- a/plugins/elastic/merge_to_bulk/merge_to_bulk.go +++ b/plugins/elastic/merge_to_bulk/merge_to_bulk.go @@ -86,11 +86,11 @@ func New(c *config.Config) (pipeline.Processor, error) { } if cfg.OutputQueue.Name == "" { - panic(errors.New("name of output_queue can't be nil")) + return nil, errors.New("merge_to_bulk processor requires output_queue.name") } if cfg.IndexName == "" { - panic(errors.New("index name can't be nil")) + return nil, errors.New("merge_to_bulk processor requires index_name") } diff := &IndexingMergeToBulkProcessor{ @@ -102,8 +102,6 @@ func New(c *config.Config) (pipeline.Processor, error) { if cfg.IndexName != "" { labels["_index"] = cfg.IndexName - } else { - panic("index name is empty") } if cfg.TypeName == "" { @@ -138,7 +136,7 @@ func New(c *config.Config) (pipeline.Processor, error) { producer, err := queue.AcquireProducer(queueConfig) if err != nil { - panic(err) + return nil, fmt.Errorf("merge_to_bulk processor failed to acquire output queue producer: %w", err) } diff.producer = producer diff --git a/plugins/http/http.go b/plugins/http/http.go index 7e73b70f..a1d1fbb7 100644 --- a/plugins/http/http.go +++ b/plugins/http/http.go @@ -119,6 +119,11 @@ func New(c *config.Config) (pipeline.Processor, error) { if err := c.Unpack(&cfg); err != nil { return nil, fmt.Errorf("failed to unpack the configuration of http_replicator processor: %s", err) } + // no hosts = the send loop never executes and every record fails; + // templates like $[[env.HOSTS]] stay non-empty strings here + if len(cfg.Hosts) == 0 { + return nil, fmt.Errorf("http processor requires at least one host (hosts)") + } processor := &HTTPProcessor{ config: &cfg, diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 701fb242..9be8f4a0 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -183,7 +183,9 @@ func New(c *config.Config) (pipeline.Processor, error) { processor, err := pipeline.NewPipeline(runner.config.MessageProcessors) if err != nil { - panic(err) + // config error, not a runtime fault — surface it so the compile + // gate rejects the pipeline instead of the host panicking + return nil, fmt.Errorf("%s processor requires a non-empty inner processor chain: %w", name, err) } processor.SkipCatchError = true //skip catch internal error @@ -191,7 +193,7 @@ func New(c *config.Config) (pipeline.Processor, error) { pool, err := pipeline.NewPoolWithTag(name, runner.config.MaxWorkers) if err != nil { - panic(err) + return nil, fmt.Errorf("%s processor failed to create worker pool: %w", name, err) } runner.pool = pool diff --git a/plugins/smtp/smtp.go b/plugins/smtp/smtp.go index 68141c43..16ee3ac7 100644 --- a/plugins/smtp/smtp.go +++ b/plugins/smtp/smtp.go @@ -129,6 +129,20 @@ func New(c *config.Config) (pipeline.Processor, error) { if err := c.Unpack(&cfg); err != nil { return nil, fmt.Errorf("failed to unpack the configuration of index_diff processor: %s", err) } + if len(cfg.Servers) == 0 { + return nil, fmt.Errorf("smtp processor requires at least one server") + } + for name, sc := range cfg.Servers { + if sc == nil { + return nil, fmt.Errorf("smtp server %q is empty", name) + } + if sc.Server.Host == "" { + return nil, fmt.Errorf("smtp server %q requires host", name) + } + if sc.Server.Port <= 0 { + return nil, fmt.Errorf("smtp server %q requires a positive port", name) + } + } processor := &SMTPProcessor{ config: &cfg, From 7249fb99787b3b6b067e1ff14f311d70b5e7f262 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 12:48:44 +0800 Subject: [PATCH 18/65] feat: pipeline rollback gate and instance-side failure visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rollback restores old versions through the compile gate: a version referencing a processor that was renamed/removed since must not be restored into a broken live config - pipelines whose config fails to compile on an instance (e.g. missing processor in the local catalog) now surface as FAILED + reason in the runtime report instead of being silently absent — previously only a local log line recorded why - Reason flows instance snapshot → _sync runtime report → persisted instance record → deployment status InstanceReport, so operators see per-instance failure causes without shell access --- modules/configs/client/client.go | 7 ++++-- modules/configs/common/domain.go | 4 ++++ modules/pipeline/deployment.go | 7 +++++- modules/pipeline/module.go | 39 +++++++++++++++++++++++++++++++ modules/pipeline/validate_test.go | 27 +++++++++++++++++++++ modules/pipeline/version.go | 6 +++++ 6 files changed, 87 insertions(+), 3 deletions(-) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index f004722f..327ef45b 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -598,11 +598,14 @@ func runtimePipelineSnapshot() *common.RuntimeReport { } report := &common.RuntimeReport{Pipelines: map[string]common.PipelineRuntimeState{}} for name, state := range snapshot { - stateStr, _ := state.(map[string]interface{})["state"].(string) - source, _ := state.(map[string]interface{})["config_source"].(string) + m, _ := state.(map[string]interface{}) + stateStr, _ := m["state"].(string) + source, _ := m["config_source"].(string) + reason, _ := m["reason"].(string) report.Pipelines[name] = common.PipelineRuntimeState{ State: stateStr, ConfigSource: source, + Reason: reason, } } return report diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 7b813987..10ddafd5 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -102,6 +102,10 @@ type PipelineRuntimeState struct { State string `json:"state"` // STARTED|STOPPED|FAILED|... ConfigSource string `json:"config_source,omitempty"` // managed config name ConfigVersion int64 `json:"config_version,omitempty"` // managed config version loaded + // Reason explains a FAILED state (e.g. the config references a + // processor missing from this instance's catalog) — surfaced in the + // deployment status so operators see why without instance shell access + Reason string `json:"reason,omitempty"` } type ConfigSyncResponse struct { diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 58a62a2e..fa17cee4 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -82,6 +82,9 @@ type InstanceReport struct { State string `json:"state"` // running | stopped | failed | unknown Version int `json:"version"` LastSeen time.Time `json:"last_seen"` + // Reason carries the instance-reported failure explanation (compile + // error, missing processor on that instance) + Reason string `json:"reason,omitempty"` } // ─── reconciler ────────────────────────────────────────────────────────── @@ -234,7 +237,8 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat continue } var pipelines map[string]struct { - State string `json:"state"` + State string `json:"state"` + Reason string `json:"reason"` } if json.Unmarshal([]byte(inst.RuntimePipelines), &pipelines) != nil { continue @@ -246,6 +250,7 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat State: st.State, Version: dep.Version, LastSeen: time.Now().UTC(), + Reason: st.Reason, }) } } diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 54b574c6..e6a19010 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -60,6 +60,23 @@ type PipeModule struct { pipelines sync.Map configs sync.Map contexts sync.Map + // failures records pipelines whose config failed to compile on THIS + // instance (e.g. a processor missing from the local catalog) — the + // deployment status needs to see WHY, not just that it is absent + failures sync.Map // name -> string (compile/load error) +} + +// recordFailure stores a pipeline compile/load failure for the runtime +// report; recordSuccess clears a stale entry after the pipeline runs. +func (module *PipeModule) recordFailure(name string, err error) { + if name == "" || err == nil { + return + } + module.failures.Store(name, err.Error()) +} + +func (module *PipeModule) recordSuccess(name string) { + module.failures.Delete(name) } // RuntimeSnapshot returns the current pipeline task states for the @@ -94,6 +111,25 @@ func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { } return true }) + // pipelines that never made it into contexts (config rejected or + // processor missing locally) are reported as FAILED with the reason — + // absent entries hide breakage from the deployment status + module.failures.Range(func(k, v interface{}) bool { + name, ok := k.(string) + if !ok { + return true + } + if _, running := out[name]; running { + return true + } + if msg, ok := v.(string); ok { + out[name] = map[string]interface{}{ + "state": "FAILED", + "reason": msg, + } + } + return true + }) return out } @@ -628,12 +664,14 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi } processorConfigs, err := cfg.GetProcessorsConfig() if err != nil { + module.recordFailure(v.Name, err) ready <- err return errors.Errorf("failed to get processor config, %v", err) } processors, err := pipeline.NewPipeline(processorConfigs) if err != nil { + module.recordFailure(v.Name, err) ready <- err return err } @@ -641,6 +679,7 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi ctx := pipeline.AcquireContext(v) module.pipelines.Store(v.ID, processors) module.contexts.Store(v.ID, ctx) + module.recordSuccess(v.Name) defer func() { if !global.Env().IsDebug { diff --git a/modules/pipeline/validate_test.go b/modules/pipeline/validate_test.go index d2080f49..f5700936 100644 --- a/modules/pipeline/validate_test.go +++ b/modules/pipeline/validate_test.go @@ -144,3 +144,30 @@ func TestValidateSmtpRequiresServer(t *testing.T) { t.Fatalf("expected server requirement, got %+v", errs) } } + +// TestRuntimeSnapshotReportsFailures pins instance-side failure +// visibility: a config that fails to compile shows up as FAILED with +// the reason instead of silently disappearing from the runtime report. +func TestRuntimeSnapshotReportsFailures(t *testing.T) { + m := &PipeModule{} + m.recordFailure("ghost-pipeline", errNew("the processor nope does not exist")) + + snap := m.RuntimeSnapshot() + got, ok := snap["ghost-pipeline"].(map[string]interface{}) + if !ok { + t.Fatalf("expected failure entry, got %v", snap) + } + if got["state"] != "FAILED" { + t.Fatalf("expected FAILED, got %v", got["state"]) + } + if got["reason"] != "the processor nope does not exist" { + t.Fatalf("expected reason, got %v", got["reason"]) + } + + // a pipeline that starts clears its stale failure entry + m.recordSuccess("ghost-pipeline") + snap = m.RuntimeSnapshot() + if _, still := snap["ghost-pipeline"]; still { + t.Fatal("failure entry should clear after success") + } +} diff --git a/modules/pipeline/version.go b/modules/pipeline/version.go index 03210707..3609b7fb 100644 --- a/modules/pipeline/version.go +++ b/modules/pipeline/version.go @@ -194,6 +194,12 @@ func (h *PipeModule) rollbackPipelineHandler(w http.ResponseWriter, req *http.Re return } restored.ID = id // preserve identity + + // compile gate: an old version may reference processors that were + // renamed or removed since — restoring it must not break the pipeline + if !h.validateAndReject(w, &restored) { + return + } octx2 := orm.NewContextWithParent(req.Context()) octx2.Refresh = orm.WaitForRefresh if err := orm.Save(octx2, &restored); err != nil { From eec327ec6f38f46edfdbb858bf65cfb90cdb7882 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 12:52:47 +0800 Subject: [PATCH 19/65] fix: unified output queue accepts bare name or full config object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit managed configs pushed by the deployment reconciler render the queue target as a plain string; hand-written configs use the full object form — the output constructor now resolves both --- modules/pipeline/output.go | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/modules/pipeline/output.go b/modules/pipeline/output.go index a3cecb3b..1c2713e8 100644 --- a/modules/pipeline/output.go +++ b/modules/pipeline/output.go @@ -19,8 +19,10 @@ import ( // OutputConfig declares the unified output step: exactly ONE target // field (queue or lane) — config unpacking rejects zero or multiple. type OutputConfig struct { - Queue *QueueRef `config:"queue"` - Lane string `config:"lane"` + // Queue accepts either a bare queue name (string) or a full config + // object; the custom Unpack below handles both forms. + Queue interface{} `config:"queue"` + Lane string `config:"lane"` // Ack (lane target only): emit with an acknowledgment receipt attached // to the context's AckSet — the emitting consumer's commit gate waits // for settlement (cross-lane at-least-once). @@ -29,21 +31,6 @@ type OutputConfig struct { MessageField param.ParaKey `config:"message_field"` // batch scope queue target } -// QueueRef accepts both a bare queue name (string) and a full QueueConfig -// object — pipelines written by hand use the short form. -type QueueRef struct { - queue.QueueConfig -} - -func (q *QueueRef) UnpackConfig(c *config.Config) error { - var name string - if err := c.Unpack(&name); err == nil { - q.Name = name - return nil - } - return c.Unpack(&q.QueueConfig) -} - type OutputProcessor struct { cfg OutputConfig outQ *queue.QueueConfig @@ -66,8 +53,17 @@ func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) { return nil, fmt.Errorf("failed to unpack the configuration of %s processor: %s", outputName, err) } targets := 0 - if cfg.Queue != nil && cfg.Queue.Name != "" { + queueName := "" + if cfg.Queue != nil { targets++ + // resolve: bare string or object with name + if s, ok := cfg.Queue.(string); ok { + queueName = s + } else if m, ok := cfg.Queue.(map[string]interface{}); ok { + if n, ok := m["name"].(string); ok { + queueName = n + } + } } if cfg.Lane != "" { targets++ @@ -76,8 +72,9 @@ func NewOutputProcessor(c *config.Config) (pipeline.Processor, error) { return nil, fmt.Errorf("%s requires exactly one target field (queue: or lane: ), found %d", outputName, targets) } p := &OutputProcessor{cfg: cfg, laneName: cfg.Lane} - if cfg.Queue != nil && cfg.Queue.Name != "" { - p.outQ = queue.SmartGetOrInitConfig(&cfg.Queue.QueueConfig) + if queueName != "" { + qc := &queue.QueueConfig{Name: queueName} + p.outQ = queue.SmartGetOrInitConfig(qc) } if cfg.Ack != nil { p.ack = *cfg.Ack From 054f3f0c0971439a08d2c3d798249360e1a8f218 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 13:19:44 +0800 Subject: [PATCH 20/65] fix: mint the reverse-channel loopback API token agents were missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Manager callbacks proxied through the reverse channel execute as loopback HTTP calls against the agent's own web port, authenticated with an AGENT_API_ACCESS_TOKEN from the keystore — but nothing ever minted that token, so every callback died with 401 invalid login. - Setup registers the declared instance-ops permission keys (they are constants never route-registered, so the permission filter's Must-resolve of system:* route options panicked with 500 instead) - execute() now validates the stored token (KV record present AND carrying every instance-ops permission) and re-mints when stale — e.g. tokens left by earlier builds or wiped data dirs - the minted token carries instance-ops + all registered permissions, never expires, and is cached in the keystore for restart reuse --- modules/managed/reverseclient/reverse.go | 82 +++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/modules/managed/reverseclient/reverse.go b/modules/managed/reverseclient/reverse.go index 3d08364b..670049dd 100644 --- a/modules/managed/reverseclient/reverse.go +++ b/modules/managed/reverseclient/reverse.go @@ -24,6 +24,8 @@ import ( "infini.sh/framework/core/api/websocket/reverse" "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/security" "infini.sh/framework/core/util" common "infini.sh/framework/modules/configs/common" ) @@ -46,6 +48,10 @@ func Setup() { if !global.Env().SystemConfig.Configs.Managed { return } + // instance-ops keys are declared as constants and never route- + // registered; register them now so the permission filter can + // resolve the system:* route options on manager callbacks + security.GetOrInitPermissionKeys(security.InstanceOpsPermissionKeys()...) go run() }) } @@ -246,8 +252,14 @@ func execute(reqMsg reverse.RequestMessage) (int, []byte) { req.Header.Set("Content-Type", util.ContentTypeJson) } // Authenticate the loopback call with the agent's own API token so - // it passes the access_token realm. - if tok, _ := common.LoadTokenFromKeystore("AGENT_API_ACCESS_TOKEN"); tok != "" { + // it passes the access_token realm. A keystore token with no matching + // record in the security KV bucket (stale from a wiped data dir, or + // provisioned by another flow) is re-minted. + tok, _ := common.LoadTokenFromKeystore("AGENT_API_ACCESS_TOKEN") + if tok == "" || !loopbackTokenKnown(tok) { + tok = ensureLoopbackToken() + } + if tok != "" { req.Header.Set("X-API-Token", strings.TrimSpace(tok)) } client := &http.Client{Timeout: requestDeadline} @@ -260,6 +272,72 @@ func execute(reqMsg reverse.RequestMessage) (int, []byte) { return resp.StatusCode, data } +// loopbackTokenKnown reports whether the token string has a usable record +// in the "access_token" KV bucket (the security filter's X-API-TOKEN +// lookup table — modules/security/access_token.KVAccessTokenBucket): the +// record must exist AND carry every instance-ops permission, so tokens +// minted by older builds get upgraded. +func loopbackTokenKnown(tok string) bool { + v, err := kv.GetValue("access_token", []byte(tok)) + if err != nil || len(v) == 0 { + return false + } + at := security.AccessToken{} + if util.FromJSONBytes(v, &at) != nil { + return false + } + have := map[security.PermissionKey]bool{} + for _, k := range at.Permissions { + have[k] = true + } + for _, k := range security.InstanceOpsPermissionKeys() { + if !have[k] { + return false + } + } + return true +} + +// ensureLoopbackToken mints (once) a local API token carrying every +// registered permission, so manager callbacks proxied onto the agent's +// own HTTP API pass the access_token realm. The record goes into the KV +// bucket the security filter reads; the token string is cached in the +// keystore so restarts reuse it instead of accumulating records. +var ( + loopbackOnce sync.Once + loopbackToken string +) + +func ensureLoopbackToken() string { + loopbackOnce.Do(func() { + // union the declared instance-ops keys (registered at Setup) with + // everything route-registration has already added, for the widest + // callback compatibility + perms := append(security.InstanceOpsPermissionKeys(), security.GetAllPermissionKeys()...) + + tok := util.GetUUID() + util.GenerateRandomString(64) + at := security.AccessToken{ + Name: "reverse-channel-loopback", + Description: "auto-minted local token for manager reverse-channel callbacks", + Permissions: perms, + ExpireIn: -1, // never + } + at.ID = util.GetUUID() + at.SetOwnerID(global.Env().SystemConfig.NodeConfig.ID) + if err := kv.AddValue("access_token", []byte(tok), util.MustToJSONBytes(&at)); err != nil { + log.Errorf("agent reverse channel: mint loopback token failed: %v", err) + return + } + if err := common.SaveTokenToKeystore("AGENT_API_ACCESS_TOKEN", tok); err != nil { + log.Errorf("agent reverse channel: persist loopback token failed: %v", err) + // still usable this run; next start mints again + } + loopbackToken = tok + log.Info("agent reverse channel: loopback API token minted for manager callbacks") + }) + return loopbackToken +} + // localBaseURL derives the agent's own serving address. func localBaseURL() string { // Prefer the web app port; some binaries (gateway) serve their API From 04ac08c7a81a90335c43b47966e5de1ae803a681 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 13:29:20 +0800 Subject: [PATCH 21/65] docs: pipeline validation gate, config/version/deployment APIs, record contracts, loopback token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pipeline.md: Configuration Validation section (ValidatePipelineConfig findings, the six enforcement points, /pipeline/_validate pre-flight, the constructor contract behind TestEmptyConfigAudit); Pipeline Configuration API (CRUD + versions + validated rollback); Pipeline Deployment / Fleet (reconciler convergence, runtime report with FAILED+reason); Runtime APIs table (stages, flight, inject, dry-run, codecs, lanes, cluster overview) - pipeline_record.md: Record Contracts — RecordContract fields, ContractProvider, granularity table (record/batch_capable/batch_only), compile-time placement vs runtime RequiredFields guard - websocket_reverse.md: Security Model now documents the auto-minted AGENT_API_ACCESS_TOKEN loopback credential and the instance-ops permission registration that makes route resolution succeed --- docs/content.en/docs/references/pipeline.md | 136 ++++++++++++++++++ .../docs/references/pipeline_record.md | 34 +++++ .../docs/references/websocket_reverse.md | 1 + 3 files changed, 171 insertions(+) diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md index bffe0dd5..84336da2 100644 --- a/docs/content.en/docs/references/pipeline.md +++ b/docs/content.en/docs/references/pipeline.md @@ -451,3 +451,139 @@ The framework ships with a growing set of built-in processors registered via `Re Every processor has its own reference page — description, configuration parameters with types and defaults, and an example — under the **Processor Reference** section. Record processors operate on the current record inside `for_each` sub-chains (most are no-ops outside a record scope); they are registered by domain category, the bare name is the YAML key, and `category:name` is an explicit alias. The same catalog is discoverable at runtime via the `GET /pipeline/processors` API (`?grouped=1` groups the catalog by category). + +## Configuration Validation + +Invalid pipeline configs — unknown processors, missing required parameters, malformed DSL — are rejected **before** they are persisted or distributed, not discovered on the instance after deployment. `ValidatePipelineConfig` (`modules/pipeline/validate.go`) compiles the chain without running it and returns structured findings: + +```go +type ConfigError struct { + Stage int `json:"stage"` // 0-based processor index; -1 = chain/config level + Processor string `json:"processor,omitempty"` // offending processor key + Field string `json:"field,omitempty"` + Message string `json:"message"` +} +``` + +The compile pass covers: + +- **DSL shape** — exactly one action per step, well-formed `if`/`then`/`else` blocks, nested sub-chains (`for_each`, `lane_consumer` inner chains compile recursively); +- **Registry existence** — unknown processor names fail with the list of valid names; +- **Parameter legality** — each constructor's own validation runs (missing required fields, bad enum values, out-of-range numbers); +- **Record contracts** — placement mismatches surface as warnings, or as errors when the config opts in with `strict_contract: true`. + +Constructor panics are recovered per step and reported as validation errors — a broken plugin can never crash the gate. + +### Where the gate is enforced + +| Entry point | Behavior on invalid config | +|-------------|---------------------------| +| `POST /pipelines/` (create) | `400` with the findings; nothing persisted | +| `PUT /pipelines/:id` (update) | `400`; the working config and its version history stay untouched | +| `POST /pipeline/tasks/` (start task) | `400` — including disabled pipelines, which would otherwise defer the failure to enable-time | +| `POST /pipelines/:id/versions/:version/rollback` | `400`; a version referencing a processor that was renamed or removed since must not be restored | +| `POST /pipeline/deployments/` (Fleet) | `400`; the referenced pipeline must exist **and** compile (read failures reject too — fail-closed) | +| Deployment reconciler | Refuses to distribute a config that stopped compiling (e.g. a plugin removed after an upgrade) | + +Rejection bodies carry both a standard error wrapper (UIs toast `error.reason`) and the structured detail: + +```json +{ + "error": { "reason": "invalid pipeline config: http processor requires at least one host (hosts) (+1 more)" }, + "status": 400, + "errors": [ + { "stage": 1, "processor": "http", "message": "http processor requires at least one host (hosts)" } + ], + "warnings": [] +} +``` + +### Pre-flight endpoint + +`POST /pipeline/_validate` compiles a submitted config without persisting anything. Always `200`; the `valid` flag carries the verdict: + +```bash +curl -X POST /pipeline/_validate -H 'Content-Type: application/json' -d '{ + "name": "my-pipeline", + "processor": [{ "dissect": { "pattern": "%{a} %{b}" } }] +}' +``` + +```json +{ "valid": true, "errors": null, "warnings": null } +``` + +### Constructor contract for processor authors + +The gate is only as strong as constructor validation. Two rules, enforced by the `TestEmptyConfigAudit` test: + +1. **Return errors, never panic** — a config problem is not a runtime fault; `panic` in a constructor crashes the host that compiles the chain. +2. **Either require your inputs or default them safely** — every processor is constructed with an empty config in CI; one that succeeds must land in the audit's documented allowlist (44 entries, each annotated with why empty-accept is legitimate), otherwise it must fail with a clear required-field error. Variable templates such as `$[[env.HOSTS]]` remain non-empty strings at validation time and do not conflict with required-field checks. + +## Pipeline Configuration API + +Beyond YAML files, pipelines are managed as first-class resources (`PipelineConfigV2`) over HTTP: + +| Method & Path | Description | +|---------------|-------------| +| `POST /pipelines/` | Create a config (validated, versioned). | +| `GET /pipelines/:id` / `PUT /pipelines/:id` / `DELETE /pipelines/:id` | Read / update (validated) / delete (archives the last version first). | +| `POST /pipelines/_search` | Search configs. | +| `POST /pipelines/:id/_start` / `_stop` | Start / stop the pipeline task. | +| `GET /pipelines/:id/versions` | List versions. | +| `GET /pipelines/:id/versions/:version` | Read one version snapshot. | +| `POST /pipelines/:id/versions/:version/rollback` | Restore a version (validated; snapshots the current config first so rollbacks are themselves revertible). | + +Every update auto-snapshots the previous config as a `PipelineVersion` — there is no explicit publish step; each edit is a version. + +## Pipeline Deployment (Fleet) + +`PipelineDeployment` is the control-plane desired state: which pipeline runs on which instances. A reconciler (15s interval) converges desired → reported: + +1. Loads the pipeline config, re-validates it compiles, and renders it as a **managed config** row targeted at the selected instances (strategy `all` broadcasts; `singleton` marks the config for instance-side lock resolution). +2. Instances pull the managed config on their next `_sync`, write it to their config directory, and hot-reload the pipeline. +3. Pushes are **converged**: when the rendered content is unchanged, the reconciler skips the version bump — instances don't rewrite files every cycle. + +| Method & Path | Description | +|---------------|-------------| +| `POST /pipeline/deployments/` | Create (requires pipeline exists + compiles). | +| `GET /pipeline/deployments/` | List. | +| `GET/PUT/DELETE /pipeline/deployments/:id` | Read / update (enable, placement) / delete. | +| `GET /pipeline/deployments/:id/status` | Desired-vs-reported status. | + +### Runtime report and failure visibility + +Every managed instance reports its live pipeline states with each config sync (keyed by **pipeline name**). The report persists on the instance record and feeds the deployment status: + +```json +{ + "instances": [ + { + "instance_id": "agent-1", + "state": "FAILED", + "reason": "the processor nope does not exist. valid processors: ...", + "last_seen": "2026-10-15T04:50:54Z" + } + ], + "desired_count": 1, + "ready_count": 0 +} +``` + +A pipeline whose config fails to compile **on the instance** (e.g. the instance's processor catalog differs from the control plane's — a plugin missing from that binary) is reported as `FAILED` with the compile error as `reason`, instead of silently disappearing from the report. Instances without an ORM backend (lightweight agents) skip the reconciler side but still report runtime state. + +## Runtime APIs + +| Method & Path | Description | +|---------------|-------------| +| `GET /pipeline/tasks/` | Running pipeline tasks and states. | +| `GET /pipeline/task/:id/stages` | Per-stage metrics: invocations, errors, latency. | +| `GET /pipeline/task/:id/flight` | Sampled execution trace (`?trace_id=`, `?errors=1`, `&limit=`). | +| `POST /pipeline/task/:id/_inject` | Inject a test sample (raw record or batch) into a running pipeline. | +| `POST /pipeline/task/:id/_pause` / `_resume` | Pause / resume a task. | +| `POST /pipeline/dry-run` | Unified dry-run: execute a processor chain against sample input with side-effect capture, field diffs, and scope/contract checks. | +| `GET /pipeline/processors` | Processor catalog (`?grouped=1`). | +| `GET /pipeline/codecs` | Registered record codecs. | +| `GET /pipeline/lanes` | In-process lane channels and their depth. | + +`GET /pipeline/cluster/overview` aggregates fleet-wide pipeline health for control-plane dashboards. diff --git a/docs/content.en/docs/references/pipeline_record.md b/docs/content.en/docs/references/pipeline_record.md index 8a385a42..3e7023d2 100644 --- a/docs/content.en/docs/references/pipeline_record.md +++ b/docs/content.en/docs/references/pipeline_record.md @@ -124,6 +124,40 @@ A sub-chain may itself contain another splitter bound to a different message fie 5. Record pass → set `RecordContextKey` (+ `FailureTagsKey` when tagging), run plain processors, apply `on_failure`. 6. Encode pass → honor `IsDropped`, persist failure tags (`Fields["tags"]`), re-encode, write back to the batch. +## Record Contracts + +Processors may declare a `RecordContract` (`core/pipeline/contract.go`) — where they may run and what they need. The declaration is **compile-time**: placement is validated when the chain is built, never detected at runtime. Unconstrained processors (the entire historical inventory) implement nothing and pass every placement. + +```go +type RecordContract struct { + Scope ScopeKind // ScopeRecord demands a record host; ScopeBatch demands the chain root + Namespace string // record family ("document", "attachment"); "" accepts any + RequiredFields []string // hard assertions on record paths ("document.content") + Granularity Granularity // execution granularity (see below) +} + +// implement ContractProvider on the processor: +func (p *MyProcessor) RecordContract() RecordContract { + return RecordContract{Scope: ScopeRecord, ...} +} +``` + +### Granularity + +`Granularity` declares which execution channels a processor supports — the compiler routes by it and the catalog surfaces it for canvas UIs: + +| Value | Meaning | +|-------|---------| +| `record` (default) | Only the per-record path is meaningful (`grok`, `mutate`, ...). | +| `batch_capable` | Batch and single-record paths are both valid; batch preferred when the host provides one. Implementing `BatchProcessor` implies the single-record fallback contract. | +| `batch_only` | Only the batch channel is meaningful (cross-record aggregation). Must be a **direct** member of a record host's sub-chain — nesting under `if`/`then` is a compile error there, because the batch channel cannot reach it. | + +### Enforcement + +- **Placement** (`Scope`, `Granularity`) is checked by `NewPipelineWithScope` at compile time. By default a mismatch is a warning; the pipeline config's `strict_contract: true` turns it into a compile error, and the `POST /pipeline/_validate` gate reports it accordingly. +- **`RequiredFields`** are data assertions — enforced by the runtime guard `CheckContract(ctx, contract)` (defense in depth for chains that bypass compile-time validation) and by dry-run sample validation, never by placement (data does not exist at compile time). +- Record hosts (`for_each`, `lane_consumer`) pass their codec/lane namespace into the compile scope, so a processor requiring `Namespace: "document"` fails to compile inside a `codec: otel` chain. + ## Compatibility Notes - Sub-processors written against the original for_each (pre-convention) work unchanged: per-record `Process`, drop markers, and `ignore`-on-error semantics (error skips the rest of the sub-chain for that record) are preserved exactly. diff --git a/docs/content.en/docs/references/websocket_reverse.md b/docs/content.en/docs/references/websocket_reverse.md index 20fb6e6a..e9c3a62c 100644 --- a/docs/content.en/docs/references/websocket_reverse.md +++ b/docs/content.en/docs/references/websocket_reverse.md @@ -210,6 +210,7 @@ responses. Agent implementations should always terminate via The reverse channel is a **transport**, not an authenticator. Compose it with the framework's existing security layers: - **Control plane → agent authorization**: the agent's own API authentication applies to the proxied request. The `AccessToken` field carries the caller's bearer token end-to-end, so the agent's API layer can validate it exactly as if the call were direct. +- **Loopback token (managed agents)**: the framework's reverse client (`modules/managed/reverseclient`) executes proxied requests as loopback HTTP calls against the agent's own web port, authenticated with an `AGENT_API_ACCESS_TOKEN` from the keystore. When no token is provisioned — or the stored one has no matching record in the security KV bucket (stale from a wiped data dir or an earlier build) — the client mints one automatically: an `AccessToken` carrying every instance-ops permission plus all registered permissions, never expiring, cached in the keystore for restart reuse. The declared instance-ops permission keys are registered at reverse-channel setup, so route-level permission resolution succeeds even though those keys are constants that no route registers itself. - **Agent → control plane**: the outbound websocket rides the agent's existing authenticated channel to the control plane (e.g. Console's managed websocket with instance credentials). - **Session binding**: the handshake check (`RegisterPendingSession` → `HandleHelloPayload`) prevents an agent from claiming another peer's session slot. From 0659874dac48f96c1c64102e076acfb87d7d6b8e Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 15:47:27 +0800 Subject: [PATCH 22/65] feat: deployment disable propagates to instances; delete cascades config recall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disabling a deployment previously stopped reconciliation only — the pushed managed config stayed live and the pipeline KEPT RUNNING on every instance. Now: - reconcileOnce processes ALL deployments (disabled included); a disabled one keeps convering with enabled:false + auto_start:false in the rendered config — the instance's config watcher stops and cleans the task while keeping the file, so re-enabling is instant - deleteDeploymentHandler cascades: when no other deployment references the same pipeline, the pushed managed config row is removed too (shared references are protected) — instances see the file Deleted on next sync and the watcher removes + stops the task; previously the row leaked with a TODO --- modules/pipeline/deployment.go | 53 +++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index fa17cee4..d2028a14 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -118,7 +118,11 @@ func (module *PipeModule) reconcileOnce() { octx := orm.NewContext().DirectAccess() orm.WithModel(octx, &PipelineDeployment{}) - q := orm.NewQuery().Filter(orm.TermQuery("enabled", true)).Size(100) + // ALL deployments, disabled included: a disabled deployment keeps + // converging with enabled:false in the rendered config, so instances + // STOP the task while KEEPING the config (instant re-enable). Skipping + // disabled ones would leave the pipeline running on every instance. + q := orm.NewQuery().Size(200) res, err := orm.SearchV2(octx, q) if err != nil { return @@ -156,12 +160,25 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { // uses — instances pull and hot-reload) // v1: broadcast to all instances (strategy "all") // TODO: selector-based targeting, singleton lock, staged rollout + // disabled deployment renders the pipeline disabled: the instance's + // config watcher stops (and cleans) the running task on the + // enabled→disabled transition while keeping the file for instant + // re-enable + pipelineEnabled := cfg.Enabled + pipelineAutoStart := cfg.AutoStart + if !dep.Enabled { + // explicit false pointer (nil would mean "default enabled" at + // the instance's isPipelineEnabled check) + f := false + pipelineEnabled = &f + pipelineAutoStart = false + } configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID) cfgJSON := util.MustToJSONBytes(map[string]interface{}{ "pipeline": []interface{}{map[string]interface{}{ "name": cfg.Name, - "enabled": cfg.Enabled, - "auto_start": cfg.AutoStart, + "enabled": pipelineEnabled, + "auto_start": pipelineAutoStart, "keep_running": cfg.KeepRunning, "singleton": dep.Placement.Strategy == "singleton", "processor": cfg.Processors, @@ -373,13 +390,41 @@ func (h *PipeModule) updateDeploymentHandler(w http.ResponseWriter, req *http.Re func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) dep := PipelineDeployment{} dep.ID = id + if exists, gerr := orm.GetV2(octx, &dep); gerr != nil || !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } if err := orm.Delete(octx, &dep); err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - // TODO: remove the managed config row (cascade) + // cascade: drop the pushed managed config when no OTHER deployment + // still references the same pipeline — instances see the file Deleted + // on their next sync and the config watcher removes + stops the task + others := orm.NewQuery(). + Filter(orm.TermQuery("pipeline_id", dep.PipelineID)).Size(10) + if res, qerr := orm.SearchV2(octx, others); qerr == nil && res != nil { + if rows, _, derr := elastic.DecodeHits[PipelineDeployment](res); derr == nil && len(rows) == 0 { + configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID) + // switch the ORM context to the managed-config model — the + // queries above ran against PipelineDeployment + orm.WithModel(octx, &configs_server.ManagedConfig{}) + mcQ := orm.NewQuery(). + Filter(orm.TermQuery("name", configName)). + Filter(orm.TermQuery("instance_id", "*")).Size(5) + if mcRes, merr := orm.SearchV2(octx, mcQ); merr == nil && mcRes != nil { + if mcRows, _, mderr := elastic.DecodeHits[configs_server.ManagedConfig](mcRes); mderr == nil { + for _, row := range mcRows { + _ = orm.Delete(octx, &row) + } + log.Infof("deployment %s deleted; cascaded managed config %s removal (instances will recall the file)", dep.Name, configName) + } + } + } + } h.WriteJSON(w, util.MapStr{"deleted": true, "id": id}, http.StatusOK) } From 2fcbfc7fe47ce6742f03d3544b10dd9a6558dd98 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 15 Oct 2026 16:49:21 +0800 Subject: [PATCH 23/65] feat: deployment placement targeting with exact per-instance convergence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconciler previously ignored placement.selector/instances and always broadcast a single '*' managed config. It now resolves the scope against the registered fleet (label AND-match + explicit IDs union, broadcast when neither is set) and converges the managed-config row set EXACTLY: one row per target instance, rows for instances that left every scope are deleted so their next sync recalls the file and stops the task. - deployments sharing a pipeline merge target sets (enabled outranks disabled, singleton wins; any broadcast subsumes targeting) - row versions stay monotonic per config name so migrated rows land above what clients previously pulled (sync applies newest-per-name) - legacy '*' rows migrate in place onto per-instance rows - PUT re-validates pipeline exists + compiles (edit can rebind it); strategy whitelist all|singleton; empty selector keys dropped - status computes desired_count from the resolved scope, reports in-scope instances as PENDING before first runtime data, and marks out-of-scope runners stale (recall in flight/failed) - delete cascade removes every row of the config name (was: '*' only) Covered by deployment_targeting_test.go: resolve modes, label-scope exact convergence without version churn, scope-change recall with version monotonicity, broadcast→targeted migration. --- docs/content.en/docs/references/pipeline.md | 37 +- modules/pipeline/deployment.go | 328 ++++++++++++++---- modules/pipeline/deployment_targeting_test.go | 281 +++++++++++++++ 3 files changed, 574 insertions(+), 72 deletions(-) create mode 100644 modules/pipeline/deployment_targeting_test.go diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md index 84336da2..41f291d8 100644 --- a/docs/content.en/docs/references/pipeline.md +++ b/docs/content.en/docs/references/pipeline.md @@ -540,16 +540,43 @@ Every update auto-snapshots the previous config as a `PipelineVersion` — there `PipelineDeployment` is the control-plane desired state: which pipeline runs on which instances. A reconciler (15s interval) converges desired → reported: -1. Loads the pipeline config, re-validates it compiles, and renders it as a **managed config** row targeted at the selected instances (strategy `all` broadcasts; `singleton` marks the config for instance-side lock resolution). +1. Loads the pipeline config, re-validates it compiles, and renders it as **managed config** rows — one per target instance (`singleton` marks the config for instance-side lock resolution). 2. Instances pull the managed config on their next `_sync`, write it to their config directory, and hot-reload the pipeline. 3. Pushes are **converged**: when the rendered content is unchanged, the reconciler skips the version bump — instances don't rewrite files every cycle. +### Placement scope (targeting) + +`placement` decides **where** the pipeline runs — resolved against the registered instance fleet on every cycle: + +```json +{ + "placement": { + "strategy": "all", + "selector": { "env": "prod" }, + "instances": ["gw-2"] + } +} +``` + +| Field | Semantics | +|-------|-----------| +| `selector` | Label match (AND across entries) — an instance is in scope when its labels carry every key=value. | +| `instances` | Explicit instance IDs (union with the selector match; unknown IDs are ignored until they register). | +| *(neither set)* | Broadcast: a single `*` row every instance pulls. | + +Convergence is **exact** — the row set always equals the desired set: + +- Instances that fall out of every scope get their row **deleted**; their next sync reports the file `Deleted`, the config watcher removes it and stops the task. No duplicate deployments linger after a scope change A→B. +- Label membership is re-evaluated every cycle: an instance that gains/loses a matching label is added/removed automatically. +- Several deployments may share one pipeline; their target sets merge (an enabled deployment outranks a disabled one on the same instance, `singleton` wins over `all`). Any broadcast deployment subsumes targeting for that pipeline. +- Row versions are monotonic **per config name** — the instance sync applies updates strictly by version (`newest wins`), so a migrated row always lands above what clients previously pulled. + | Method & Path | Description | |---------------|-------------| -| `POST /pipeline/deployments/` | Create (requires pipeline exists + compiles). | +| `POST /pipeline/deployments/` | Create (requires pipeline exists + compiles; strategy limited to `all`\|`singleton`). | | `GET /pipeline/deployments/` | List. | -| `GET/PUT/DELETE /pipeline/deployments/:id` | Read / update (enable, placement) / delete. | -| `GET /pipeline/deployments/:id/status` | Desired-vs-reported status. | +| `GET/PUT/DELETE /pipeline/deployments/:id` | Read / update (name, pipeline rebind, placement, enable — re-validated) / delete (cascades managed-config recall when no other deployment references the pipeline). | +| `GET /pipeline/deployments/:id/status` | Desired-vs-reported status (`desired_count` is the resolved scope size). | ### Runtime report and failure visibility @@ -570,7 +597,7 @@ Every managed instance reports its live pipeline states with each config sync (k } ``` -A pipeline whose config fails to compile **on the instance** (e.g. the instance's processor catalog differs from the control plane's — a plugin missing from that binary) is reported as `FAILED` with the compile error as `reason`, instead of silently disappearing from the report. Instances without an ORM backend (lightweight agents) skip the reconciler side but still report runtime state. +A pipeline whose config fails to compile **on the instance** (e.g. the instance's processor catalog differs from the control plane's — a plugin missing from that binary) is reported as `FAILED` with the compile error as `reason`, instead of silently disappearing from the report. Reports carry `"stale": true` when the instance still runs the pipeline but the placement scope no longer includes it (recall in flight, or a recall that failed). Instances without an ORM backend (lightweight agents) skip the reconciler side but still report runtime state. ## Runtime APIs diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index d2028a14..7187a6e5 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -85,6 +85,9 @@ type InstanceReport struct { // Reason carries the instance-reported failure explanation (compile // error, missing processor on that instance) Reason string `json:"reason,omitempty"` + // Stale marks an instance that still runs the pipeline although the + // deployment scope no longer includes it (recall in flight/failed) + Stale bool `json:"stale,omitempty"` } // ─── reconciler ────────────────────────────────────────────────────────── @@ -116,37 +119,98 @@ func (module *PipeModule) reconcileOnce() { return } octx := orm.NewContext().DirectAccess() - orm.WithModel(octx, &PipelineDeployment{}) + + // registered instances — the universe that selector/explicit scopes + // resolve against (labels refresh on every sync heartbeat) + orm.WithModel(octx, &model.Instance{}) + instRes, ierr := orm.SearchV2(octx, orm.NewQuery().Size(500)) + if ierr != nil { + return + } + instances, _, _ := elastic.DecodeHits[model.Instance](instRes) // ALL deployments, disabled included: a disabled deployment keeps // converging with enabled:false in the rendered config, so instances // STOP the task while KEEPING the config (instant re-enable). Skipping // disabled ones would leave the pipeline running on every instance. - q := orm.NewQuery().Size(200) - res, err := orm.SearchV2(octx, q) + orm.WithModel(octx, &PipelineDeployment{}) + res, err := orm.SearchV2(octx, orm.NewQuery().Size(200)) if err != nil { return } deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res) - for _, dep := range deployments { - if rerr := module.reconcileDeployment(&dep); rerr != nil { - log.Warnf("reconciler: deployment %s: %v", dep.Name, rerr) + // group by pipeline: managed-config rows are named per PIPELINE + // (pipeline-deploy-.yml), so every deployment of the same + // pipeline must converge together — one adds instances, another's + // scope-shrink must not delete rows the first one still needs + byPipeline := map[string][]*PipelineDeployment{} + for i := range deployments { + byPipeline[deployments[i].PipelineID] = append(byPipeline[deployments[i].PipelineID], &deployments[i]) + } + for pipelineID, deps := range byPipeline { + if rerr := module.reconcilePipeline(octx, pipelineID, deps, instances); rerr != nil { + log.Warnf("reconciler: %v", rerr) } } } -// reconcileDeployment converges ONE deployment toward its desired state. -// v1: push the pipeline config as a managed config row targeting the -// selected instances. Instance-side lock contention resolves singleton. -func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { - // load the pipeline config - octx := orm.NewContext().DirectAccess() +// renderSpec is how ONE target instance should run the pipeline — the +// merge result of every deployment of that pipeline targeting it. +type renderSpec struct { + enabled bool + singleton bool +} + +// resolveTargets maps a deployment's placement onto registered instances: +// label selector matches AND explicit instance IDs union. Neither set = +// broadcast (every instance, the v1 semantics). +func resolveTargets(dep *PipelineDeployment, instances []model.Instance) (map[string]bool, bool) { + if len(dep.Placement.Selector) == 0 && len(dep.Placement.Instances) == 0 { + return nil, true + } + known := map[string]bool{} + targets := map[string]bool{} + for i := range instances { + known[instances[i].ID] = true + // an empty selector must not match everything — when only + // explicit IDs are set, label matching is skipped entirely + if len(dep.Placement.Selector) > 0 && instanceMatchesLabels(&instances[i], dep.Placement.Selector) { + targets[instances[i].ID] = true + } + } + // explicit IDs resolve against REGISTERED instances only — a row for + // an unknown ID is invisible garbage no one pulls; when the instance + // later registers, the next cycle picks it up (self-healing scope) + for _, id := range dep.Placement.Instances { + if known[id] { + targets[id] = true + } + } + return targets, false +} + +func instanceMatchesLabels(inst *model.Instance, selector map[string]string) bool { + for k, v := range selector { + if inst.Labels[k] != v { + return false + } + } + return true +} + +// reconcilePipeline converges ONE pipeline's deployments into the EXACT +// set of managed-config rows: one row per target instance (or a single +// broadcast "*" row). Rows for instances that fell out of every scope are +// deleted — the instance's next sync reports the file Deleted and the +// config watcher removes it and stops the task. Nothing more, nothing +// less: converging means the row set equals the desired set. +func (module *PipeModule) reconcilePipeline(octx *orm.Context, pipelineID string, deps []*PipelineDeployment, instances []model.Instance) error { cfg := pipeline.PipelineConfigV2{} - cfg.ID = dep.PipelineID + cfg.ID = pipelineID exists, err := orm.GetV2(octx, &cfg) if err != nil || !exists { - return fmt.Errorf("pipeline %s not found", dep.PipelineID) + return fmt.Errorf("pipeline %s not found (deployments: %s)", pipelineID, deploymentNames(deps)) } // defensive gate: never distribute a config that no longer compiles @@ -156,70 +220,140 @@ func (module *PipeModule) reconcileDeployment(dep *PipelineDeployment) error { cfg.Name, errs[0].Message) } - // translate to a managed-config row (the same channel stream publish - // uses — instances pull and hot-reload) - // v1: broadcast to all instances (strategy "all") - // TODO: selector-based targeting, singleton lock, staged rollout - // disabled deployment renders the pipeline disabled: the instance's - // config watcher stops (and cleans) the running task on the - // enabled→disabled transition while keeping the file for instant - // re-enable + // merge the target set + render flags across all deployments of this + // pipeline: an enabled deployment outranks a disabled one targeting + // the same instance; singleton wins over all. Any broadcast + // deployment subsumes targeting (a "*" row already reaches everyone). + broadcast := false + desired := map[string]renderSpec{} + for _, dep := range deps { + targets, bcast := resolveTargets(dep, instances) + if bcast { + broadcast = true + continue + } + spec := renderSpec{enabled: dep.Enabled, singleton: dep.Placement.Strategy == "singleton"} + for id := range targets { + prev := desired[id] + prev.enabled = prev.enabled || spec.enabled + prev.singleton = prev.singleton || spec.singleton + desired[id] = prev + } + } + if broadcast { + spec := renderSpec{} + for _, dep := range deps { + spec.enabled = spec.enabled || dep.Enabled + spec.singleton = spec.singleton || dep.Placement.Strategy == "singleton" + } + desired = map[string]renderSpec{configs_server.AllInstancesID: spec} + } + + configName := fmt.Sprintf("pipeline-deploy-%s.yml", pipelineID) + + // existing rows of this config name (deployment-owned convention) + orm.WithModel(octx, &configs_server.ManagedConfig{}) + rowsRes, rerr := orm.SearchV2(octx, orm.NewQuery(). + Filter(orm.TermQuery("name", configName)).Size(500)) + if rerr != nil { + return fmt.Errorf("query managed config %s: %w", configName, rerr) + } + existingRows, _, _ := elastic.DecodeHits[configs_server.ManagedConfig](rowsRes) + existing := map[string]configs_server.ManagedConfig{} + for _, row := range existingRows { + existing[row.InstanceID] = row + } + + // instance syncs apply updates strictly by version ("newest version + // per name" wins on the server too) — every write must land ABOVE the + // name's current max version or clients keep stale content forever + // (e.g. migrating the legacy broadcast row onto per-instance rows) + maxVersion := int64(0) + for _, row := range existingRows { + if row.Version > maxVersion { + maxVersion = row.Version + } + } + + for instID, spec := range desired { + content := string(renderDeployedPipeline(&cfg, spec)) + row, ok := existing[instID] + // converged: the pushed content already matches — skip the + // version bump and re-push so instances don't rewrite files + if ok && row.Content == content { + continue + } + row.Name = configName + row.InstanceID = instID + row.Content = content + row.Version = maxVersion + 1 + row.Readonly = true // deployment-owned + if row.ID == "" { + // broadcast keeps the legacy v1 ID so an existing row + // updates in place instead of duplicating + if instID == configs_server.AllInstancesID { + row.ID = fmt.Sprintf("deploy-%s", pipelineID) + } else { + row.ID = fmt.Sprintf("deploy-%s-%s", pipelineID, instID) + } + } + if serr := orm.Save(octx, &row); serr != nil { + return fmt.Errorf("push managed config %s→%s: %w", configName, instID, serr) + } + maxVersion = row.Version + log.Debugf("reconciler: %s → instance %s v%d (%d bytes, enabled=%v singleton=%v)", + configName, instID, row.Version, len(content), spec.enabled, spec.singleton) + } + + // garbage-collect rows that no deployment targets anymore — the + // instance recalls the file (and stops the task) on its next sync + for instID, row := range existing { + if _, ok := desired[instID]; !ok { + if derr := orm.Delete(octx, &row); derr != nil { + return fmt.Errorf("recall managed config %s→%s: %w", configName, instID, derr) + } + log.Infof("reconciler: %s dropped instance %s from scope — recalling config", configName, instID) + } + } + return nil +} + +func deploymentNames(deps []*PipelineDeployment) string { + names := "" + for i, dep := range deps { + if i > 0 { + names += ", " + } + names += dep.Name + } + return names +} + +// renderDeployedPipeline serializes the pipeline config handed to +// instances. A disabled spec renders enabled:false + auto_start:false — +// the instance's config watcher stops (and cleans) the running task on +// the enabled→disabled transition while keeping the file for instant +// re-enable. +func renderDeployedPipeline(cfg *pipeline.PipelineConfigV2, spec renderSpec) []byte { pipelineEnabled := cfg.Enabled pipelineAutoStart := cfg.AutoStart - if !dep.Enabled { + if !spec.enabled { // explicit false pointer (nil would mean "default enabled" at // the instance's isPipelineEnabled check) f := false pipelineEnabled = &f pipelineAutoStart = false } - configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID) - cfgJSON := util.MustToJSONBytes(map[string]interface{}{ + return util.MustToJSONBytes(map[string]interface{}{ "pipeline": []interface{}{map[string]interface{}{ "name": cfg.Name, "enabled": pipelineEnabled, "auto_start": pipelineAutoStart, "keep_running": cfg.KeepRunning, - "singleton": dep.Placement.Strategy == "singleton", + "singleton": spec.singleton, "processor": cfg.Processors, }}, }) - - // upsert the ManagedConfig row (instances pull on next _sync) - orm.WithModel(octx, &configs_server.ManagedConfig{}) - existing := configs_server.ManagedConfig{} - foundQ := orm.NewQuery(). - Filter(orm.TermQuery("name", configName)). - Filter(orm.TermQuery("instance_id", "*")).Size(1) - foundRes, ferr := orm.SearchV2(octx, foundQ) - if ferr == nil && foundRes != nil { - if rows, _, derr := elastic.DecodeHits[configs_server.ManagedConfig](foundRes); derr == nil && len(rows) > 0 { - existing = rows[0] - } - } - - // converged: the pushed content already matches — skip the version - // bump and re-push so instances don't rewrite files every cycle - if existing.ID != "" && existing.Content == string(cfgJSON) { - return nil - } - - existing.Name = configName - existing.InstanceID = "*" // v1: broadcast; selector targeting next - existing.Content = string(cfgJSON) - existing.Version++ - existing.Readonly = true // deployment-owned - - if existing.ID == "" { - existing.ID = fmt.Sprintf("deploy-%s", dep.PipelineID) - } - if serr := orm.Save(octx, &existing); serr != nil { - return fmt.Errorf("push managed config: %w", serr) - } - - log.Debugf("reconciler: deployment %s → config %s v%d (%d bytes, strategy=%s)", - dep.Name, configName, existing.Version, len(cfgJSON), dep.Placement.Strategy) - return nil } // getDeploymentStatus builds the REPORTED state from instance runtime @@ -249,8 +383,22 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat return status } + targets, broadcast := resolveTargets(dep, instances) + for _, inst := range instances { + inScope := broadcast || targets[inst.ID] if inst.RuntimePipelines == "" { + // in-scope instances that have not reported yet are part of + // the desired count — surface them as pending instead of + // silently counting toward a number the list never explains + if inScope && dep.Enabled { + status.Instances = append(status.Instances, InstanceReport{ + InstanceID: inst.ID, + State: "PENDING", + Version: dep.Version, + LastSeen: time.Now().UTC(), + }) + } continue } var pipelines map[string]struct { @@ -268,15 +416,25 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat Version: dep.Version, LastSeen: time.Now().UTC(), Reason: st.Reason, + // stale = the instance still runs the pipeline but the + // scope no longer includes it (config recall in flight, + // or a recall that failed — visible instead of silent) + Stale: !inScope, }) } } - status.DesiredCount = 1 // v1: all strategy = every instance; singleton = 1 + status.DesiredCount = len(instances) + if !broadcast { + status.DesiredCount = len(targets) + } if dep.Placement.Strategy == "singleton" { status.DesiredCount = 1 } status.ReadyCount = 0 for _, ir := range status.Instances { + if ir.Stale { + continue + } if ir.State == "STARTED" { status.ReadyCount++ } @@ -300,6 +458,15 @@ func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Re if dep.Placement.Strategy == "" { dep.Placement.Strategy = "all" // v1 default (D-12: no sharded) } + if dep.Placement.Strategy != "all" && dep.Placement.Strategy != "singleton" { + h.WriteError(w, fmt.Sprintf("unknown strategy %q (all|singleton)", dep.Placement.Strategy), http.StatusBadRequest) + return + } + for k, v := range dep.Placement.Selector { + if k == "" || v == "" { + delete(dep.Placement.Selector, k) + } + } // the referenced pipeline must exist AND compile — a deployment is a // distribution promise; accepting one that ships a broken config to @@ -375,6 +542,29 @@ func (h *PipeModule) updateDeploymentHandler(w http.ResponseWriter, req *http.Re return } + // re-validation on update: the edit can rebind pipeline_id — the same + // distribution promise as create (must exist AND compile) holds + cfg := pipeline.PipelineConfigV2{} + cfg.ID = newDep.PipelineID + pexists, perr := orm.GetV2(octx, &cfg) + if perr != nil || !pexists { + h.WriteError(w, fmt.Sprintf("pipeline %s not found", newDep.PipelineID), http.StatusBadRequest) + return + } + if !h.validateAndReject(w, &cfg) { + return + } + + // normalize placement: default strategy, drop empty selector keys + if newDep.Placement.Strategy == "" { + newDep.Placement.Strategy = "all" + } + for k, v := range newDep.Placement.Selector { + if k == "" || v == "" { + delete(newDep.Placement.Selector, k) + } + } + // preserve identity + bump version newDep.ID = id newDep.Version = old.Version + 1 @@ -403,7 +593,8 @@ func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Re } // cascade: drop the pushed managed config when no OTHER deployment // still references the same pipeline — instances see the file Deleted - // on their next sync and the config watcher removes + stops the task + // on their next sync and the config watcher removes + stops the task. + // Rows are per-instance (or the "*" broadcast) — remove them all. others := orm.NewQuery(). Filter(orm.TermQuery("pipeline_id", dep.PipelineID)).Size(10) if res, qerr := orm.SearchV2(octx, others); qerr == nil && res != nil { @@ -413,14 +604,17 @@ func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Re // queries above ran against PipelineDeployment orm.WithModel(octx, &configs_server.ManagedConfig{}) mcQ := orm.NewQuery(). - Filter(orm.TermQuery("name", configName)). - Filter(orm.TermQuery("instance_id", "*")).Size(5) + Filter(orm.TermQuery("name", configName)).Size(500) + removed := 0 if mcRes, merr := orm.SearchV2(octx, mcQ); merr == nil && mcRes != nil { if mcRows, _, mderr := elastic.DecodeHits[configs_server.ManagedConfig](mcRes); mderr == nil { for _, row := range mcRows { _ = orm.Delete(octx, &row) + removed++ + } + if removed > 0 { + log.Infof("deployment %s deleted; cascaded %d managed config rows of %s (instances will recall the file)", dep.Name, removed, configName) } - log.Infof("deployment %s deleted; cascaded managed config %s removal (instances will recall the file)", dep.Name, configName) } } } diff --git a/modules/pipeline/deployment_targeting_test.go b/modules/pipeline/deployment_targeting_test.go new file mode 100644 index 00000000..f5c4c4fe --- /dev/null +++ b/modules/pipeline/deployment_targeting_test.go @@ -0,0 +1,281 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/modules/configs/server" + sqlite "infini.sh/framework/modules/sqlite" +) + +// deployTestEnv wires an in-memory control plane: sqlite ORM with the +// three schemas the reconciler touches (deployments, instances, managed +// configs) plus one saved, compilable pipeline config. The ORM handler +// is process-global, so every env gets unique instance/label/pipeline +// IDs and tests stay isolated. +type deployTestEnv struct { + t *testing.T + pid string + instA string + instB string + instC string + prod string // label value shared by instA+instC + stage string // label value of instB + edge string // extra label of instC +} + +var ( + deployEnvSeq int + deployORMOnce sync.Once +) + +func newDeployTestEnv(t *testing.T) *deployTestEnv { + deployEnvSeq++ + sfx := fmt.Sprintf("%d", deployEnvSeq) + // ONE handler for the whole test binary: the ORM registry is a + // process-global, and a per-test DB would be closed under the next + // test's feet. Each env isolates via unique IDs instead. + deployORMOnce.Do(func() { + handler := &sqlite.SQLiteORM{Config: sqlite.SQLiteConfig{Enabled: true, + DBPath: filepath.Join(os.TempDir(), fmt.Sprintf("pipeline-deploy-test-%d.db", time.Now().UnixNano()))}} + if err := handler.Open(); err != nil { + t.Fatal(err) + } + orm.Register("deploy-test", handler) + orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs-test") + orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments-test") + orm.MustRegisterSchemaWithIndexName(model.Instance{}, "instance-test") + orm.MustRegisterSchemaWithIndexName(server.ManagedConfig{}, "managed-configs-test") + if err := orm.InitSchema(); err != nil { + t.Fatal(err) + } + }) + + env := &deployTestEnv{ + t: t, pid: "pipe-" + sfx, + instA: "gw-a-" + sfx, instB: "gw-b-" + sfx, instC: "gw-c-" + sfx, + prod: "prod-" + sfx, stage: "staging-" + sfx, edge: "edge-" + sfx, + } + env.saveInstance(env.instA, map[string]string{"env": env.prod}) + env.saveInstance(env.instB, map[string]string{"env": env.stage}) + env.saveInstance(env.instC, map[string]string{"env": env.prod, "tier": env.edge}) + + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &pipeline.PipelineConfigV2{}) + cfg := pipeline.PipelineConfigV2{} + cfg.ID = env.pid + cfg.Name = "deploy-test-pipeline-" + sfx + cfg.Processors = []map[string]interface{}{ + {"validate_test_echo": map[string]interface{}{"message": "hi"}}, + } + if err := orm.Save(octx, &cfg); err != nil { + t.Fatal(err) + } + return env +} + +func (e *deployTestEnv) saveInstance(id string, labels map[string]string) { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &model.Instance{}) + inst := model.Instance{} + inst.ID = id + inst.Labels = labels + if err := orm.Save(octx, &inst); err != nil { + e.t.Fatal(err) + } +} + +func (e *deployTestEnv) saveDeployment(dep *PipelineDeployment) { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + if err := orm.Save(octx, dep); err != nil { + e.t.Fatal(err) + } +} + +// rows returns the deployment-owned managed-config rows keyed by +// instance_id. +func (e *deployTestEnv) rows() map[string]server.ManagedConfig { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &server.ManagedConfig{}) + res, err := orm.SearchV2(octx, orm.NewQuery().Filter(orm.TermQuery("name", fmt.Sprintf("pipeline-deploy-%s.yml", e.pid))).Size(100)) + if err != nil { + e.t.Fatal(err) + } + rows, _, _ := elastic.DecodeHits[server.ManagedConfig](res) + out := map[string]server.ManagedConfig{} + for _, r := range rows { + out[r.InstanceID] = r + } + return out +} + +func (e *deployTestEnv) reconcile() { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &model.Instance{}) + instRes, err := orm.SearchV2(octx, orm.NewQuery().Size(100)) + if err != nil { + e.t.Fatal(err) + } + instances, _, _ := elastic.DecodeHits[model.Instance](instRes) + + orm.WithModel(octx, &PipelineDeployment{}) + res, err := orm.SearchV2(octx, orm.NewQuery().Size(100)) + if err != nil { + e.t.Fatal(err) + } + deployments, _, _ := elastic.DecodeHits[PipelineDeployment](res) + for i := range deployments { + if err := e.module().reconcilePipeline(octx, e.pid, []*PipelineDeployment{&deployments[i]}, instances); err != nil { + e.t.Fatalf("reconcile: %v", err) + } + } +} + +func (e *deployTestEnv) module() *PipeModule { return &PipeModule{} } + +func TestResolveTargetsModes(t *testing.T) { + instances := []model.Instance{ + {ORMObjectBase: orm.ORMObjectBase{ID: "a"}, Labels: map[string]string{"env": "prod"}}, + {ORMObjectBase: orm.ORMObjectBase{ID: "b"}, Labels: map[string]string{"env": "staging"}}, + } + // no selector + no instances → broadcast + targets, broadcast := resolveTargets(&PipelineDeployment{}, instances) + if !broadcast || targets != nil { + t.Fatalf("expected broadcast, got %v %v", targets, broadcast) + } + // label match + targets, broadcast = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod"}}}, instances) + if broadcast || len(targets) != 1 || !targets["a"] { + t.Fatalf("expected {a}, got %v broadcast=%v", targets, broadcast) + } + // multi-label AND + targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod", "tier": "edge"}}}, instances) + if len(targets) != 0 { + t.Fatalf("expected no match for AND miss, got %v", targets) + } + // explicit IDs, unknown dropped + targets, broadcast = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Instances: []string{"b", "ghost"}}}, instances) + if broadcast || len(targets) != 1 || !targets["b"] { + t.Fatalf("expected {b}, got %v broadcast=%v", targets, broadcast) + } + // explicit + labels union + targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod"}, Instances: []string{"b"}}}, instances) + if len(targets) != 2 || !targets["a"] || !targets["b"] { + t.Fatalf("expected {a,b}, got %v", targets) + } +} + +func TestDeploymentLabelScopeConvergesExactly(t *testing.T) { + env := newDeployTestEnv(t) + env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "prod-only", PipelineID: env.pid, Enabled: true, + Placement: PlacementSpec{Strategy: "all", Selector: map[string]string{"env": env.prod}}}) + + env.reconcile() + rows := env.rows() + // exactly the two env=prod instances, no broadcast row, no staging + if len(rows) != 2 || rows[env.instA].ID == "" || rows[env.instC].ID == "" || rows[env.instB].ID != "" || rows["*"].ID != "" { + t.Fatalf("expected exactly {gw-a,gw-c}, got %v", rowIDs(rows)) + } + v1 := rows[env.instA].Version + + // converge again → no version churn, still exactly two rows + env.reconcile() + rows = env.rows() + if len(rows) != 2 || rows[env.instA].Version != v1 { + t.Fatalf("expected stable convergence, got %s v%d", rowIDs(rows), rows["gw-a"].Version) + } +} + +func TestDeploymentScopeChangeRecallsOldInstances(t *testing.T) { + env := newDeployTestEnv(t) + env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "mover", PipelineID: env.pid, Enabled: true, + Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA}}}) + + env.reconcile() + if rows := env.rows(); len(rows) != 1 || rows[env.instA].ID == "" { + t.Fatalf("expected {gw-a}, got %v", rowIDs(rows)) + } + oldVersion := env.rows()[env.instA].Version + + // scope A → B: switch to gw-b; gw-a must be recalled (row deleted), + // gw-b's row must land ABOVE gw-a's old version or gw-a's client + // would refuse the "update" it never sees... (version monotonicity + // per name is what keeps instances from sticking to stale content) + dep := PipelineDeployment{} + { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + dep.ID = "d1" + if found, err := orm.GetV2(octx, &dep); err != nil || !found { + t.Fatal("deployment vanished") + } + } + dep.Placement.Instances = []string{env.instB} + env.saveDeployment(&dep) + + env.reconcile() + rows := env.rows() + if len(rows) != 1 || rows[env.instB].ID == "" { + t.Fatalf("expected scope change to leave exactly {gw-b}, got %v", rowIDs(rows)) + } + if rows[env.instB].Version <= oldVersion { + t.Fatalf("new row version %d must exceed old %d (per-name monotonicity)", rows["gw-b"].Version, oldVersion) + } +} + +func TestDeploymentBroadcastMigratesToTargeted(t *testing.T) { + env := newDeployTestEnv(t) + env.saveDeployment(&PipelineDeployment{ORMObjectBase: orm.ORMObjectBase{ID: "d1"}, Name: "legacy", PipelineID: env.pid, Enabled: true, + Placement: PlacementSpec{Strategy: "all"}}) + + env.reconcile() + rows := env.rows() + if len(rows) != 1 || rows["*"].ID == "" { + t.Fatalf("expected broadcast row, got %v", rowIDs(rows)) + } + broadcastVersion := rows["*"].Version + + // edit the deployment to target one instance — the legacy "*" row + // must be replaced by the per-instance row, seeded above its version + dep := PipelineDeployment{} + { + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + dep.ID = "d1" + if found, err := orm.GetV2(octx, &dep); err != nil || !found { + t.Fatal("deployment vanished") + } + } + dep.Placement.Selector = map[string]string{"tier": env.edge} + env.saveDeployment(&dep) + + env.reconcile() + rows = env.rows() + if len(rows) != 1 || rows[env.instC].ID == "" || rows["*"].ID != "" { + t.Fatalf("expected {gw-c} only, got %v", rowIDs(rows)) + } + if rows[env.instC].Version <= broadcastVersion { + t.Fatalf("migrated row version %d must exceed broadcast %d", rows["gw-c"].Version, broadcastVersion) + } +} + +func rowIDs(rows map[string]server.ManagedConfig) []string { + out := []string{} + for id := range rows { + out = append(out, id) + } + return out +} From 3b64c1976149d6302731a9050d8696c66246035c Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 20 Oct 2026 18:10:39 +0800 Subject: [PATCH 24/65] feat(pipeline): add case/branches multi-way dispatch processor Chained if/then/else blocks get unwieldy once one input routes to three or more destinations. The new `case` step is an ordered multi-way dispatch where the FIRST matching branch runs (exactly one arm), with an optional `default` chain: - case: field: log_level branches: - values: [error, fatal] # membership, like the `in` condition then: [...] - when: # any conditions.Config operator prefix: {log_level: wa} then: [...] default: [...] # optional; no-match without it is a no-op Arm chains compile through newPipelineWithScope so they inherit the enclosing scope (record contracts, batch_only placement rules), exactly like if/then/else arms. Value resolution mirrors if/then/else too: the current record takes precedence, context fallback at pipeline level. Which arm ran is recorded in the flow trace as case:, case:default, or case:no-match. Compile errors carry the offending branch index. --- core/pipeline/case_processor.go | 239 ++++++++++++++++ core/pipeline/case_processor_test.go | 302 ++++++++++++++++++++ core/pipeline/processor.go | 18 ++ docs/content.en/docs/references/pipeline.md | 61 +++- 4 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 core/pipeline/case_processor.go create mode 100644 core/pipeline/case_processor_test.go diff --git a/core/pipeline/case_processor.go b/core/pipeline/case_processor.go new file mode 100644 index 00000000..97fa7bc0 --- /dev/null +++ b/core/pipeline/case_processor.go @@ -0,0 +1,239 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pipeline + +import ( + "errors" + "fmt" + "reflect" + "strings" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/conditions" + "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +// caseBranchConfig is one arm of a case block as written in YAML: an +// optional enumerated value set (matched against the case block's field), +// an optional full condition (`when`, any operator the if/then/else block +// accepts), and the chain to run on match. At least one of values/when is +// required; specifying both means AND. +type caseBranchConfig struct { + Values []interface{} `config:"values"` + When conditions.Config `config:"when"` + Then *config.Config `config:"then"` +} + +type caseConfig struct { + Field string `config:"field"` + Branches []caseBranchConfig `config:"branches"` + Default *config.Config `config:"default"` +} + +// caseBranch is the compiled form of one arm. +type caseBranch struct { + values []interface{} + hasValues bool + cond conditions.Condition // nil = values-only arm + then *Processors +} + +// CaseProcessor is the multi-way dispatch the if/then/else ladder becomes +// unwieldy for: ordered branches, FIRST match wins (exactly one arm runs), +// an optional default chain when nothing matches. +// +// - case: +// field: level +// branches: +// - values: [error, fatal] +// then: [ ... ] +// - when: +// prefix: {level: wa} +// then: [ ... ] +// default: [ ... ] +type CaseProcessor struct { + field string // "" = condition-only arms + branches []caseBranch + def *Processors +} + +// NewCaseProcessor compiles a case step config at the default batch scope. +func NewCaseProcessor(cfg *config.Config) (*CaseProcessor, error) { + return NewCaseProcessorWithScope(cfg, ScopeInfo{Kind: ScopeBatch}) +} + +// NewCaseProcessorWithScope compiles the arm chains under the ENCLOSING +// scope — the same inheritance if/then/else applies: inside a per-record +// sub-chain (for_each, lane_consumer) record-scope contracts keep +// applying and batch_only processors stay rejected in nested chains. +func NewCaseProcessorWithScope(cfg *config.Config, scope ScopeInfo) (*CaseProcessor, error) { + sub, err := cfg.Child("case", -1) + if err != nil { + return nil, err + } + + var c caseConfig + if err := sub.Unpack(&c); err != nil { + return nil, err + } + if len(c.Branches) == 0 { + return nil, errors.New("case requires at least one branch") + } + + newChain := func(cc *config.Config) (*Processors, error) { + if cc == nil { + return nil, nil + } + if !cc.IsArray() { + return newPipelineWithScope([]*config.Config{cc}, scope, false) + } + var pc []*config.Config + if err := cc.Unpack(&pc); err != nil { + return nil, err + } + return newPipelineWithScope(pc, scope, false) + } + + p := &CaseProcessor{field: c.Field} + for i, b := range c.Branches { + hasValues := len(b.Values) > 0 + hasWhen := !reflect.DeepEqual(b.When, conditions.Config{}) + if !hasValues && !hasWhen { + return nil, fmt.Errorf("case branch %d requires `values` or `when`", i+1) + } + if hasValues && c.Field == "" { + return nil, fmt.Errorf("case branch %d uses `values` but the case block declares no `field`", i+1) + } + arm := caseBranch{values: b.Values, hasValues: hasValues} + if hasWhen { + cond, err := conditions.NewCondition(&b.When) + if err != nil { + return nil, fmt.Errorf("case branch %d: %v", i+1, err) + } + arm.cond = cond + } + then, err := newChain(b.Then) + if err != nil { + return nil, fmt.Errorf("case branch %d: %v", i+1, err) + } + if then == nil { + return nil, fmt.Errorf("case branch %d requires a `then` chain", i+1) + } + arm.then = then + p.branches = append(p.branches, arm) + } + + def, err := newChain(c.Default) + if err != nil { + return nil, fmt.Errorf("case default: %v", err) + } + p.def = def + return p, nil +} + +// Process routes to the first matching arm. Value resolution matches +// if/then/else: inside a per-record sub-chain arms see the current +// record's fields; pipeline-level usage reads the context. +func (p *CaseProcessor) Process(ctx *Context) error { + if !ctx.ShouldContinue() { + if global.Env().IsDebug { + log.Debugf("filter [%v] not continued", p.Name()) + } + ctx.AddFlowProcess(p.Name() + "-skipped") + return nil + } + + values := conditions.ValuesMap(ctx) + if rec, ok := CurrentRecord(ctx); ok { + values = rec + } + + for i := range p.branches { + arm := &p.branches[i] + if arm.hasValues && !p.matchValues(values, arm.values) { + continue + } + if arm.cond != nil && !arm.cond.Check(values) { + continue + } + if global.Env().IsDebug { + log.Tracef("case -> branch %d", i+1) + } + ctx.AddFlowProcess(fmt.Sprintf("%s:%d", p.Name(), i+1)) + return arm.then.Process(ctx) + } + + if p.def != nil { + if global.Env().IsDebug { + log.Trace("case -> default") + } + ctx.AddFlowProcess(p.Name() + ":default") + return p.def.Process(ctx) + } + ctx.AddFlowProcess(p.Name() + ":no-match") + return nil +} + +// matchValues mirrors the `in` condition's membership semantics (typed +// comparison via util.ContainsAnyInAnyIntArray), so `field`+`values` arms +// and `when: {in: ...}` behave identically. +func (p *CaseProcessor) matchValues(values conditions.ValuesMap, want []interface{}) bool { + v, err := values.GetValue(p.field) + if err != nil { + return false + } + return util.ContainsAnyInAnyIntArray(v, want) +} + +func (p *CaseProcessor) Name() string { + return "case" +} + +func (p *CaseProcessor) String() string { + var sb strings.Builder + sb.WriteString("case ") + for i, arm := range p.branches { + if i > 0 { + sb.WriteString("; ") + } + if arm.hasValues { + sb.WriteString(fmt.Sprintf("%v in %v", p.field, arm.values)) + } + if arm.cond != nil { + if arm.hasValues { + sb.WriteString(" and ") + } + sb.WriteString(arm.cond.String()) + } + sb.WriteString(" then ") + sb.WriteString(arm.then.Name()) + } + if p.def != nil { + sb.WriteString("; default ") + sb.WriteString(p.def.Name()) + } + return sb.String() +} diff --git a/core/pipeline/case_processor_test.go b/core/pipeline/case_processor_test.go new file mode 100644 index 00000000..fbdb05b7 --- /dev/null +++ b/core/pipeline/case_processor_test.go @@ -0,0 +1,302 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +// buildCase compiles `body` as the content of a case step. +func buildCase(t *testing.T, body util.MapStr) *CaseProcessor { + t.Helper() + c, err := config.NewConfigFrom(util.MapStr{"case": body}) + if err != nil { + t.Fatal(err) + } + p, err := NewCaseProcessor(c) + if err != nil { + t.Fatalf("NewCaseProcessor: %v", err) + } + return p +} + +func mark(mark string) util.MapStr { + return util.MapStr{"test_branch_marker": util.MapStr{"mark": mark}} +} + +func levelSwitch(withDefault bool) util.MapStr { + body := util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"error", "fatal"}, "then": mark("sev1")}, + util.MapStr{"values": []interface{}{"warn"}, "then": mark("sev2")}, + }, + } + if withDefault { + body["default"] = mark("sev3") + } + return body +} + +// Enumerated switch on a record field: each value set routes to its arm. +func TestCaseSwitchEnumRouting(t *testing.T) { + p := buildCase(t, levelSwitch(true)) + + for _, tc := range []struct { + level string + want string + }{ + {"error", "sev1"}, + {"fatal", "sev1"}, + {"warn", "sev2"}, + {"debug", "sev3"}, // no arm matches -> default + } { + ctx := newRecordContext(t, util.MapStr{"level": tc.level}) + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Equal(t, tc.want, branchOf(t, ctx), "level=%s", tc.level) + } +} + +// No match and no default: the case block is a no-op, nothing crashed. +func TestCaseNoMatchWithoutDefault(t *testing.T) { + p := buildCase(t, levelSwitch(false)) + ctx := newRecordContext(t, util.MapStr{"level": "debug"}) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + rec, ok := CurrentRecord(ctx) + assert.True(t, ok) + assert.NotContains(t, rec.Fields, "branch") + assert.Contains(t, ctx.GetFlowProcess(), "case:no-match") +} + +// A record missing the switch field never matches a values arm. +func TestCaseFieldMissingFallsThrough(t *testing.T) { + p := buildCase(t, levelSwitch(true)) + ctx := newRecordContext(t, util.MapStr{"other": 1}) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Equal(t, "sev3", branchOf(t, ctx)) +} + +// Arms accept full conditions (`when`) with any operator the if/then/else +// block supports; evaluated against the current record inside sub-chains. +func TestCaseWhenArmsOnRecord(t *testing.T) { + p := buildCase(t, util.MapStr{ + "branches": []interface{}{ + util.MapStr{"when": util.MapStr{"equals": util.MapStr{"svc": "api"}}, "then": mark("api")}, + util.MapStr{"when": util.MapStr{"prefix": util.MapStr{"svc": "db"}}, "then": mark("db")}, + }, + "default": mark("other"), + }) + + api := newRecordContext(t, util.MapStr{"svc": "api"}) + if err := p.Process(api); err != nil { + t.Fatal(err) + } + assert.Equal(t, "api", branchOf(t, api)) + + db := newRecordContext(t, util.MapStr{"svc": "db-replica"}) + if err := p.Process(db); err != nil { + t.Fatal(err) + } + assert.Equal(t, "db", branchOf(t, db)) + + other := newRecordContext(t, util.MapStr{"svc": "cache"}) + if err := p.Process(other); err != nil { + t.Fatal(err) + } + assert.Equal(t, "other", branchOf(t, other)) +} + +// First match wins even when several arms match. +func TestCaseFirstMatchWins(t *testing.T) { + p := buildCase(t, util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"warn"}, "then": mark("first")}, + util.MapStr{"values": []interface{}{"warn", "info"}, "then": mark("second")}, + }, + }) + ctx := newRecordContext(t, util.MapStr{"level": "warn"}) + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Equal(t, "first", branchOf(t, ctx)) +} + +// values + when on the same arm AND together; values alone on another arm +// still matches the same record — the mixed-style ladder stays coherent. +func TestCaseValuesAndWhenCombineAsAnd(t *testing.T) { + p := buildCase(t, util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{ + "values": []interface{}{"error"}, + "when": util.MapStr{"exists": []interface{}{"svc"}}, + "then": mark("both"), + }, + util.MapStr{"values": []interface{}{"error"}, "then": mark("values-only")}, + }, + }) + + withSvc := newRecordContext(t, util.MapStr{"level": "error", "svc": "api"}) + if err := p.Process(withSvc); err != nil { + t.Fatal(err) + } + assert.Equal(t, "both", branchOf(t, withSvc)) + + withoutSvc := newRecordContext(t, util.MapStr{"level": "error"}) + if err := p.Process(withoutSvc); err != nil { + t.Fatal(err) + } + assert.Equal(t, "values-only", branchOf(t, withoutSvc)) +} + +// Pipeline-level usage (no record bound) evaluates arms against the +// context parameters, mirroring if/then/else. +func TestCaseContextFallbackWithoutRecord(t *testing.T) { + p := buildCase(t, util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"error"}, "then": mark("err")}, + }, + "default": mark("rest"), + }) + + ctx := &Context{Context: t.Context()} + ctx.Set("level", "error") + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Equal(t, "err", util.ToString(ctx.Get("branch"))) + + ctx2 := &Context{Context: t.Context()} + ctx2.Set("level", "info") + if err := p.Process(ctx2); err != nil { + t.Fatal(err) + } + assert.Equal(t, "rest", util.ToString(ctx2.Get("branch"))) +} + +// A bound record takes precedence over context parameters carrying the +// same key (same rule the if/then/else block applies). +func TestCaseRecordTakesPrecedenceOverContext(t *testing.T) { + p := buildCase(t, util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"error"}, "then": mark("err")}, + }, + }) + ctx := &Context{Context: t.Context()} + ctx.Set("level", "error") + ctx.Set(RecordContextKey, &event.Event{Fields: util.MapStr{"level": "info"}}) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Contains(t, ctx.GetFlowProcess(), "case:no-match") +} + +// The step compiles through the generic chain compiler (the same path +// ValidatePipelineConfig takes) and records its routing in the flow trace. +func TestCaseStepCompilesViaNewPipeline(t *testing.T) { + step := util.MapStr{"case": util.MapStr{ + "field": "level", + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"warn"}, "then": []interface{}{mark("sev2")}}, + }, + }} + c, err := config.NewConfigFrom(step) + if err != nil { + t.Fatal(err) + } + procs, err := NewPipeline([]*config.Config{c}) + if err != nil { + t.Fatalf("NewPipeline: %v", err) + } + assert.Equal(t, 1, len(procs.List)) + assert.Equal(t, "case", procs.List[0].Name()) + + ctx := newRecordContext(t, util.MapStr{"level": "warn"}) + if err := procs.Process(ctx); err != nil { + t.Fatal(err) + } + assert.Equal(t, "sev2", branchOf(t, ctx)) + assert.Contains(t, ctx.GetFlowProcess(), "case:1") + assert.Contains(t, ctx.GetFlowProcess(), "test_branch_marker") +} + +func TestCaseCompileErrors(t *testing.T) { + mk := func(body util.MapStr) *config.Config { + c, err := config.NewConfigFrom(util.MapStr{"case": body}) + if err != nil { + t.Fatal(err) + } + return c + } + + for name, body := range map[string]util.MapStr{ + "no branches": { + "field": "level", + }, + "arm without values or when": { + "field": "level", + "branches": []interface{}{util.MapStr{"then": mark("x")}}, + }, + "values without field": { + "branches": []interface{}{ + util.MapStr{"values": []interface{}{"error"}, "then": mark("x")}, + }, + }, + "arm without then": { + "field": "level", + "branches": []interface{}{util.MapStr{"values": []interface{}{"error"}}}, + }, + "unknown condition operator": { + "branches": []interface{}{ + util.MapStr{"when": util.MapStr{"nope": util.MapStr{"x": 1}}, "then": mark("x")}, + }, + }, + } { + _, err := NewCaseProcessor(mk(body)) + assert.Error(t, err, name) + } +} + +// Arm chains inherit the enclosing scope: inside a record host (strict +// scope) a batch_only processor nested in an arm is a compile error, the +// same placement rule if/then/else branches follow. +func TestCaseArmInheritsRecordScope(t *testing.T) { + step := util.MapStr{"case": util.MapStr{ + "branches": []interface{}{ + util.MapStr{"when": util.MapStr{"equals": util.MapStr{"x": 1}}, "then": []interface{}{ + util.MapStr{"test_batch_only": util.MapStr{}}, + }}, + }, + }} + c, err := config.NewConfigFrom([]util.MapStr{step}) + if err != nil { + t.Fatal(err) + } + var list []*config.Config + if err := c.Unpack(&list); err != nil { + t.Fatal(err) + } + _, err = NewPipelineWithScope(list, + ScopeInfo{Kind: ScopeRecord, Strict: true}, true) + assert.Error(t, err) + assert.Contains(t, err.Error(), "batch_only") +} diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go index b5feb21d..dd471753 100755 --- a/core/pipeline/processor.go +++ b/core/pipeline/processor.go @@ -146,6 +146,24 @@ func newPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (* continue } + // Handle the case/branches multi-way dispatch: like if/then/else it + // owns its step and its arm chains must compile under the enclosing + // scope for contract placement. + if procConfig.HasField("case") { + p, err := NewCaseProcessorWithScope(procConfig, scope) + if err != nil { + return nil, errors.Wrap(err, "failed to make case/branches processor") + } + if err := validatePlacement(p, stepIdx, scope, false); err != nil { + if scope.Strict { + return nil, err + } + log.Warnf("contract (non-strict): %v", err) + } + procs.AddProcessor(p) + continue + } + if len(procConfig.GetFields()) != 1 { return nil, errors.Errorf("each processor must have exactly one "+ "action, but found %d actions (%v)", diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md index 41f291d8..2e982d58 100644 --- a/docs/content.en/docs/references/pipeline.md +++ b/docs/content.en/docs/references/pipeline.md @@ -308,6 +308,65 @@ processor: message: "non-POST request" ``` +### Multi-branch Dispatch with `case` + +Chaining `if`/`then`/`else` blocks gets unwieldy once the same input routes to three or more destinations. The `case` block is an ordered multi-way dispatch: the **first** matching branch runs (exactly one arm executes), and an optional `default` chain handles everything else. + +```yaml +processor: + - case: + field: log_level + branches: + - values: [error, fatal] + then: + - echo: + message: "route to the alerting pipeline" + - values: [warn] + then: + - echo: + message: "route to the warning pipeline" + default: + - echo: + message: "keep on the default route" +``` + +Branch arms support two match styles, freely mixed within one block: + +| Field | Matches when | +|-------|--------------| +| `values` | The current value of the case block's `field` equals any entry (same semantics as the [`in`](conditions/in/) condition). Requires `field` on the case block. | +| `when` | Any condition operator from the table above (`equals`, `regexp`, `and`, ...). | +| both on one branch | Both must hold (AND). | +| `then` | The processor (or list of processors) to run when the branch matches — required on every branch. | + +Branches without `values` do not need `field`, so a `case` block also works as a pure ordered condition ladder: + +```yaml +processor: + - case: + branches: + - when: + and: + - equals: + env: prod + - regexp: + file.path: '\.json$' + then: + - echo: + message: "structured prod logs" + - when: + exists: + - file.path + then: + - echo: + message: "plain file logs" + default: + - echo: + message: "everything else" +``` + +Value resolution follows `if`/`then`/`else`: inside a per-record sub-chain (e.g. `for_each`) branches see the current record's fields; at pipeline level they read the context through the `_ctx.` prefix. Which arm ran is recorded in the flow trace as `case:1`, `case:2`, ..., `case:default`, or `case:no-match`. + ## Error Handling Pipeline error handling follows these rules: @@ -467,7 +526,7 @@ type ConfigError struct { The compile pass covers: -- **DSL shape** — exactly one action per step, well-formed `if`/`then`/`else` blocks, nested sub-chains (`for_each`, `lane_consumer` inner chains compile recursively); +- **DSL shape** — exactly one action per step, well-formed `if`/`then`/`else` and `case`/`branches` blocks, nested sub-chains (`for_each`, `lane_consumer` inner chains compile recursively); - **Registry existence** — unknown processor names fail with the list of valid names; - **Parameter legality** — each constructor's own validation runs (missing required fields, bad enum values, out-of-range numbers); - **Record contracts** — placement mismatches surface as warnings, or as errors when the config opts in with `strict_contract: true`. From 0355665c1d5bd59bed64390fd8c05e1b9999aba6 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 10:58:46 +0800 Subject: [PATCH 25/65] feat: core and module support for lane-based bulk indexing - lane.GetOrCreate replaces a registered-but-CLOSED lane instead of returning it: consumers close their lane on task exit, and without replacement every producer would emit-fail 'closed' forever with no way for a consumer restart to recover (double-checked under the registry write lock) - safe-read helpers with tests; vfs and instance-model extensions - otel: log-record conversions (+tests) - cluster secrets/loader hardening; host utilization stats counters --- core/elastic/cluster_secrets.go | 20 ++++++--- core/lane/lane.go | 31 +++++++++++--- core/lane/lane_test.go | 39 ++++++++++++++++++ core/model/instance.go | 5 +++ core/otel/convert.go | 67 +++++++++++++++++++++++++++++++ core/otel/log_record.go | 2 + core/otel/otel_test.go | 49 ++++++++++++++++++++++ core/queue/api.go | 4 ++ core/util/saferead.go | 27 +++++++++++++ core/util/saferead_test.go | 60 +++++++++++++++++++++++++++ core/vfs/fs.go | 22 ++++++++++ modules/elastic/cluster_loader.go | 8 ++++ modules/elastic/module.go | 9 +++++ modules/stats/simple.go | 18 +++++++++ 14 files changed, 351 insertions(+), 10 deletions(-) diff --git a/core/elastic/cluster_secrets.go b/core/elastic/cluster_secrets.go index 156f9a67..57ad61d4 100644 --- a/core/elastic/cluster_secrets.go +++ b/core/elastic/cluster_secrets.go @@ -107,22 +107,25 @@ func HydrateClusterSecrets(cfg *ElasticsearchConfig) { v, ok := loadClusterSecret(ClusterBasicAuthPasswordKey(cfg.ID)) if ok { if v == "" { - log.Warnf("cluster [%s]'s basic auth password is empty: %v", cfg.ID) + log.Warnf("cluster [%s]'s basic auth password is empty", cfg.ID) } cfg.BasicAuth.Password = ucfg.SecretString(v) } else { - log.Warnf("cluster [%s]'s basic auth password not found in the keystore: %v", cfg.ID) + log.Warnf("cluster [%s]'s basic auth password not found in the keystore", cfg.ID) } } - if !isRealSecret(cfg.Token.Get()) { + // Empty token = the cluster never had one (no-auth) — nothing to + // hydrate and nothing to warn about. Only a MASKED token (record says a + // token exists, value stashed elsewhere) requires the keystore lookup. + if tok := cfg.Token.Get(); tok != "" && !isRealSecret(tok) { v, ok := loadClusterSecret(ClusterTokenKey(cfg.ID)) if ok { if v == "" { - log.Warnf("cluster [%s]'s token is empty: %v", cfg.ID) + log.Warnf("cluster [%s]'s token is empty", cfg.ID) } cfg.Token = ucfg.SecretString(v) } else { - log.Warnf("cluster [%s]'s token not found in the keystore: %v", cfg.ID) + log.Warnf("cluster [%s]'s token not found in the keystore", cfg.ID) } } } @@ -171,6 +174,13 @@ func isRealSecret(s string) bool { return s != "" && s != ucfg.SecretShadowText } +// IsRealSecret is the exported form of isRealSecret, for app-side code +// building credential payloads from ORM-loaded cluster records (deciding +// whether a hydrated value is safe to ship). +func IsRealSecret(s string) bool { + return isRealSecret(s) +} + func asMapStr(v interface{}) map[string]interface{} { switch m := v.(type) { case map[string]interface{}: diff --git a/core/lane/lane.go b/core/lane/lane.go index add7dca9..d681efbd 100644 --- a/core/lane/lane.go +++ b/core/lane/lane.go @@ -421,15 +421,30 @@ func Create(cfg LaneConfig) (Lane, error) { return l, nil } -// GetOrCreate returns the named lane, creating it lazily on first Emit -// (dynamic pipelines; owner tags it for cascade cleanup). +// GetOrCreate returns the named OPEN lane, creating it lazily on first Emit +// (dynamic pipelines; owner tags it for cascade cleanup). A registered but +// CLOSED instance is replaced with a fresh one built from cfg — lane +// consumers close their lane on task exit, and without replacement the +// registry would hold a permanently-unusable lane that wedges every +// producer (emit fails "closed" forever, consumer restarts cannot recover). func GetOrCreate(cfg LaneConfig) Lane { regMu.RLock() l, ok := registry[cfg.Name] + closed := ok && l.Snapshot().Closed regMu.RUnlock() - if ok { + if ok && !closed { return l } + if closed { + regMu.Lock() + // re-check under the write lock: a concurrent replacer may have won + if cur, still := registry[cfg.Name]; still && !cur.Snapshot().Closed { + regMu.Unlock() + return cur + } + delete(registry, cfg.Name) + regMu.Unlock() + } l, err := Create(cfg) if err != nil { regMu.RLock() @@ -440,11 +455,17 @@ func GetOrCreate(cfg LaneConfig) Lane { return l } -// Get returns the named lane (nil when absent). +// Get returns the named OPEN lane; nil when absent or closed. A closed lane +// accepts no emits and no subscribers — callers want a usable channel, and +// GetOrCreate is the re-creation path. func Get(name string) Lane { regMu.RLock() defer regMu.RUnlock() - return registry[name] + l := registry[name] + if l == nil || l.Snapshot().Closed { + return nil + } + return l } // Delete removes a lane; refuses while records are in flight unless force. diff --git a/core/lane/lane_test.go b/core/lane/lane_test.go index fa46ef11..d1d00918 100644 --- a/core/lane/lane_test.go +++ b/core/lane/lane_test.go @@ -26,6 +26,45 @@ func mustCreate(t *testing.T, cfg LaneConfig) Lane { return l } +// TestRegistry_ClosedLaneSelfHeals pins the wedge fix: a lane closed by its +// consumer's task exit must not linger as a permanently-unusable registry +// entry — Get skips it and GetOrCreate replaces it with a fresh instance. +func TestRegistry_ClosedLaneSelfHeals(t *testing.T) { + l1 := mustCreate(t, LaneConfig{Name: "heal", Capacity: 4}) + if err := l1.Close(); err != nil { + t.Fatal(err) + } + if Get("heal") != nil { + t.Fatal("Get must skip a closed lane") + } + l2 := GetOrCreate(LaneConfig{Name: "heal", Capacity: 4}) + if l2 == nil { + t.Fatal("GetOrCreate must replace the closed lane") + } + if l2 == l1 { + t.Fatal("GetOrCreate returned the closed instance") + } + if err := l2.Emit(rec("fresh")); err != nil { + t.Fatalf("emit on replaced lane: %v", err) + } + if err := l1.Emit(rec("stale")); err == nil { + t.Fatal("stale handle must keep erroring (drained instance)") + } + got := make(chan *event.Event, 1) + l2.Subscribe(func(r *event.Event) { got <- r }) + select { + case r := <-got: + if r.Fields["message"] != "fresh" { + t.Fatalf("got %v", r) + } + case <-time.After(time.Second): + t.Fatal("replaced lane did not deliver") + } + if err := Delete("heal", true); err != nil { + t.Fatal(err) + } +} + func TestLane_BlockBackpressure(t *testing.T) { l := mustCreate(t, LaneConfig{Name: "bp", Capacity: 2}) // no subscriber yet: fills up deterministically diff --git a/core/model/instance.go b/core/model/instance.go index bdd7a0e2..7f652d85 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -65,6 +65,11 @@ type Instance struct { // registration/heartbeat upserts preserve the stored value. Groups []string `json:"groups,omitempty" elastic_mapping:"groups:{type:keyword}"` + // Alias is a SERVER-OWNED display name (UI-managed, e.g. "边缘网关-生产"): + // shown in the management UI in place of the auto-generated instance + // name. Like Groups, heartbeat upserts preserve the stored value. + Alias string `json:"alias,omitempty" elastic_mapping:"alias:{type:keyword,copy_to:search_text}"` + //user can pass Description string `json:"description,omitempty" config:"description" elastic_mapping:"description:{type:keyword}"` diff --git a/core/otel/convert.go b/core/otel/convert.go index c3e8c96e..2620de85 100644 --- a/core/otel/convert.go +++ b/core/otel/convert.go @@ -61,6 +61,7 @@ func FromEvent(e *event.Event) *LogRecord { if v, ok := resourceMap(e.Meta); ok { r.Resource = v } + r.Resource = agentToResource(e.Agent, r.Resource) r.SeverityText, _ = r.Attributes[FieldLogLevel].(string) r.Body = r.Attributes[FieldMessage] @@ -120,9 +121,75 @@ func (r *LogRecord) ToEvent() *event.Event { } e.Meta[MetaResourceKey] = r.Resource.Clone() } + if e.Agent == nil { + e.Agent = agentFromResource(r.Resource) + } return e } +// agentToResource merges the collecting agent's identity into the record's +// Resource attributes so it survives the OTLP wire — the LogRecord model has +// no other carrier for it. Existing keys win: a resource that already +// describes its host (e.g. written by an upstream processor) is not +// overwritten, which also keeps cascaded exports idempotent. The merge is +// deterministic, so mutating a referenced resource map in place is safe. +func agentToResource(a *event.AgentMeta, res util.MapStr) util.MapStr { + if a == nil { + return res + } + if res == nil { + res = util.MapStr{} + } + setIfAbsent(res, ResourceHostName, a.Hostname) + setIfAbsent(res, ResourceHostIP, a.MajorIP) + setIfAbsent(res, ResourceAgentID, a.AgentID) + if len(a.Tags) > 0 { + setIfAbsent(res, ResourceAgentTags, a.Tags) + } + return res +} + +// agentFromResource rebuilds the collecting agent identity from Resource +// attributes written by agentToResource — the OTLP intake path has no other +// source for it. A resource without any agent-describing key yields nil. +func agentFromResource(res util.MapStr) *event.AgentMeta { + if res == nil { + return nil + } + a := &event.AgentMeta{} + a.AgentID, _ = res[ResourceAgentID].(string) + a.Hostname, _ = res[ResourceHostName].(string) + a.MajorIP, _ = res[ResourceHostIP].(string) + switch v := res[ResourceAgentTags].(type) { + case []string: + a.Tags = v + case []interface{}: // OTLP wire round-trip decodes arrays as []interface{} + tags := make([]string, 0, len(v)) + for _, item := range v { + if s, ok := item.(string); ok { + tags = append(tags, s) + } + } + if len(tags) > 0 { + a.Tags = tags + } + } + if a.AgentID == "" && a.Hostname == "" && a.MajorIP == "" && len(a.Tags) == 0 { + return nil + } + return a +} + +func setIfAbsent(m util.MapStr, k string, v interface{}) { + if _, ok := m[k]; ok { + return + } + if s, ok := v.(string); ok && s == "" { + return + } + m[k] = v +} + func resourceMap(meta util.MapStr) (util.MapStr, bool) { if meta == nil { return nil, false diff --git a/core/otel/log_record.go b/core/otel/log_record.go index 32d9708f..477d10e2 100644 --- a/core/otel/log_record.go +++ b/core/otel/log_record.go @@ -74,6 +74,8 @@ const ( const ( ResourceHostName = "host_name" ResourceHostIP = "host_ip" + ResourceAgentID = "agent_id" + ResourceAgentTags = "agent_tags" ResourceServiceName = "service_name" ResourceServiceVersion = "service_version" ResourceProcessPID = "process_pid" diff --git a/core/otel/otel_test.go b/core/otel/otel_test.go index 35245546..83d1b3fe 100644 --- a/core/otel/otel_test.go +++ b/core/otel/otel_test.go @@ -69,6 +69,55 @@ func TestEventRoundTrip(t *testing.T) { } } +// The agent identity must survive the OTLP LogRecord projection: FromEvent +// merges it into Resource attributes, ToEvent restores it when the inbound +// event has none. A resource key set upstream must not be overwritten, and +// an event that already carries its Agent must keep it as-is. +func TestAgentIdentitySurvivesLogRecord(t *testing.T) { + e := &event.Event{ + Timestamp: time.Now().UTC(), + Agent: &event.AgentMeta{ + AgentID: "agent-7", + Hostname: "web-1", + MajorIP: "10.0.0.5", + Tags: []string{"prod", "edge"}, + }, + Fields: util.MapStr{FieldMessage: "m"}, + } + + rec := FromEvent(e) + if rec.Resource[ResourceAgentID] != "agent-7" || rec.Resource[ResourceHostName] != "web-1" { + t.Fatalf("agent identity not merged into resource: %v", rec.Resource) + } + if rec.Resource[ResourceHostIP] != "10.0.0.5" { + t.Fatalf("host_ip = %v", rec.Resource) + } + + back := rec.ToEvent() + if back.Agent == nil || back.Agent.AgentID != "agent-7" || back.Agent.Hostname != "web-1" || + back.Agent.MajorIP != "10.0.0.5" { + t.Fatalf("agent not restored: %v", back.Agent) + } + + // upstream resource keys win over the agent merge + e2 := &event.Event{ + Agent: &event.AgentMeta{AgentID: "agent-7", Hostname: "web-1"}, + Meta: util.MapStr{MetaResourceKey: util.MapStr{ResourceHostName: "already-set"}}, + Fields: util.MapStr{FieldMessage: "m"}, + } + rec2 := FromEvent(e2) + if rec2.Resource[ResourceHostName] != "already-set" { + t.Fatalf("existing resource keys must not be overwritten: %v", rec2.Resource) + } + + // no agent anywhere: nothing restored, no empty AgentMeta injected + e3 := &event.Event{Fields: util.MapStr{FieldMessage: "m"}} + back3 := FromEvent(e3).ToEvent() + if back3.Agent != nil { + t.Fatalf("agent must stay nil without identity, got %v", back3.Agent) + } +} + func TestEnvelopeRoundTrip(t *testing.T) { ts := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC) e := &event.Event{ diff --git a/core/queue/api.go b/core/queue/api.go index 6d11e81c..3f3fd41e 100755 --- a/core/queue/api.go +++ b/core/queue/api.go @@ -38,6 +38,10 @@ import ( "infini.sh/framework/core/util" ) +// ErrEmptyQueue 队列尚无任何数据 (无写段) — 浏览/取样场景按"无消息"处理, +// 不作为错误。DiskQueue 高级处理器在 AcquireConsumer 时返回。 +var ErrEmptyQueue = errors.New("empty queue") + type QueueAPI interface { Name() string Init(string) error diff --git a/core/util/saferead.go b/core/util/saferead.go index 07cf200f..36645ac2 100644 --- a/core/util/saferead.go +++ b/core/util/saferead.go @@ -111,6 +111,33 @@ func isTooBroadReadRoot(c string) bool { return false } +// ValidateReadRoot applies the read policy to a configured root without +// requiring it to exist yet: system trees are never allowed and a +// wholesale top-level tree is treated as misconfiguration. Collection +// roots (agent file intake) may legitimately appear after startup, so +// unlike NewReadGuard existence is left to the caller. +func ValidateReadRoot(root string) error { + c := canonicalReadPath(root) + if IsSystemReadPath(c) { + return fmt.Errorf("refusing to read from system path: %s", c) + } + if isTooBroadReadRoot(c) { + return fmt.Errorf("path %s is too broad, use a specific log directory instead", c) + } + return nil +} + +// IsConfinedUnder reports whether path stays at or below root once both +// sides are canonicalized (symlink-resolved). It is the collection-side +// counterpart of ReadGuard's containment rule: a file reached through a +// symlink that jumps out of the collection root is not confined, +// whatever spelling the walk handed over. +func IsConfinedUnder(root, path string) bool { + r := canonicalReadPath(root) + c := canonicalReadPath(path) + return c == r || strings.HasPrefix(c, r+string(filepath.Separator)) +} + // ReadGuard confines file reads to a fixed set of allowed roots. Roots // are canonicalized (absolute + symlink-resolved) once at construction; // every later ResolveUnder call re-resolves the target the same way so a diff --git a/core/util/saferead_test.go b/core/util/saferead_test.go index 35a75bcb..27f0180e 100644 --- a/core/util/saferead_test.go +++ b/core/util/saferead_test.go @@ -274,3 +274,63 @@ func TestIsTooBroadReadRoot(t *testing.T) { t.Error("expected specific subdir to be acceptable") } } + +func TestValidateReadRoot(t *testing.T) { + dir := t.TempDir() + denied := []string{ + "/", "/etc", "/etc/nginx", "/root", + "/home", "/var", "/opt", // wholesale top-level trees + "/var/log/../../etc", // traversal into a system tree + } + for _, p := range denied { + if err := ValidateReadRoot(p); err == nil { + t.Errorf("expected [%s] to be rejected", p) + } + } + allowed := []string{ + "/var/log", + "/var/log/elasticsearch", + "/opt/myapp/logs", + filepath.Join(dir, "logs"), + filepath.Join(dir, "not-created-yet"), // existence is the caller's concern + } + for _, p := range allowed { + if err := ValidateReadRoot(p); err != nil { + t.Errorf("expected [%s] to pass: %v", p, err) + } + } +} + +func TestIsConfinedUnder(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "secret.log") + + if !IsConfinedUnder(root, root) { + t.Error("the root itself must be confined") + } + if !IsConfinedUnder(root, filepath.Join(root, "app", "a.log")) { + t.Error("file under the root must be confined") + } + if !IsConfinedUnder(root, filepath.Join(root, "sub", "..", "a.log")) { + t.Error("cleaned traversal inside the root must be confined") + } + if IsConfinedUnder(root, outside) { + t.Error("file outside the root must not be confined") + } + if IsConfinedUnder(root, root+"-escape") { + t.Error("sibling sharing the prefix must not pass as confined") + } + + link := filepath.Join(root, "jump.log") + // the target must exist, otherwise EvalSymlinks cannot resolve the + // link and the deepest-ancestor fallback would compare the raw link + if err := os.WriteFile(outside, []byte("secret"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, link); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + if IsConfinedUnder(root, link) { + t.Error("symlink escaping the root must not be confined") + } +} diff --git a/core/vfs/fs.go b/core/vfs/fs.go index 32197661..4d6c59bf 100755 --- a/core/vfs/fs.go +++ b/core/vfs/fs.go @@ -43,6 +43,7 @@ import ( "os" "path" "path/filepath" + "regexp" "runtime" "sort" "strconv" @@ -597,9 +598,30 @@ func serveFile(w http.ResponseWriter, r *http.Request, fs http.FileSystem, name // serveContent will check modification time sizeFunc := func() (int64, error) { return d.Size(), nil } + if cc := spaCacheControl(r.URL.Path); cc != "" { + w.Header().Set("Cache-Control", cc) + } serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) } +// vfsAssetHashRE 匹配前端构建产物文件名里的内容哈希段（如 +// index-DPak5aSO.js、pizza_engine_bg-B2pSYpQe.wasm）。 +var vfsAssetHashRE = regexp.MustCompile(`-[A-Za-z0-9_]{8,}\.[A-Za-z0-9]+$`) + +// spaCacheControl SPA 静态资源缓存策略：/assets/ 下带内容哈希的构建产物 +// 内容永不原地变更，标记 immutable 让浏览器跳过重复校验（大 bundle 场景 +// 二次访问收益明显）；HTML 入口必须始终再验证，保证发版后能拉到新 hash +// 的资源清单。返回空串表示不干预（沿用 Last-Modified 协商缓存）。 +func spaCacheControl(p string) string { + if strings.HasPrefix(p, "/assets/") && vfsAssetHashRE.MatchString(p) { + return "public, max-age=31536000, immutable" + } + if strings.HasSuffix(p, ".html") || strings.HasSuffix(p, "/") || p == "" { + return "no-cache" + } + return "" +} + // toHTTPError returns a non-specific HTTP error message and status code // for a given non-nil error value. It's important that toHTTPError does not // actually return err.Error(), since msg and httpStatus are returned to users, diff --git a/modules/elastic/cluster_loader.go b/modules/elastic/cluster_loader.go index f1db9bd0..b5bd7b92 100644 --- a/modules/elastic/cluster_loader.go +++ b/modules/elastic/cluster_loader.go @@ -23,6 +23,14 @@ import ( // created via the /easysearch/ API (served by modules/easysearch) usable as // live clients after (re)start. func LoadClustersFromORM() { + // Lightweight instances (e.g. agents) run without any ORM backend — + // getHandler() panics when none is registered, and the agent's module + // start order may also reach here before the backend registers. Nothing + // to load in either case; skip quietly. + if !orm.IsHandlerRegistered() { + log.Debug("no ORM backend registered, skip loading clusters from ORM") + return + } ctx := orm.NewContext().DirectAccess() orm.WithModel(ctx, &elastic.ElasticsearchConfig{}) res, err := orm.SearchV2(ctx, orm.NewQuery().Size(1000)) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 7bad426f..b9f6ad0c 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -423,6 +423,15 @@ var ormInited bool func (module *ElasticModule) Start() error { if !moduleConfig.Enabled { + // The client provider and the ORM change hook are registered even + // when this module is disabled, so runtime cluster pushes via the + // /easysearch/ API work without it. A restart must not silently lose + // that persisted state: restore dynamic clusters as live clients + // here too. Best-effort — the sqlite ORM backend starts before this + // module, failures just log a warning. + if global.Env().SystemConfig.ORMConfig.Enabled { + LoadClustersFromORM() + } return nil } diff --git a/modules/stats/simple.go b/modules/stats/simple.go index 7a9877b8..1639ddd4 100755 --- a/modules/stats/simple.go +++ b/modules/stats/simple.go @@ -40,6 +40,9 @@ import ( log "github.com/cihub/seelog" "github.com/segmentio/encoding/json" + gcpu "github.com/shirou/gopsutil/v4/cpu" + gmem "github.com/shirou/gopsutil/v4/mem" + gnet "github.com/shirou/gopsutil/v4/net" "github.com/shirou/gopsutil/v4/process" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" @@ -388,6 +391,21 @@ func (s *Stats) StatsAll() string { systemStats["store"] = int64(storage) //maybe too heavy to call frequently } + // host-level utilization + network counters (cumulative since boot, + // consumers compute rates from deltas between polls). Best-effort: + // any failure just omits the keys, never fails the whole response. + if hostPercents, err := gcpu.Percent(0, false); err == nil && len(hostPercents) > 0 { + systemStats["host_cpu_percent"] = int64(hostPercents[0]) + } + if vm, err := gmem.VirtualMemory(); err == nil && vm != nil { + systemStats["host_mem_total_bytes"] = int64(vm.Total) + systemStats["host_mem_used_bytes"] = int64(vm.Used) + } + if ioCounters, err := gnet.IOCounters(false); err == nil && len(ioCounters) > 0 { + systemStats["net_in_bytes"] = int64(ioCounters[0].BytesRecv) + systemStats["net_out_bytes"] = int64(ioCounters[0].BytesSent) + } + result["system"] = systemStats return util.ToJson(result, false) From c27efadb9233482daa57dc4e92d1f68a1fc965ff Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 10:58:46 +0800 Subject: [PATCH 26/65] feat(queue): disk-queue consumer self-heal A consumer parked at the write head whose offset's data files are gone no longer errors out: re-open the segment file if it exists by now and continue, otherwise report an empty fetch so the consumer keeps waiting. openReadFile factors the shared tail of ResetOffset and the self-healing reset paths. Covered by consumer_selfheal_test. --- modules/queue/api.go | 22 +- modules/queue/disk_queue/consumer.go | 116 ++++++++- .../disk_queue/consumer_selfheal_test.go | 222 ++++++++++++++++++ modules/queue/disk_queue/diskqueue.go | 5 + modules/queue/disk_queue/module.go | 2 +- 5 files changed, 352 insertions(+), 15 deletions(-) create mode 100644 modules/queue/disk_queue/consumer_selfheal_test.go diff --git a/modules/queue/api.go b/modules/queue/api.go index d134d87c..5f56f516 100644 --- a/modules/queue/api.go +++ b/modules/queue/api.go @@ -27,6 +27,7 @@ package queue import ( + stderrors "errors" "fmt" "infini.sh/framework/core/global" "infini.sh/framework/core/security" @@ -73,6 +74,11 @@ func init() { //reset consumer offset api.HandleAPIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset) api.HandleUIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueUpdate)) + // 消费者管理: 位点读取与消费者删除 (残留消费组/位点清理, Console 同款)。 + api.HandleAPIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset) + api.HandleUIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueRead)) + api.HandleAPIMethod(api.DELETE, "/queue/:id/consumer/:consumer_id", module.QueueDeleteConsumerByID) + api.HandleUIMethod(api.DELETE, "/queue/:id/consumer/:consumer_id", module.QueueDeleteConsumerByID, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueUpdate)) //get consumer offset api.HandleAPIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset) api.HandleUIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueRead)) @@ -331,8 +337,14 @@ func (module *API) QueueExplore(w http.ResponseWriter, req *http.Request, ps htt result := util.MapStr{} status := 200 if err != nil { - result["error"] = err.Error() - status = 500 + // 空队列 (尚无写入) 是浏览/取样的常态, 按"无消息"返回而非 500 + if stderrors.Is(err, queue1.ErrEmptyQueue) { + result["empty"] = true + err = nil + } else { + result["error"] = err.Error() + status = 500 + } } if len(messages) > 0 { if dataIsString { @@ -396,7 +408,11 @@ func (module *API) QueueExplore(w http.ResponseWriter, req *http.Request, ps htt return } } else { - log.Errorf("can't acquire consumer [%v] for [%v]", consumer.Key(), qConfig.Name) + if stderrors.Is(err, queue1.ErrEmptyQueue) { + log.Debugf("queue [%v] is empty, skip consumer [%v]", qConfig.Name, consumer.Key()) + } else { + log.Errorf("can't acquire consumer [%v] for [%v]", consumer.Key(), qConfig.Name) + } } } else { diff --git a/modules/queue/disk_queue/consumer.go b/modules/queue/disk_queue/consumer.go index c18bbd83..83ffe1c0 100644 --- a/modules/queue/disk_queue/consumer.go +++ b/modules/queue/disk_queue/consumer.go @@ -144,7 +144,20 @@ READ_MSG: // check reader if d.reader == nil { - return messages, false, errors.New("reader is nil") + //parked at the write head (offset's data files are gone): the + //write segment file may exist by now, re-open it and continue; + //otherwise report an empty fetch so the consumer keeps waiting + //instead of erroring out + fileName, exists, _ := SmartGetFileName(d.mCfg, d.queue, d.segment) + if !exists && !util.FileExists(fileName) { + if len(messages) == 0 && d.cCfg.EOFRetryDelayInMs > 0 { + time.Sleep(time.Duration(d.cCfg.EOFRetryDelayInMs) * time.Millisecond) + } + return messages, false, nil + } + if rerr := d.ResetOffset(d.segment, d.readPos); rerr != nil || d.reader == nil { + return messages, false, nil + } } //read message size err = binary.Read(d.reader, binary.BigEndian, &msgSize) @@ -477,13 +490,15 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { } if segment > d.diskQueue.writeSegmentNum { - log.Errorf("reading segment [%v] is greater than writing segment [%v]", segment, d.diskQueue.writeSegmentNum) - return io.EOF + log.Warnf("queue:%v,%v, consumer:%v, offset [%v,%v] is beyond the write segment [%v], the queue's data was likely wiped or recreated, reset to the earliest retained data", + d.qCfg.Name, d.queue, d.cCfg.Key(), segment, readPos, d.diskQueue.writeSegmentNum) + return d.resetToEarliestRetainedData() } if segment == d.diskQueue.writeSegmentNum && readPos > d.diskQueue.writePos { - log.Errorf("reading position [%v] is greater than writing position [%v]", readPos, d.diskQueue.writePos) - return io.EOF + log.Warnf("queue:%v,%v, consumer:%v, read position [%v] is beyond the write position [%v] on segment [%v], restart the segment from the beginning", + d.qCfg.Name, d.queue, d.cCfg.Key(), readPos, d.diskQueue.writePos, segment) + readPos = 0 } if d.segment != segment { @@ -512,13 +527,16 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { if d.mCfg.AutoSkipCorruptFile { nextSegment := d.segment + 1 if nextSegment > d.diskQueue.writeSegmentNum { - return errors.Errorf("%v not found and next segment(%v) greater than current write segment(%v)", fileName, nextSegment, d.diskQueue.writeSegmentNum) + log.Debugf("queue:%v,%v, consumer:%v, offset:%v,%v, file %v is missing and no later segment exists, park at the write head", + d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, fileName) + d.parkAtWriteHead() + return nil } log.Warnf("queue:%v,%v, consumer:%v, offset:%v,%v, file missing: %v, auto skip to next file", d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, fileName) RETRY_NEXT_FILE: - // there are segments in the middle - if nextSegment < d.diskQueue.writeSegmentNum { + // there are segments in the middle, or the write segment itself + if nextSegment <= d.diskQueue.writeSegmentNum { fileName, exists, next_file_exists = SmartGetFileName(d.mCfg, d.queue, nextSegment) if exists || util.FileExists(fileName) { log.Debugf("retry skip to next file: %v, exists", fileName) @@ -532,7 +550,10 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { goto RETRY_NEXT_FILE } } else { - return errors.New(fileName + " not found, next segment greater than current write segment") + log.Debugf("queue:%v,%v, consumer:%v, offset:%v,%v, no segment file exists up to the write segment %v, park at the write head", + d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, d.diskQueue.writeSegmentNum) + d.parkAtWriteHead() + return nil } } else { return errors.New(fileName + " not found and auto_skip_corrupt_file not enabled.") @@ -543,13 +564,19 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { FIND_NEXT_FILE: //if next file exists, and current file is not the last file, the file should be completed loaded - if next_file_exists || d.diskQueue.writeSegmentNum > segment { + if next_file_exists || d.diskQueue.writeSegmentNum > d.segment { d.fileLoadCompleted = true } else { d.fileLoadCompleted = false } - var err error + return d.openReadFile(fileName) +} + +// openReadFile points the consumer's reader at an existing segment file, +// seeking to the current read position. It is the shared tail of +// ResetOffset and the self-healing reset paths below. +func (d *Consumer) openReadFile(fileName string) error { readFile, err := os.OpenFile(fileName, os.O_RDONLY, 0600) if err != nil { log.Error(err) @@ -582,3 +609,70 @@ FIND_NEXT_FILE: d.fileName = fileName return nil } + +// resetToEarliestRetainedData re-anchors the consumer onto the earliest +// segment file that still exists on disk. It runs when the requested +// offset segment is beyond the current write segment: the queue's data +// files were wiped (or the queue was deleted and recreated) while the +// consumer's persisted offset still references the removed segments. +// Returning an error there used to wedge pipeline consumers in an +// acquire-retry panic loop; instead, consume whatever data survives +// (at-least-once), and when nothing is retained park at the write head +// and report EOF so the owning worker retries once data arrives. +func (d *Consumer) resetToEarliestRetainedData() error { + for seg := int64(0); seg <= d.diskQueue.writeSegmentNum; seg++ { + fileName, exists, nextFileExists := SmartGetFileName(d.mCfg, d.queue, seg) + if !exists && !util.FileExists(fileName) { + continue + } + log.Warnf("queue:%v,%v, consumer:%v, reset to the earliest retained segment: %v", d.qCfg.Name, d.queue, d.cCfg.Key(), fileName) + if d.readFile != nil { + //potential file handler leak + d.readFile.Close() + } + d.segment = seg + d.readPos = 0 + d.maxBytesPerFileRead = 0 + d.diskQueue.UpdateSegmentConsumerInReading(d.ID, d.segment) + if nextFileExists || d.diskQueue.writeSegmentNum > seg { + d.fileLoadCompleted = true + } else { + d.fileLoadCompleted = false + } + return d.openReadFile(fileName) + } + + //nothing retained on disk: park at the write head and wait for new data + log.Debugf("queue:%v,%v, consumer:%v, no retained data on disk, park at the write head: segment %v", + d.qCfg.Name, d.queue, d.cCfg.Key(), d.diskQueue.writeSegmentNum) + d.parkAtWriteHead() + return nil +} + +// parkAtWriteHead leaves the consumer waiting for new data at the current +// write position, with no reader attached: every data file the offset +// referenced is gone, so there is nothing to read until the writer +// creates the write segment file. FetchMessages re-opens the segment as +// soon as it appears, so callers see a healthy (if idle) consumer +// instead of an error that would panic-loop the owning worker. +func (d *Consumer) parkAtWriteHead() { + if d.readFile != nil { + d.readFile.Close() + d.readFile = nil + } + d.reader = nil + d.segment = d.diskQueue.writeSegmentNum + d.readPos = 0 + d.maxBytesPerFileRead = 0 + d.lastFileSize = 0 + d.fileLoadCompleted = false + d.diskQueue.UpdateSegmentConsumerInReading(d.ID, d.segment) + + //idle workers get recycled and re-acquire on a regular cadence, which + //would replay this warning forever — rate-limit it to once a minute + if now := time.Now().UnixMilli(); now-d.diskQueue.parkWarnAt.Load() > 60*1000 { + d.diskQueue.parkWarnAt.Store(now) + log.Warnf("queue:%v,%v, consumer:%v, offset:%v,%v, data files are missing (queue wiped or recreated), park at the write head: segment %v, waiting for new data", + d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, d.diskQueue.writeSegmentNum) + } +} diff --git a/modules/queue/disk_queue/consumer_selfheal_test.go b/modules/queue/disk_queue/consumer_selfheal_test.go new file mode 100644 index 00000000..f2329a38 --- /dev/null +++ b/modules/queue/disk_queue/consumer_selfheal_test.go @@ -0,0 +1,222 @@ +package queue + +import ( + "os" + "testing" + "time" + + "infini.sh/framework/core/global" + "infini.sh/framework/core/queue" +) + +// Regression tests for consumers whose persisted offset points at segment +// files that no longer exist (queue wiped / deleted and recreated, or +// retention removed files under a lagging consumer). AcquireConsumer used +// to return "not found, next segment greater than current write segment" +// and wedge pipeline consumers in an acquire-retry panic loop; the +// consumer must instead self-heal onto whatever data is still on disk. + +func newSelfhealTestQueue(t *testing.T, name string) (*DiskBasedQueue, string) { + t.Helper() + global.Env().SystemConfig.PathConfig.Data = t.TempDir() + dataPath := GetDataPath(name) + _ = os.MkdirAll(dataPath, 0755) + + cfg := &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 104857600, + MaxBytesPerFile: 1024, // small segments: multiple files quickly + SyncEveryRecords: 1, + SyncTimeoutInMS: 10, + WriteTimeoutInMS: 1000, + EOFRetryDelayInMs: 100, + ReadChanBuffer: 10, + WriteChanBuffer: 10, + AutoSkipCorruptFile: true, + } + return NewDiskQueueByConfig(name, dataPath, cfg), dataPath +} + +func putTestRecords(t *testing.T, d *DiskBasedQueue, n int, payload string) { + t.Helper() + for i := 0; i < n; i++ { + res := d.Put([]byte(payload)) + if res.Error != nil { + t.Fatalf("put: %v", res.Error) + } + } + time.Sleep(500 * time.Millisecond) +} + +// Offset lands inside the range of segments, but every file up to (not +// including) the current write segment was wiped: the skip-forward walk +// must include the write segment itself, which it previously never +// checked. +func TestAcquireConsumerSkipsToWriteSegment(t *testing.T) { + name := "test-selfheal-skip" + d, _ := newSelfhealTestQueue(t, name) + defer d.Close() + + putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef") + if d.writeSegmentNum < 2 { + t.Fatalf("expected several segments, got write segment %v", d.writeSegmentNum) + } + + // wipe every segment file below the last written one (old data gone, + // fresh data still sits in the newest segment file) + for seg := int64(0); seg < d.writeSegmentNum-1; seg++ { + if err := os.Remove(GetFileName(name, seg)); err != nil && !os.IsNotExist(err) { + t.Fatalf("remove segment %v: %v", seg, err) + } + } + + c, err := d.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-skip"}, queue.NewOffset(1, 512)) + if err != nil { + t.Fatalf("acquire with wiped offset should self-heal, got: %v", err) + } + defer c.Close() + + msgs, _, err := c.FetchMessages(&queue.Context{}, 500) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if len(msgs) == 0 { + t.Fatal("expected the records retained in the write segment to be consumable") + } + t.Logf("self-healed onto write segment %v, fetched %v messages", d.writeSegmentNum, len(msgs)) +} + +// Offset is beyond the write segment entirely (queue deleted and +// recreated, writer restarted from a low segment): the consumer must +// re-anchor onto the earliest retained segment instead of erroring. +func TestAcquireConsumerResetsToEarliestRetainedData(t *testing.T) { + name := "test-selfheal-reset" + d, dataPath := newSelfhealTestQueue(t, name) + + putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef") + d.Close() + + // wipe the whole queue dir except meta, then reopen the writer and + // produce fresh data into low segments — the "queue recreated" state + entries, err := os.ReadDir(dataPath) + if err != nil { + t.Fatalf("readdir: %v", err) + } + for _, e := range entries { + if e.Name() != "meta.dat" { + if err := os.Remove(dataPath + string(os.PathSeparator) + e.Name()); err != nil { + t.Fatalf("remove %v: %v", e.Name(), err) + } + } + } + + d2 := NewDiskQueueByConfig(name, dataPath, &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 104857600, + MaxBytesPerFile: 1048576, + SyncEveryRecords: 1, + SyncTimeoutInMS: 10, + WriteTimeoutInMS: 1000, + EOFRetryDelayInMs: 100, + ReadChanBuffer: 10, + WriteChanBuffer: 10, + AutoSkipCorruptFile: true, + }) + defer d2.Close() + putTestRecords(t, d2, 5, "post-wipe message") + + // stale offset far beyond the recreated queue's write segment + c, err := d2.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-reset"}, queue.NewOffset(7, 104857842)) + if err != nil { + t.Fatalf("acquire with offset beyond retained data should self-heal, got: %v", err) + } + defer c.Close() + + // the writer flushes asynchronously: poll until the 5 post-wipe + // messages are all consumed + total := 0 + deadline := time.Now().Add(5 * time.Second) + for total < 5 && time.Now().Before(deadline) { + msgs, _, err := c.FetchMessages(&queue.Context{}, 10) + if err != nil { + t.Fatalf("fetch: %v", err) + } + total += len(msgs) + time.Sleep(100 * time.Millisecond) + } + if total != 5 { + t.Fatalf("expected the 5 post-wipe messages, got %v", total) + } +} + +// No data file survives at all (fully wiped queue, nothing written since): +// acquiring with a stale offset must succeed with the consumer parked at +// the write head — FetchMessages reports empty instead of erroring — and +// consumption must resume in place once the writer produces data. +func TestAcquireConsumerParksAtWriteHeadWhenNoData(t *testing.T) { + name := "test-selfheal-park" + d, dataPath := newSelfhealTestQueue(t, name) + + putTestRecords(t, d, 200, "0123456789abcdef0123456789abcdef0123456789abcdef") + d.Close() + + // wipe the whole queue dir except meta and reopen the writer with no + // data on disk + entries, err := os.ReadDir(dataPath) + if err != nil { + t.Fatalf("readdir: %v", err) + } + for _, e := range entries { + if e.Name() != "meta.dat" { + if err := os.Remove(dataPath + string(os.PathSeparator) + e.Name()); err != nil { + t.Fatalf("remove %v: %v", e.Name(), err) + } + } + } + + d2 := NewDiskQueueByConfig(name, dataPath, &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 104857600, + MaxBytesPerFile: 1048576, + SyncEveryRecords: 1, + SyncTimeoutInMS: 10, + WriteTimeoutInMS: 1000, + EOFRetryDelayInMs: 100, + ReadChanBuffer: 10, + WriteChanBuffer: 10, + AutoSkipCorruptFile: true, + }) + defer d2.Close() + time.Sleep(500 * time.Millisecond) + + // stale offset beyond everything: parks at the write head + c, err := d2.AcquireConsumer(&queue.QueueConfig{ID: name, Name: name}, &queue.ConsumerConfig{Name: "selfheal-park", EOFRetryDelayInMs: 50}, queue.NewOffset(20, 5)) + if err != nil { + t.Fatalf("acquire with no retained data should park, got: %v", err) + } + defer c.Close() + + msgs, _, err := c.FetchMessages(&queue.Context{}, 10) + if err != nil { + t.Fatalf("fetch on parked consumer should be a clean empty, got: %v", err) + } + if len(msgs) != 0 { + t.Fatalf("expected empty fetch on parked consumer, got %v", len(msgs)) + } + + // data arrives: the parked consumer picks it up without re-acquire + putTestRecords(t, d2, 3, "post-park message") + total := 0 + deadline := time.Now().Add(5 * time.Second) + for total < 3 && time.Now().Before(deadline) { + msgs, _, err = c.FetchMessages(&queue.Context{}, 10) + if err != nil { + t.Fatalf("fetch after data arrived: %v", err) + } + total += len(msgs) + time.Sleep(100 * time.Millisecond) + } + if total != 3 { + t.Fatalf("expected 3 post-park messages, got %v", total) + } +} diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go index 8f5949a5..f4ebc0f8 100644 --- a/modules/queue/disk_queue/diskqueue.go +++ b/modules/queue/disk_queue/diskqueue.go @@ -60,6 +60,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" log "github.com/cihub/seelog" @@ -91,6 +92,10 @@ type DiskBasedQueue struct { exitFlag int32 needSync bool + //last time a consumer parked at the write head logged a warning, + //rate-limited because idle workers re-acquire (and re-park) regularly + parkWarnAt atomic.Int64 + consumerMode bool // read related diff --git a/modules/queue/disk_queue/module.go b/modules/queue/disk_queue/module.go index f8996db3..961bb10f 100644 --- a/modules/queue/disk_queue/module.go +++ b/modules/queue/disk_queue/module.go @@ -390,7 +390,7 @@ func (module *DiskQueue) AcquireConsumer(qconfig *queue.QueueConfig, consumer *q q1 := q.(*DiskBasedQueue) if q1.writeSegmentNum == 0 && q1.writePos == 0 { //empty queue, no need to create consumer - return nil, errors.New("empty queue") + return nil, queue.ErrEmptyQueue } return q1.AcquireConsumer(qconfig, consumer, offset) } From 029fdc4707afcfca00ea5e4e33ae66984e3b0a67 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 10:58:46 +0800 Subject: [PATCH 27/65] feat(configs): per-instance config rows shadow group defaults Same-name rows resolve by specificity: per-instance rows (edge-rule variants) shadow group/catch-all rows for that instance; heartbeat keeps server-owned Alias/Description display metadata. --- modules/configs/config/config.go | 17 ++++++++++ modules/configs/config/config_test.go | 18 ++++++++++ modules/configs/server/server.go | 48 ++++++++++++++++++++++++--- modules/configs/server/server_test.go | 37 +++++++++++++++++++++ 4 files changed, 116 insertions(+), 4 deletions(-) diff --git a/modules/configs/config/config.go b/modules/configs/config/config.go index ca0b4dc8..af39787b 100755 --- a/modules/configs/config/config.go +++ b/modules/configs/config/config.go @@ -178,6 +178,17 @@ func deleteConfigAction(w http.ResponseWriter, req *http.Request, params httprou var versionRegexp = regexp.MustCompile(`#MANAGED_CONFIG_VERSION:\s*(?P\d+)`) var managedRegexp = regexp.MustCompile(`#MANAGED:\s*(?P\w+)`) +// managedMarkerLine matches whole trailer lines (`#MANAGED...`) so the +// content digest can exclude client-local metadata stamps. +var managedMarkerLine = regexp.MustCompile(`(?m)^\s*#MANAGED(_CONFIG_VERSION)?:[^\n]*$`) + +// stripManagedMarkers removes the managed trailer lines: the config server +// digests the raw row content, while the local file carries the version/ +// managed stamps the client appends — hashing the stamped bytes would make +// every managed file look permanently changed (rewrite loop each sync). +func stripManagedMarkers(input string) string { + return strings.TrimSpace(managedMarkerLine.ReplaceAllString(input, "")) +} func parseConfigVersion(input string) int64 { matches := versionRegexp.FindStringSubmatch(util.TrimSpaces(input)) @@ -313,6 +324,12 @@ func GetConfigFromFile(cfgDir, filename string) (*common.ConfigFile, error) { Updated: f.ModTime().Unix(), Size: f.Size(), } + // Content digest: the config server diffs by hash when present — + // per-row version counters aren't comparable across the same-name rows + // an instance may be assigned (instance variant vs group base). The + // digest must match the server's (raw row content): strip the managed + // trailer lines first. + cfg.Hash = util.MD5digest(stripManagedMarkers(content)) cfg.Content = content cfg.Managed = parseConfigManaged(content, cfg.Version > 0 || global.Env().SystemConfig.Configs.ConfigFileManagedByDefault) diff --git a/modules/configs/config/config_test.go b/modules/configs/config/config_test.go index 4e46f283..3d189095 100644 --- a/modules/configs/config/config_test.go +++ b/modules/configs/config/config_test.go @@ -77,3 +77,21 @@ func TestSaveConfigStr(t *testing.T) { t.Fatalf("Failed to remove test file: %v", err) } } + +// TestStripManagedMarkersSymmetry — 客户端剥离标记行后的哈希必须与服务端 +// (原始行内容 TrimSpace) 一致, 否则每个同步周期都判定内容变化、重写文件。 +func TestStripManagedMarkersSymmetry(t *testing.T) { + row := "{\"pipeline\":[{\"name\":\"x\"}]}" + stamped := row + "\n#MANAGED_CONFIG_VERSION: 15\n#MANAGED: true" + if got := stripManagedMarkers(stamped); got != row { + t.Fatalf("strip = %q, want %q", got, row) + } + // 标记粘行 (历史叠加): 整行剔除 + glued := row + "\n#MANAGED: trueMANAGED_CONFIG_VERSION: 15" + if got := stripManagedMarkers(glued); got != row { + t.Fatalf("strip glued = %q, want %q", got, row) + } + if stripManagedMarkers(row) != row { + t.Fatal("bare content must pass through") + } +} diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go index 592a1aad..2c18a253 100644 --- a/modules/configs/server/server.go +++ b/modules/configs/server/server.go @@ -472,6 +472,10 @@ func upsertInstance(instance *model.Instance) (bool, error) { // Groups are server-owned (UI-managed targeting): the heartbeat // payload does not carry them — keep the stored value. instanceCopy.Groups = existing.Groups + // Alias/Description are server-owned display metadata (UI-managed): + // keep the stored value across heartbeats. + instanceCopy.Alias = existing.Alias + instanceCopy.Description = existing.Description // Heartbeat syncs carry no access token; keep the registered one // (wiping it here would 401 the very next sync). if instanceCopy.AccessToken == nil { @@ -579,7 +583,8 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt // loadAssignedConfigs returns the server-side config files assigned to the // instance: its own + the "*" catch-all + any group-targeted config whose -// Groups intersect the instance's groups. Newest version per name. +// Groups intersect the instance's groups. Same-name rows resolve by +// specificity: per-instance rows shadow group/catch-all rows. // A query FAILURE returns an error — callers must NOT treat it as "no // configs": a transiently empty read (e.g. manager just booting, ORM not // ready) would diff every client file as Deleted and wipe managed configs @@ -630,15 +635,38 @@ func loadAssignedConfigs(instanceID string) ([]common.ConfigFile, error) { } } - out := make([]common.ConfigFile, 0, len(stored)) - for _, mc := range stored { + // Same-name resolution: a per-instance row (edge-rule variant) must + // SHADOW the group/catch-all rows of the same file for that instance — + // per-row version counters are not comparable across rows, so + // specificity decides and version only breaks ties within the same + // specificity. Without this the winner is query-order luck and an + // instance variant may never apply. + rank := func(mc *ManagedConfig) int { + if mc.InstanceID == instanceID { + return 2 // per-instance override + } + return 1 // "*" catch-all or group-targeted base + } + byName := make(map[string]ManagedConfig, len(stored)) + for i := range stored { + mc := stored[i] + cur, ok := byName[mc.Name] + if !ok || rank(&mc) > rank(&cur) || (rank(&mc) == rank(&cur) && mc.Version > cur.Version) { + byName[mc.Name] = mc + } + } + + out := make([]common.ConfigFile, 0, len(byName)) + for _, mc := range byName { out = append(out, common.ConfigFile{ Name: mc.Name, Location: mc.Location, Content: mc.Content, Version: mc.Version, Managed: true, - Hash: util.MD5digest(mc.Content), + // TrimSpace mirrors the client's digest (managed trailer + // lines stripped on the client side before hashing). + Hash: util.MD5digest(strings.TrimSpace(mc.Content)), Size: int64(len(mc.Content)), Updated: time.Now().UnixMilli(), }) @@ -696,6 +724,18 @@ func diffConfigs(assigned []common.ConfigFile, clientConfigs map[string]common.C if !cc.Managed { continue // client opted this file out of management } + // Content-hash comparison when the client reports one (newer + // clients): the assigned row may be a DIFFERENT row than the one + // the client's file came from (instance variant vs group base), + // and per-row version counters cannot be compared across rows — + // the hash says exactly whether the bytes differ. Version + // comparison remains the fallback for old clients. + if sc.Hash != "" && cc.Hash != "" { + if sc.Hash != cc.Hash { + resp.Configs.UpdatedConfigs[name] = sc + } + continue + } if sc.Version > cc.Version { resp.Configs.UpdatedConfigs[name] = sc } diff --git a/modules/configs/server/server_test.go b/modules/configs/server/server_test.go index 65a13864..2834ee4a 100644 --- a/modules/configs/server/server_test.go +++ b/modules/configs/server/server_test.go @@ -225,3 +225,40 @@ func TestHashTokenOneWay(t *testing.T) { t.Fatal("different inputs must hash differently") } } + +// TestDiffConfigs_HashAware — 内容哈希比较 (新客户端): 跨行版本号不可比 +// (实例变体 v1 vs 客户端持有的基础行 v12), 哈希不同即更新; 哈希相同则即使 +// 版本号更小也不重发。旧客户端 (无哈希) 回退版本比较。 +func TestDiffConfigs_HashAware(t *testing.T) { + base := cfg("s.yml", 12) + base.Hash = "hash-base" + variant := cfg("s.yml", 1) + variant.Hash = "hash-variant" + + t.Run("variant applied over higher-versioned base content", func(t *testing.T) { + client := map[string]common.ConfigFile{"s.yml": base} + resp := diffConfigs([]common.ConfigFile{variant}, client) + if !resp.Changed || resp.Configs.UpdatedConfigs["s.yml"].Hash != "hash-variant" { + t.Fatalf("variant must win by hash, got %+v", resp.Configs.UpdatedConfigs) + } + }) + + t.Run("identical hash, lower version → no churn", func(t *testing.T) { + same := cfg("s.yml", 3) + same.Hash = "hash-base" + client := map[string]common.ConfigFile{"s.yml": base} + resp := diffConfigs([]common.ConfigFile{same}, client) + if resp.Changed { + t.Fatal("same content must not re-send") + } + }) + + t.Run("old client without hash falls back to version", func(t *testing.T) { + oldClient := cfg("s.yml", 12) // no Hash + client := map[string]common.ConfigFile{"s.yml": oldClient} + resp := diffConfigs([]common.ConfigFile{variant}, client) + if resp.Changed { + t.Fatal("version 1 must not beat 12 for hashless clients (logpilot bumps row versions instead)") + } + }) +} From efcb929b23cb7fc933443848ef6b6a2622424f98 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 10:58:46 +0800 Subject: [PATCH 28/65] feat(pipeline): lane-based bulk indexing processors New memory-path bulk pipeline: merge_to_bulk_lane merges records onto a lane; bulk_indexing_lane subscribes to the merged-bulk lane and ships each payload to Elasticsearch with batch-level receipt settle (the memory-path twin of bulk_indexing). Lane consumer lifecycle adjusted for restart; conditionals inside per-record sub-chains (for_each, direct-ship) now see the current record's fields; indexing_merge aligned with the lane path. --- core/pipeline/conditionals.go | 11 +- modules/pipeline/empty_config_audit_test.go | 1 + modules/pipeline/lane_consumer.go | 18 +- modules/pipeline/tasks.go | 7 + .../elastic/indexing_merge/indexing_merge.go | 20 +- .../indexing_merge/indexing_merge_test.go | 39 +- .../elastic/lane_bulk/bulk_indexing_lane.go | 313 +++++++++ plugins/elastic/lane_bulk/lane_bulk_test.go | 638 ++++++++++++++++++ .../elastic/lane_bulk/merge_to_bulk_lane.go | 521 ++++++++++++++ plugins/elastic/lane_bulk/payload.go | 121 ++++ 10 files changed, 1665 insertions(+), 24 deletions(-) create mode 100644 plugins/elastic/lane_bulk/bulk_indexing_lane.go create mode 100644 plugins/elastic/lane_bulk/lane_bulk_test.go create mode 100644 plugins/elastic/lane_bulk/merge_to_bulk_lane.go create mode 100644 plugins/elastic/lane_bulk/payload.go diff --git a/core/pipeline/conditionals.go b/core/pipeline/conditionals.go index 689a8a33..4aac4f48 100644 --- a/core/pipeline/conditionals.go +++ b/core/pipeline/conditionals.go @@ -92,7 +92,16 @@ func (r WhenProcessor) Process(ctx *Context) error { return nil } - if !(r.condition).Check(ctx) { + // Inside a per-record sub-chain (for_each, direct-ship chains) the + // condition must see the current record's fields — the same fallback + // IfThenElseProcessor applies; pipeline-level usage keeps reading the + // parameters. + values := conditions.ValuesMap(ctx) + if rec, ok := CurrentRecord(ctx); ok { + values = rec + } + + if !(r.condition).Check(values) { ctx.AddFlowProcess(r.p.Name() + "-skipped") return nil } diff --git a/modules/pipeline/empty_config_audit_test.go b/modules/pipeline/empty_config_audit_test.go index 1c92c9ee..027e99e1 100644 --- a/modules/pipeline/empty_config_audit_test.go +++ b/modules/pipeline/empty_config_audit_test.go @@ -16,6 +16,7 @@ import ( _ "infini.sh/framework/plugins/elastic/bulk_indexing" _ "infini.sh/framework/plugins/elastic/indexing_merge" _ "infini.sh/framework/plugins/elastic/json_indexing" + _ "infini.sh/framework/plugins/elastic/lane_bulk" _ "infini.sh/framework/plugins/elastic/merge_to_bulk" _ "infini.sh/framework/plugins/enterprise/otlp/export" _ "infini.sh/framework/plugins/enterprise/processors/decode_duration" diff --git a/modules/pipeline/lane_consumer.go b/modules/pipeline/lane_consumer.go index 4fe7d735..3386421f 100644 --- a/modules/pipeline/lane_consumer.go +++ b/modules/pipeline/lane_consumer.go @@ -69,10 +69,13 @@ func NewLaneConsumerProcessor(c *config.Config) (pipeline.Processor, error) { if cfg.FailureTag == "" { cfg.FailureTag = "_processing_failed" } - l := lane.Get(cfg.Lane) - if l == nil { - return nil, fmt.Errorf("%s: lane %q not found (define it under lane: or create it dynamically)", laneConsumerName, cfg.Lane) - } + // GetOrCreate (not Get+error): a lane_consumer task restart races its + // producers (output{lane} / consumer{output_lane}) re-creating the lane + // after this task's previous instance deleted it on exit — a hard "not + // found" here would leave the pipeline dead until the next producer + // emit. An explicit `lane:` stanza still wins when present; only the + // absent/closed case falls back to a default-config lane. + l := lane.GetOrCreate(lane.LaneConfig{Name: cfg.Lane}) sub, err := pipeline.NewPipelineWithScope(cfg.Processors, pipeline.ScopeInfo{ Kind: pipeline.ScopeRecord, Namespace: l.Config().Namespace, @@ -101,7 +104,12 @@ func (p *LaneConsumerProcessor) Process(ctx *pipeline.Context) error { } _ = p.lane.SubscribeBatch(p.cfg.BatchMax, time.Duration(p.cfg.BatchFlushMs)*time.Millisecond, batchFn) <-ctx.Done() - p.lane.Close() + // Delete (close + unregister), not bare Close: a closed-but-registered + // lane accepts nothing forever and wedges every producer, and this + // task's own restart would re-bind the dead instance. Delete keeps the + // Close drain semantics (buffered batch flushes, receipts settle) and + // lets GetOrCreate re-create the lane on the next cycle. + _ = lane.Delete(p.cfg.Lane, true) return nil } diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index 184deedd..c1824dac 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -27,6 +27,7 @@ import ( "encoding/json" "net/http" "strconv" + "time" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" @@ -171,6 +172,12 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req * id := ps.ByName("id") _, exists := module.contexts.Load(id) if exists { + // 先取消再删表: 管线 goroutine 阻塞在 Process 内部 (consumer 工 + // 作循环等 IsCanceled / lane 处理器等 ctx.Done), 只清映射表不取消 + // 会让 goroutine 永远活着 — 任务注册表停留在 STARTED (僵尸"运行 + // 中")。与 auto_reload 路径同款: 取消并等释放 (≤1min), 超时也强制 + // 删表。 + module.stopAndWaitForRelease([]string{id}, time.Minute) module.deleteTask(id) module.WriteAckOKJSON(w) } else { diff --git a/plugins/elastic/indexing_merge/indexing_merge.go b/plugins/elastic/indexing_merge/indexing_merge.go index 3b79a582..3c8eea16 100644 --- a/plugins/elastic/indexing_merge/indexing_merge.go +++ b/plugins/elastic/indexing_merge/indexing_merge.go @@ -371,9 +371,13 @@ CLEAN_BUFFER: // first-class doc fields) // - a top-level @timestamp is derived from payload.timestamp / // payload.observed_timestamp / the envelope timestamp / now — data -// streams reject documents without a date @timestamp -// - timestamp mirrors @timestamp (conventional sort field) -// - metadata.file (agent collection origin) is kept as top-level "file" +// streams reject documents without a date @timestamp. @timestamp is the +// document's one true timestamp (house standard): the mirrored top-level +// "timestamp" and any payload "timestamp" are folded into it and removed. +// - a bare document (no "payload" key) keeps its own @timestamp when +// present; only a missing one is stamped with now. +// - metadata.file (agent collection origin) lives in @agent.file_* — the +// former top-level "file" promotion is gone (zero readers). // // Documents that are not envelopes (no "payload" key) pass through // unchanged except for the @timestamp stamp. @@ -397,6 +401,11 @@ func normalizeDataStreamDoc(doc []byte) []byte { ts = v } } + if ts == "" { + if v, ok := m["@timestamp"].(string); ok { // bare doc already carrying one + ts = v + } + } if ts == "" { ts = time.Now().UTC().Format(time.RFC3339Nano) } @@ -412,12 +421,9 @@ func normalizeDataStreamDoc(doc []byte) []byte { } } out["@timestamp"] = ts - out["timestamp"] = ts + delete(out, "timestamp") // @timestamp is the single source of truth if meta, ok := m["metadata"].(map[string]interface{}); ok { - if f, ok := meta["file"].(map[string]interface{}); ok { - out["file"] = f - } if rt, ok := meta["log_type"].(string); ok && rt != "" { out["log_type"] = rt } diff --git a/plugins/elastic/indexing_merge/indexing_merge_test.go b/plugins/elastic/indexing_merge/indexing_merge_test.go index cc7ab327..12da9df4 100644 --- a/plugins/elastic/indexing_merge/indexing_merge_test.go +++ b/plugins/elastic/indexing_merge/indexing_merge_test.go @@ -31,16 +31,19 @@ func TestNormalizeDataStreamDoc(t *testing.T) { t.Fatalf("payload key should not survive: %v", doc) } - // @timestamp: payload 无显式 timestamp 时 observed_timestamp 优先于信封级; - // timestamp 镜像之。 - if doc["@timestamp"] != "2026-08-23T14:38:32.982744Z" || doc["timestamp"] != "2026-08-23T14:38:32.982744Z" { - t.Fatalf("timestamp fields: @=%v ts=%v", doc["@timestamp"], doc["timestamp"]) + // @timestamp: payload 无显式 timestamp 时 observed_timestamp 优先于信封级。 + // @timestamp 是唯一真实时间戳 — legacy "timestamp" mirror 不再写出。 + if doc["@timestamp"] != "2026-08-23T14:38:32.982744Z" { + t.Fatalf("@timestamp = %v", doc["@timestamp"]) + } + if _, ok := doc["timestamp"]; ok { + t.Fatalf("timestamp mirror must not be written: %v", doc["timestamp"]) } - // metadata promotion. - file, ok := doc["file"].(map[string]interface{}) - if !ok || file["path"] != "/var/log/system.log" { - t.Fatalf("file metadata not promoted: %v", doc["file"]) + // metadata promotion: file origin 不再提升为顶层 "file" (采集溯源由 + // @agent.file_* 承载), 仅 log_type/log_kind 标签提升。 + if _, ok := doc["file"]; ok { + t.Fatalf("top-level file promotion is gone: %v", doc["file"]) } if doc["log_type"] != "text" || doc["log_kind"] != "server" { t.Fatalf("label fields not promoted: %v", doc) @@ -64,11 +67,14 @@ func TestNormalizeDataStreamDocTimestampPriority(t *testing.T) { if doc["@timestamp"] != tc.want { t.Fatalf("%s: @timestamp = %v, want %v", tc.name, doc["@timestamp"], tc.want) } + if _, ok := doc["timestamp"]; ok { + t.Fatalf("%s: payload/envelope timestamp must fold into @timestamp only: %v", tc.name, doc) + } } } // TestNormalizeDataStreamDocBareDoc: non-envelope docs keep their fields -// and only get the @timestamp stamp. +// and only get the @timestamp stamp (single truth — no mirror). func TestNormalizeDataStreamDocBareDoc(t *testing.T) { out := normalizeDataStreamDoc([]byte(`{"message":"raw","level":"info"}`)) var doc util.MapStr @@ -76,8 +82,19 @@ func TestNormalizeDataStreamDocBareDoc(t *testing.T) { if doc["message"] != "raw" || doc["level"] != "info" { t.Fatalf("bare doc fields lost: %v", doc) } - if doc["timestamp"] == "" || doc["@timestamp"] == "" { - t.Fatalf("bare doc missing timestamp stamps: %v", doc) + if doc["@timestamp"] == "" { + t.Fatalf("bare doc missing @timestamp: %v", doc) + } + if _, ok := doc["timestamp"]; ok { + t.Fatalf("bare doc must not gain a timestamp mirror: %v", doc) + } + + // bare doc already carrying @timestamp keeps it (no re-stamp) + keep := normalizeDataStreamDoc([]byte(`{"message":"kept","@timestamp":"2026-05-05T00:00:00Z"}`)) + var kd util.MapStr + _ = json.Unmarshal(keep, &kd) + if kd["@timestamp"] != "2026-05-05T00:00:00Z" { + t.Fatalf("bare doc @timestamp must be kept: %v", kd["@timestamp"]) } } diff --git a/plugins/elastic/lane_bulk/bulk_indexing_lane.go b/plugins/elastic/lane_bulk/bulk_indexing_lane.go new file mode 100644 index 00000000..303243f3 --- /dev/null +++ b/plugins/elastic/lane_bulk/bulk_indexing_lane.go @@ -0,0 +1,313 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello#infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +package lane_bulk + +import ( + "fmt" + "sync" + "time" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/event" + "infini.sh/framework/core/global" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/stats" + "infini.sh/framework/core/util" +) + +// BulkIndexingLaneProcessor is the memory-path twin of bulk_indexing: it +// subscribes to a lane of merged bulk records (merge_to_bulk_lane's output) +// and sends each payload to Elasticsearch. Receipts of a batch settle when +// its sends complete (the SubscribeBatch settlement contract), which is +// also the settlement point for the merge side's EmitAck receipts. +// +// Failure policy: a failed send retries after retry_delay_in_ms. Without a +// failure_queue retries continue indefinitely — a down cluster stalls the +// lane, which backpressures the merge, which stalls input queue consumption +// (no silent loss, matching legacy bulk_indexing's stall semantics). With a +// failure_queue set, a payload diverts there after max_retries attempts and +// the lane keeps flowing. +type BulkIndexingLaneProcessor struct { + config SenderConfig + inputLane lane.Lane + failureQ *queue.QueueConfig + bulkProcessor elastic.BulkProcessor + bufferPool *elastic.BulkBufferPool +} + +func (processor *BulkIndexingLaneProcessor) Name() string { + return "bulk_indexing_lane" +} + +type SenderConfig struct { + Lane string `config:"lane"` + Elasticsearch string `config:"elasticsearch"` + + Bulk elastic.BulkProcessorConfig `config:"bulk"` + + FailureQueue string `config:"failure_queue"` + MaxRetries int `config:"max_retries"` + RetryDelayInMs int `config:"retry_delay_in_ms"` + + // WorkerSize sends one batch's payloads concurrently across workers; + // the batch function waits for every send before returning, keeping + // receipt settlement exact. + WorkerSize int `config:"worker_size"` + + // BatchFlushMs flushes a partial batch after this idle window. Must + // be > 0: without a timer, a batch below batch_max never reaches its + // size trigger and its records would sit undelivered. + BatchFlushMs int `config:"batch_flush_ms"` +} + +func init() { + pipeline.RegisterProcessorPluginWithConfigMetadata("bulk_indexing_lane", NewBulkIndexingLane, SenderConfig{}) +} + +func NewBulkIndexingLane(c *config.Config) (pipeline.Processor, error) { + cfg := SenderConfig{ + RetryDelayInMs: 3000, + WorkerSize: 1, + BatchFlushMs: 100, + } + + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of bulk_indexing_lane processor: %s", err) + } + + if cfg.Lane == "" { + return nil, errors.New("bulk_indexing_lane processor requires lane") + } + if cfg.Elasticsearch == "" { + return nil, errors.New("bulk_indexing_lane processor requires elasticsearch") + } + if cfg.RetryDelayInMs <= 0 { + cfg.RetryDelayInMs = 3000 + } + if cfg.WorkerSize <= 0 { + cfg.WorkerSize = 1 + } + if cfg.BatchFlushMs <= 0 { + cfg.BatchFlushMs = 100 + } + if cfg.FailureQueue != "" && cfg.MaxRetries <= 0 { + // a failure queue without a retry budget diverts on first failure; + // make the budget explicit instead of guessing + return nil, errors.New("bulk_indexing_lane processor requires max_retries > 0 when failure_queue is set") + } + + // GetOrCreate (not Get+error): tolerate the merge task not having run + // yet (startup ordering), and re-create the lane when a previous task + // instance closed it on exit — a bare Get would keep returning the + // closed instance and the task would silently idle forever. Capacity + // matches merge_to_bulk_lane's auto-create budget (merged bulks are + // large; an explicit `lane:` stanza overrides when present). + inputLane := lane.GetOrCreate(lane.LaneConfig{Name: cfg.Lane, Capacity: autoLaneCapacity}) + + var failureQ *queue.QueueConfig + if cfg.FailureQueue != "" { + failureQ = queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.FailureQueue}) + } + + return &BulkIndexingLaneProcessor{ + config: cfg, + inputLane: inputLane, + failureQ: failureQ, + bulkProcessor: elastic.NewBulkProcessor("bulk_indexing_lane", cfg.Elasticsearch, cfg.Bulk), + bufferPool: elastic.NewBulkBufferPool("bulk_indexing_lane", 1024*1024*1024, 100000), + }, nil +} + +// Process subscribes (once) and blocks until the pipeline task cancels — +// the host drives the lane for its whole lifetime. On exit the lane is +// DELETED (closed + unregistered), not merely closed: Close alone leaves a +// permanently-closed instance in the registry that wedges the producer +// (merge_to_bulk_lane's emits fail "closed" forever) and breaks this task's +// own restart (a re-Get would return the dead instance and the subscriber +// would exit immediately). Delete + the producers'/consumers' GetOrCreate +// re-creation makes a task stop/restart a recoverable cycle: the drain +// semantics of Close are preserved (buffered records flush through the +// batch function, in-flight receipts settle). +func (processor *BulkIndexingLaneProcessor) Process(ctx *pipeline.Context) error { + meta := processor.waitMetadata(ctx) + if meta == nil { + return nil + } + + // batch width tracks the worker fan-out: batch_max payloads are picked + // up per flush (size trigger, or batch_flush_ms idle trigger — lone + // records below batch_max rely on the timer) and sent + // worker_size-at-a-time + batchMax := processor.config.WorkerSize * 2 + _ = processor.inputLane.SubscribeBatch(batchMax, time.Duration(processor.config.BatchFlushMs)*time.Millisecond, func(recs []*event.Event) { + processor.sendBatch(ctx, meta, recs) + }) + + <-ctx.Done() + _ = lane.Delete(processor.config.Lane, true) + return nil +} + +// sendBatch delivers one micro-batch: records are dealt across workers, +// every send completes (or diverts) before returning — receipts settle only +// after that (SubscribeBatch contract), so settlement means "delivered or +// salvaged", never "still pending". +func (processor *BulkIndexingLaneProcessor) sendBatch(ctx *pipeline.Context, meta *elastic.ElasticsearchMetadata, recs []*event.Event) { + workers := processor.config.WorkerSize + if workers > len(recs) { + workers = len(recs) + } + if workers <= 1 { + for _, rec := range recs { + processor.sendRecord(ctx, meta, rec) + } + return + } + + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(chunk []*event.Event) { + defer wg.Done() + for _, rec := range chunk { + processor.sendRecord(ctx, meta, rec) + } + }(recs[w:]) + } + wg.Wait() +} + +// waitMetadata blocks until the cluster metadata is available (startup +// ordering: the elastic module may finish after pipeline tasks) or the task +// cancels. Stalling here is safe — nothing has been consumed yet. +func (processor *BulkIndexingLaneProcessor) waitMetadata(ctx *pipeline.Context) *elastic.ElasticsearchMetadata { + for { + if ctx.IsCanceled() { + return nil + } + if meta := elastic.GetMetadata(processor.config.Elasticsearch); meta != nil { + return meta + } + log.Warnf("bulk_indexing_lane: cluster metadata [%v] not ready, waiting", processor.config.Elasticsearch) + select { + case <-time.After(2 * time.Second): + case <-ctx.Done(): + return nil + } + } +} + +// sendRecord delivers one merged bulk payload with the configured retry +// policy. On task cancellation each in-flight record gets ONE final attempt +// (the legacy final-submit behavior) and unresolved payloads divert to the +// failure queue when one is configured. +func (processor *BulkIndexingLaneProcessor) sendRecord(ctx *pipeline.Context, meta *elastic.ElasticsearchMetadata, rec *event.Event) { + bulk, ok := bulkPayloadOf(rec) + if !ok { + log.Warnf("bulk_indexing_lane: record without bulk payload (trace %q), skip", pipeline.TraceID(rec)) + stats.Increment("bulk_indexing_lane", "skipped") + return + } + + id := pipeline.TraceID(rec) + if id == "" { + id = "bulk-" + util.GetUUID() + } + + // shutdown drain: one best-effort attempt, then salvage — retrying + // forever here would hang the task teardown + final := ctx.IsCanceled() || global.ShuttingDown() + + for attempt := 0; ; attempt++ { + if !final && (ctx.IsCanceled() || global.ShuttingDown()) { + final = true + } + + host := meta.GetActivePreferredSeedHost() + if elastic.IsHostDead(host) || !elastic.IsHostAvailable(host) { + host = meta.GetActiveHost() + } + + buf := processor.bufferPool.AcquireBulkBuffer() + buf.Queue = processor.config.Elasticsearch + buf.WriteMessageID(id) + buf.Write(bulk) + buf.SafetyEndWithNewline() + + continueNext, statsRet, _, err := processor.bulkProcessor.Bulk(ctx.Context, "bulk_indexing_lane", meta, host, buf) + buf.ResetData() + processor.bufferPool.ReturnBulkBuffer(buf) + + for code, n := range statsRet { + stats.IncrementBy("bulk_indexing_lane", "response_code_"+util.ToString(code), int64(n)) + } + + if err == nil { + stats.Increment("bulk_indexing_lane", "sent") + stats.IncrementBy("bulk_indexing_lane", "sent_bytes", int64(len(bulk))) + if !continueNext { + log.Warnf("bulk_indexing_lane: cluster suggested backoff (continue=false)") + } + return + } + + stats.Increment("bulk_indexing_lane", "failed") + log.Errorf("bulk_indexing_lane: bulk send failed (host: %v, attempt: %v, err: %v)", host, attempt+1, err) + + // only a configured failure queue ends the retry loop early; + // without one a hard-down cluster stalls the lane (intended) + if final || (processor.failureQ != nil && attempt+1 >= processor.config.MaxRetries) { + break + } + + select { + case <-time.After(time.Duration(processor.config.RetryDelayInMs) * time.Millisecond): + case <-ctx.Done(): + continue // loop top re-checks cancellation and flags final + } + } + + if processor.failureQ != nil { + if err := queue.Push(processor.failureQ, bulk); err != nil { + log.Errorf("bulk_indexing_lane: push to failure queue %q failed: %v", processor.failureQ.Name, err) + } else { + stats.Increment("bulk_indexing_lane", "diverted") + } + return + } + if ctx.IsCanceled() || global.ShuttingDown() { + // no salvage path configured on the shutdown drain — visible in + // stats rather than silent + stats.Increment("bulk_indexing_lane", "lost_on_shutdown") + } +} diff --git a/plugins/elastic/lane_bulk/lane_bulk_test.go b/plugins/elastic/lane_bulk/lane_bulk_test.go new file mode 100644 index 00000000..5a1f38bc --- /dev/null +++ b/plugins/elastic/lane_bulk/lane_bulk_test.go @@ -0,0 +1,638 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello#infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +package lane_bulk + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// ─── fake in-memory queue backend ──────────────────────────────────────────── + +type memQueue struct { + mu sync.Mutex + data map[string][][]byte +} + +func (q *memQueue) Name() string { return "mem" } +func (q *memQueue) Init(string) error { + q.mu.Lock() + defer q.mu.Unlock() + if q.data == nil { + q.data = map[string][][]byte{} + } + return nil +} +func (q *memQueue) Close(string) error { return nil } +func (q *memQueue) Destroy(string) error { return nil } +func (q *memQueue) GetStorageSize(string) uint64 { return 0 } +func (q *memQueue) GetQueues() []string { return nil } + +func (q *memQueue) Push(id string, v []byte) error { + q.mu.Lock() + defer q.mu.Unlock() + if q.data == nil { + q.data = map[string][][]byte{} + } + q.data[id] = append(q.data[id], v) + return nil +} + +func (q *memQueue) Pop(id string, timeout time.Duration) ([]byte, bool) { + q.mu.Lock() + if items := q.data[id]; len(items) > 0 { + v := items[0] + q.data[id] = items[1:] + q.mu.Unlock() + return v, false + } + q.mu.Unlock() + d := timeout + if d > 200*time.Millisecond { + d = 200 * time.Millisecond + } + time.Sleep(d) + return nil, true +} + +func (q *memQueue) Depth(id string) int64 { + q.mu.Lock() + defer q.mu.Unlock() + return int64(len(q.data[id])) +} + +var memQueueOnce sync.Once + +// memKV backs the queue-config registry in unit tests (GetOrInitConfig +// persists configs through the kv store; without a handler it panics). +type memKV struct { + data map[string][]byte +} + +func (m *memKV) Open() error { return nil } +func (m *memKV) Close() error { return nil } +func (m *memKV) GetValue(bucket string, key []byte) ([]byte, error) { + if v, ok := m.data[string(key)]; ok { + return append([]byte(nil), v...), nil + } + return nil, nil +} +func (m *memKV) GetCompressedValue(bucket string, key []byte) ([]byte, error) { + return m.GetValue(bucket, key) +} +func (m *memKV) AddValueCompress(bucket string, key, value []byte) error { + return m.AddValue(bucket, key, value) +} +func (m *memKV) AddValue(bucket string, key, value []byte) error { + m.data[string(key)] = append([]byte(nil), value...) + return nil +} +func (m *memKV) ExistsKey(bucket string, key []byte) (bool, error) { + _, ok := m.data[string(key)] + return ok, nil +} +func (m *memKV) DeleteKey(bucket string, key []byte) error { + delete(m.data, string(key)) + return nil +} + +// installMemQueue wires the fake queue backend and kv store so +// GetOrInitConfig-created queues (empty type) resolve end to end. +func installMemQueue() *memQueue { + q := &memQueue{} + memQueueOnce.Do(func() { + queue.RegisterDefaultHandler(q) + kv.Register("lane-bulk-test-kv", &memKV{data: map[string][]byte{}}) + }) + return q +} + +// ─── payload codec ─────────────────────────────────────────────────────────── + +func TestBulkPayloadCodecRoundTrip(t *testing.T) { + payload := []byte("{\"index\":{}}\n{\"a\":1}\n") + + rec, err := bulkPayloadCodec{}.Decode(payload) + if err != nil { + t.Fatalf("decode: %v", err) + } + if got := pipeline.CodecHint(rec); got != "bulk" { + t.Errorf("codec hint = %q, want bulk", got) + } + out, err := bulkPayloadCodec{}.Encode(rec) + if err != nil { + t.Fatalf("encode: %v", err) + } + if string(out) != string(payload) { + t.Errorf("round-trip mismatch:\n got: %q\nwant: %q", out, payload) + } + + if _, err := (bulkPayloadCodec{}).Decode(nil); err == nil { + t.Error("empty payload should error") + } + bare := &event.Event{Fields: util.MapStr{"not_bulk": 1}} + if _, err := (bulkPayloadCodec{}).Encode(bare); err == nil { + t.Error("record without payload field should error") + } +} + +// ─── merge formatting helpers ──────────────────────────────────────────────── + +func TestFingerprintID(t *testing.T) { + p := &MergeToBulkLaneProcessor{config: MergeConfig{KeyFields: []string{"file.path", "file.offset"}}} + + doc := []byte(`{"file":{"path":"/var/log/a.log","offset":42},"message":"x"}`) + + a := p.fingerprintID(doc) + b := p.fingerprintID(doc) + if a != b { + t.Errorf("fingerprint not deterministic: %v vs %v", a, b) + } + + // a different offset must yield a different id + doc2 := []byte(`{"file":{"path":"/var/log/a.log","offset":43},"message":"x"}`) + if p.fingerprintID(doc2) == a { + t.Error("offset change should change the fingerprint") + } + + // no key field matches: content-hash fallback, distinct docs stay distinct + c1 := p.fingerprintID([]byte(`{"message":"one"}`)) + c2 := p.fingerprintID([]byte(`{"message":"two"}`)) + if c1 == c2 { + t.Error("content fallback collapsed distinct documents") + } +} + +func TestTrimJSONString(t *testing.T) { + if s, ok := trimJSONString([]byte(`"hello"`)); !ok || s != "hello" { + t.Errorf("quoted token: %q %v", s, ok) + } + if _, ok := trimJSONString([]byte(`42`)); ok { + t.Error("non-string token should not trim") + } +} + +func TestNormalizeDataStreamDoc(t *testing.T) { + env := []byte(`{"metadata":{"file":{"path":"/a.log"}},"payload":{"message":"hi","timestamp":"2026-10-15T00:00:00Z"},"timestamp":"2026-10-15T00:00:00Z"}`) + + out := normalizeDataStreamDoc(env) + m := util.MapStr{} + if err := util.FromJSONBytes(out, &m); err != nil { + t.Fatalf("normalized doc is not JSON: %v", err) + } + if m["message"] != "hi" { + t.Errorf("payload.message not promoted: %v", m) + } + if m["@timestamp"] != "2026-10-15T00:00:00Z" { + t.Errorf("@timestamp = %v", m["@timestamp"]) + } + // @timestamp 是唯一真实时间戳: payload 里的 timestamp 折叠进 + // @timestamp 后删除, 不再写顶层 mirror。 + if _, ok := m["timestamp"]; ok { + t.Errorf("timestamp mirror must not survive: %v", m["timestamp"]) + } + // file origin 不再提升为顶层 "file" (采集溯源由 @agent.file_* 承载) + if _, ok := m["file"]; ok { + t.Errorf("top-level file promotion is gone: %v", m["file"]) + } + + // bare docs pass through with a stamped @timestamp + bare := normalizeDataStreamDoc([]byte(`{"message":"plain"}`)) + m2 := util.MapStr{} + if err := util.FromJSONBytes(bare, &m2); err != nil { + t.Fatalf("bare doc: %v", err) + } + if m2["message"] != "plain" { + t.Errorf("bare doc message lost: %v", m2) + } + if m2["@timestamp"] == nil { + t.Error("bare doc missing stamped @timestamp") + } +} + +// ─── config validation ─────────────────────────────────────────────────────── + +func TestNewMergeValidation(t *testing.T) { + cases := []struct { + name string + cfg map[string]interface{} + errMsg string + }{ + {"empty", map[string]interface{}{}, "exactly one of input_queue | input_lane"}, + {"both", map[string]interface{}{"input_queue": "q", "input_lane": "l"}, "exactly one of input_queue | input_lane"}, + {"no lane", map[string]interface{}{"input_queue": "q"}, "requires output_lane"}, + {"no index", map[string]interface{}{"input_queue": "q", "output_lane": "l"}, "requires index_name"}, + } + for _, c := range cases { + cfg, err := config.NewConfigFrom(c.cfg) + if err != nil { + t.Fatalf("%s: build config: %v", c.name, err) + } + _, err = NewMergeToBulkLane(cfg) + if err == nil || !strings.Contains(err.Error(), c.errMsg) { + t.Errorf("%s: err = %v, want contains %q", c.name, err, c.errMsg) + } + } +} + +func TestNewSenderValidation(t *testing.T) { + cfg, err := config.NewConfigFrom(map[string]interface{}{}) + if err != nil { + t.Fatalf("build config: %v", err) + } + if _, err := NewBulkIndexingLane(cfg); err == nil || !strings.Contains(err.Error(), "requires lane") { + t.Errorf("empty config: err = %v", err) + } + + // failure_queue without max_retries is rejected + cfg2, _ := config.NewConfigFrom(map[string]interface{}{ + "lane": "missing", "elasticsearch": "es", "failure_queue": "dead", + }) + if _, err := NewBulkIndexingLane(cfg2); err == nil || !strings.Contains(err.Error(), "max_retries") { + t.Errorf("failure queue without budget: err = %v", err) + } + + // a missing lane is auto-created: the old "not found" rejection wedged + // task restarts (consumer exits delete the lane; a restart racing its + // producers had nothing to bind and the pipeline died) + cfg3, _ := config.NewConfigFrom(map[string]interface{}{ + "lane": "test-sender-auto-lane", "elasticsearch": "es", + }) + if _, err := NewBulkIndexingLane(cfg3); err != nil { + t.Errorf("auto-create lane: err = %v", err) + } + if l := lane.Get("test-sender-auto-lane"); l == nil { + t.Error("auto-created lane not registered") + } + _ = lane.Delete("test-sender-auto-lane", true) +} + +// ─── merge end-to-end over a lane ──────────────────────────────────────────── + +func TestMergeToBulkLaneEndToEnd(t *testing.T) { + installMemQueue() + + const laneName = "test-merge-lane" + const queueName = "test-merge-queue" + l, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 16}) + if err != nil { + t.Fatalf("create lane: %v", err) + } + defer lane.Delete(laneName, true) + + qcfg := queue.GetOrInitConfig(queueName) + docs := []string{ + `{"id":"1","message":"alpha"}`, + `{"id":"2","message":"beta"}`, + `{"id":"3","message":"gamma"}`, + } + for _, d := range docs { + if err := queue.Push(qcfg, []byte(d)); err != nil { + t.Fatalf("push: %v", err) + } + } + + cfg, err := config.NewConfigFrom(map[string]interface{}{ + "input_queue": queueName, + "output_lane": laneName, + "index_name": "test-logs", + "key_field": "id", + "idle_timeout_in_seconds": 1, + "bulk_size_in_kb": 1, + }) + if err != nil { + t.Fatalf("build config: %v", err) + } + proc, err := NewMergeToBulkLane(cfg) + if err != nil { + t.Fatalf("new processor: %v", err) + } + + stdCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + pctx := &pipeline.Context{Context: stdCtx} + + type delivery struct { + payload string + traceID string + codec string + } + bulks := make(chan delivery, 8) + l.SubscribeBatch(4, 50*time.Millisecond, func(recs []*event.Event) { + for _, rec := range recs { + if b, ok := bulkPayloadOf(rec); ok { + bulks <- delivery{payload: string(b), traceID: pipeline.TraceID(rec), codec: pipeline.CodecHint(rec)} + } + } + }) + + processDone := make(chan error, 1) + go func() { processDone <- proc.Process(pctx) }() + + var combined string + var first delivery + deadline := time.After(10 * time.Second) +collect: + for { + select { + case d := <-bulks: + if first.payload == "" { + first = d + } + combined += d.payload + if strings.Count(combined, `"message"`) >= len(docs) { + break collect + } + case <-deadline: + t.Fatalf("timed out waiting for merged bulk, got: %q", combined) + } + } + + // action lines target the right index and carry the key_field-derived ids + if !strings.Contains(combined, `"_index" : "test-logs"`) { + t.Errorf("action line missing index:\n%s", combined) + } + for _, id := range []string{`"_id":"1"`, `"_id":"2"`, `"_id":"3"`} { + if !strings.Contains(combined, id) { + t.Errorf("action line missing %s:\n%s", id, combined) + } + } + // every doc body survived the merge + for _, d := range docs { + if !strings.Contains(combined, d) { + t.Errorf("doc body missing from bulk:\n%s\nwant: %s", combined, d) + } + } + // record metadata: codec hint routes spill encoding, trace id correlates + if first.codec != "bulk" { + t.Errorf("codec hint = %q, want bulk", first.codec) + } + if first.traceID == "" { + t.Error("merged record missing trace id") + } + + // cancellation stops the merge host + cancel() + select { + case <-processDone: + case <-time.After(5 * time.Second): + t.Fatal("Process did not return after cancel") + } + + // nothing hit the salvage path: the failure queue stays empty + if d := queue.Depth(queue.GetOrInitConfig(queueName + "_failure")); d != 0 { + t.Errorf("failure queue depth = %d, want 0", d) + } +} + +// ─── consumer stop/restart cycle over the lane ────────────────────────────── + +// TestBulkLaneSurvivesConsumerRestart pins the gateway wedge fix: when the +// sink task (bulk_indexing_lane) exits it DELETES the lane; the merge host +// must re-create it on the next flush and keep flowing (no "closed" emit +// errors, nothing salvaged), and a restarted consumer binds the fresh +// instance. Before the fix every flush after a sink-task stop salvaged to +// the failure queue with "lane closed" errors, forever. +func TestBulkLaneSurvivesConsumerRestart(t *testing.T) { + installMemQueue() + + const laneName = "test-restart-lane" + const queueName = "test-restart-queue" + if _, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 16}); err != nil { + t.Fatalf("create lane: %v", err) + } + defer lane.Delete(laneName, true) + + qcfg := queue.GetOrInitConfig(queueName) + + cfg, err := config.NewConfigFrom(map[string]interface{}{ + "input_queue": queueName, + "output_lane": laneName, + "index_name": "test-logs", + "idle_timeout_in_seconds": 1, + "bulk_size_in_kb": 1, + }) + if err != nil { + t.Fatalf("build config: %v", err) + } + proc, err := NewMergeToBulkLane(cfg) + if err != nil { + t.Fatalf("new processor: %v", err) + } + + stdCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + processDone := make(chan error, 1) + go func() { processDone <- proc.Process(&pipeline.Context{Context: stdCtx}) }() + + bulks := make(chan string, 8) + subscribe := func(l lane.Lane) { + l.SubscribeBatch(4, 50*time.Millisecond, func(recs []*event.Event) { + for _, rec := range recs { + if b, ok := bulkPayloadOf(rec); ok { + bulks <- string(b) + } + } + }) + } + subscribe(lane.Get(laneName)) + + // phase 1: normal flow through the first lane instance + if err := queue.Push(qcfg, []byte(`{"id":"1","message":"before-restart"}`)); err != nil { + t.Fatal(err) + } + waitBulk(t, bulks, "before-restart") + + // the sink task exiting: bulk_indexing_lane's Process deletes the lane + // on ctx.Done — exactly this call + if err := lane.Delete(laneName, true); err != nil { + t.Fatalf("delete lane: %v", err) + } + + // phase 2: the merge's next flush re-creates the lane and emits into it + if err := queue.Push(qcfg, []byte(`{"id":"2","message":"after-restart"}`)); err != nil { + t.Fatal(err) + } + + // the "restarted consumer": wait for the producer to re-create the lane + // instance, then bind it + var fresh lane.Lane + deadline := time.Now().Add(10 * time.Second) + for fresh = lane.Get(laneName); fresh == nil && time.Now().Before(deadline); fresh = lane.Get(laneName) { + time.Sleep(50 * time.Millisecond) + } + if fresh == nil { + t.Fatal("lane was not re-created after consumer exit") + } + subscribe(fresh) + waitBulk(t, bulks, "after-restart") + + cancel() + select { + case <-processDone: + case <-time.After(5 * time.Second): + t.Fatal("Process did not return after cancel") + } + + // nothing hit the salvage path — the wedge symptom was every flush + // salvaging with "lane closed" errors + if d := queue.Depth(queue.GetOrInitConfig(queueName + "_failure")); d != 0 { + t.Errorf("failure queue depth = %d, want 0", d) + } +} + +func waitBulk(t *testing.T, bulks chan string, want string) { + t.Helper() + deadline := time.After(10 * time.Second) + for { + select { + case b := <-bulks: + if strings.Contains(b, want) { + return + } + case <-deadline: + t.Fatalf("timed out waiting for bulk containing %q", want) + } + } +} + +// ─── block-on-full backpressure ────────────────────────────────────────────── + +func TestMergeToBulkLaneBlocksWhenLaneFull(t *testing.T) { + installMemQueue() + + const laneName = "test-block-lane" + const queueName = "test-block-queue" + l, err := lane.Create(lane.LaneConfig{Name: laneName, Capacity: 1}) + if err != nil { + t.Fatalf("create lane: %v", err) + } + defer lane.Delete(laneName, true) + + // big bodies force frequent flushes: ~600B docs against a 1KB bulk + // threshold yields a merged bulk roughly every two docs + qcfg := queue.GetOrInitConfig(queueName) + bigBody := strings.Repeat("x", 600) + const docCount = 20 + for i := 0; i < docCount; i++ { + doc := `{"id":"` + fmt.Sprint(i) + `","message":"` + bigBody + `"}` + if err := queue.Push(qcfg, []byte(doc)); err != nil { + t.Fatalf("push: %v", err) + } + } + + cfg, err := config.NewConfigFrom(map[string]interface{}{ + "input_queue": queueName, + "output_lane": laneName, + "index_name": "test-logs", + "idle_timeout_in_seconds": 1, + "bulk_size_in_kb": 1, + }) + if err != nil { + t.Fatalf("build config: %v", err) + } + proc, err := NewMergeToBulkLane(cfg) + if err != nil { + t.Fatalf("new processor: %v", err) + } + + stdCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + pctx := &pipeline.Context{Context: stdCtx} + + // the sole subscriber holds its first delivery until released: the + // lane (capacity 1) saturates and the merge must BLOCK in Emit + release := make(chan struct{}) + var deliveries int32 + l.SubscribeBatch(1, 0, func(recs []*event.Event) { + atomic.AddInt32(&deliveries, 1) + <-release + }) + + processDone := make(chan error, 1) + go func() { processDone <- proc.Process(pctx) }() + + // saturation point: one record held by the subscriber, one in the + // channel, further emissions blocked — input queue consumption stalls + saturated := waitFor(time.Second*20, func() bool { + return atomic.LoadInt32(&deliveries) >= 1 && l.Depth() >= 1 + }) + if !saturated { + t.Fatalf("lane did not saturate (deliveries=%d depth=%d)", atomic.LoadInt32(&deliveries), l.Depth()) + } + time.Sleep(1500 * time.Millisecond) // idle window passes while blocked + if d := queue.Depth(qcfg); d == 0 { + t.Error("input queue drained while the lane was saturated — blocking backpressure not engaged") + } + + // release the subscriber: flow resumes, the input queue drains, and + // only then does cancellation stop the host + close(release) + if !waitFor(20*time.Second, func() bool { return queue.Depth(qcfg) == 0 }) { + t.Fatalf("input queue did not drain after release, depth=%d", queue.Depth(qcfg)) + } + cancel() + select { + case <-processDone: + case <-time.After(15 * time.Second): + t.Fatal("Process did not return after cancel") + } +} + +func waitFor(timeout time.Duration, cond func() bool) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return true + } + time.Sleep(50 * time.Millisecond) + } + return cond() +} + +// ─── sender guard path ─────────────────────────────────────────────────────── + +func TestSenderSkipsPayloadlessRecord(t *testing.T) { + p := &BulkIndexingLaneProcessor{} + pctx := &pipeline.Context{Context: context.Background()} + + // a record without the bulk payload skips before any ES interaction + // (meta stays nil): the guard must hold without panicking + rec := &event.Event{Fields: util.MapStr{"something": "else"}} + p.sendRecord(pctx, nil, rec) +} diff --git a/plugins/elastic/lane_bulk/merge_to_bulk_lane.go b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go new file mode 100644 index 00000000..dcfa996c --- /dev/null +++ b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go @@ -0,0 +1,521 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello#infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +package lane_bulk + +import ( + "fmt" + "hash/fnv" + "runtime" + "strings" + "sync" + "time" + + "github.com/buger/jsonparser" + log "github.com/cihub/seelog" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/lane" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/stats" + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/bytebufferpool" + "infini.sh/framework/modules/elastic/common" +) + +// autoLaneCapacity bounds the lane auto-created when no `lane:` stanza +// defines the output lane: enough slack to smooth sender hiccups, small +// enough that block-on-full engages at a sane memory ceiling. +const autoLaneCapacity = 64 + +// MergeToBulkLaneProcessor is the memory-path twin of indexing_merge: it +// pops JSON documents off the input disk queue, merges them into ES bulk +// NDJSON payloads (bulk_size / idle_timeout), and emits each merged payload +// as ONE record into the output lane — no output disk queue in between. +// +// Durability note: the input queue pop is the point of no return (same +// semantics as indexing_merge); a crash loses whatever is buffered in +// memory. Records travel without ack receipts — a future commit-gated +// variant can switch to EmitAck and wait on settlement without changing +// this processor's configuration contract. +// +// Backpressure: lane capacity bounds in-flight bulks; a full lane BLOCKS +// (the default overflow policy) — the merge loop stalls, then input queue +// consumption stalls. That is the intended posture for the memory path: +// pressure propagates back to the durable entry point instead of paying +// intermediate disk IO. (`overflow: spill:` remains a lane-level +// opt-out; spilled payloads are raw bulk NDJSON the legacy bulk_indexing +// can consume.) +type MergeToBulkLaneProcessor struct { + config MergeConfig + failureQ *queue.QueueConfig + outputLane lane.Lane + // laneCfg is the effective output-lane config (stanza-bound or the + // auto-create default) — the blueprint for GetOrCreate re-creation. + laneCfg lane.LaneConfig +} + +func (processor *MergeToBulkLaneProcessor) Name() string { + return "merge_to_bulk_lane" +} + +type MergeConfig struct { + InputQueue string `config:"input_queue"` + // InputLane 内存输入 (与 InputQueue 二选一): 记录为 event.Event, + // 序列化其 Fields 为文档字节后走同一聚合路径 — 上游 output{lane} + // 直写, 全程内存 (staging 队列是唯一落盘点)。 + InputLane string `config:"input_lane"` + OutputLane string `config:"output_lane"` + IndexName string `config:"index_name"` + TypeName string `config:"type_name"` + WriteOpType string `config:"write_op_type"` + KeyField string `config:"key_field"` + KeyFields []string `config:"key_fields"` + + BulkSizeInKB int `config:"bulk_size_in_kb"` + BulkSizeInMB int `config:"bulk_size_in_mb"` + IdleTimeoutInSeconds int `config:"idle_timeout_in_seconds"` + NumOfWorkers int `config:"worker_size"` + + // Elasticsearch is only used to default type_name from the cluster's + // doc type; the merge itself never talks to Elasticsearch. + Elasticsearch string `config:"elasticsearch"` + + FailureQueue string `config:"failure_queue"` +} + +func init() { + pipeline.RegisterProcessorPluginWithConfigMetadata("merge_to_bulk_lane", NewMergeToBulkLane, MergeConfig{}) +} + +func NewMergeToBulkLane(c *config.Config) (pipeline.Processor, error) { + cfg := MergeConfig{ + BulkSizeInMB: 10, + NumOfWorkers: 1, + IdleTimeoutInSeconds: 5, + } + + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack the configuration of merge_to_bulk_lane processor: %s", err) + } + + if (cfg.InputQueue == "") == (cfg.InputLane == "") { + return nil, errors.New("merge_to_bulk_lane processor requires exactly one of input_queue | input_lane") + } + if cfg.OutputLane == "" { + return nil, errors.New("merge_to_bulk_lane processor requires output_lane") + } + if cfg.IndexName == "" { + return nil, errors.New("merge_to_bulk_lane processor requires index_name") + } + + if cfg.FailureQueue == "" { + fq := cfg.InputQueue + if fq == "" { + fq = cfg.InputLane + } + cfg.FailureQueue = fq + "_failure" + } + + // Bind to the static lane definition (top-level `lane:` stanza) when + // present — its capacity and overflow policy govern this producer. + // Without one, create a block-on-full lane with a MODEST capacity: + // merged bulks are large, and the 4096-record lane default would bound + // memory at capacity × bulk_size (40GB at 10MB bulks) — blocking must + // arrive at a realistic ceiling. Define an explicit stanza to tune the + // budget (capacity × bulk_size_in_mb is the worst case). + outputLane := lane.Get(cfg.OutputLane) + var laneCfg lane.LaneConfig + if outputLane != nil { + laneCfg = outputLane.Config() + } else { + log.Warnf("merge_to_bulk_lane: lane %q not defined under lane:, creating block-on-full lane with capacity %d; define an explicit stanza to tune the memory budget", cfg.OutputLane, autoLaneCapacity) + laneCfg = lane.LaneConfig{Name: cfg.OutputLane, Capacity: autoLaneCapacity} + outputLane = lane.GetOrCreate(laneCfg) + } + + return &MergeToBulkLaneProcessor{ + config: cfg, + failureQ: queue.SmartGetOrInitConfig(&queue.QueueConfig{Name: cfg.FailureQueue}), + outputLane: outputLane, + laneCfg: laneCfg, + }, nil +} + +func (processor *MergeToBulkLaneProcessor) Process(ctx *pipeline.Context) error { + defer func() { + if !global.Env().IsDebug { + if r := recover(); r != nil { + var v string + switch r := r.(type) { + case error: + v = r.Error() + case runtime.Error: + v = r.Error() + case string: + v = r + } + log.Error("error in merge_to_bulk_lane,", v) + } + } + }() + + bulkSizeInByte := 1048576 * processor.config.BulkSizeInMB + if processor.config.BulkSizeInKB > 0 { + bulkSizeInByte = 1024 * processor.config.BulkSizeInKB + } + + wg := sync.WaitGroup{} + for i := 0; i < processor.config.NumOfWorkers; i++ { + wg.Add(1) + go processor.newBulkWorker(ctx, bulkSizeInByte, &wg) + } + + wg.Wait() + + return nil +} + +func (processor *MergeToBulkLaneProcessor) newBulkWorker(ctx *pipeline.Context, bulkSizeInByte int, wg *sync.WaitGroup) { + defer func() { + if !global.Env().IsDebug { + if r := recover(); r != nil { + var v string + switch r := r.(type) { + case error: + v = r.Error() + case runtime.Error: + v = r.Error() + case string: + v = r + } + log.Error("error in merge_to_bulk_lane worker,", v) + ctx.RecordError(fmt.Errorf("merge_to_bulk_lane panic: %v", r)) + } + } + wg.Done() + }() + + log.Trace("start merge_to_bulk_lane bulk worker") + + mainBuf := bytebufferpool.Get("lane_bulk_merge") + defer bytebufferpool.Put("lane_bulk_merge", mainBuf) + + idleDuration := time.Duration(processor.config.IdleTimeoutInSeconds) * time.Second + + typeName := processor.config.TypeName + if typeName == "" && processor.config.Elasticsearch != "" { + // tolerant defaulting: an unavailable cluster leaves _type unset + // (legal in the action line) instead of failing the merge task + if meta := elastic.GetMetadata(processor.config.Elasticsearch); meta != nil { + typeName = common.GetClusterDocType(processor.config.Elasticsearch) + } + } + + writeOpType := processor.config.WriteOpType + if writeOpType == "" { + writeOpType = "index" + } + + var lastCommit time.Time = time.Now() + + // 输入源: input_lane 时订阅 lane, 序列化 Fields 为文档字节; + // input_queue 时直接 PopTimeout 磁盘队列 (原路径)。 + docs := make(chan []byte, 256) + var laneCancel func() + if processor.config.InputLane != "" { + l := lane.Get(processor.config.InputLane) + if l == nil { + l = lane.GetOrCreate(lane.LaneConfig{Name: processor.config.InputLane, Capacity: autoLaneCapacity}) + } + laneCancel = l.SubscribeBatch(64, 200*time.Millisecond, func(recs []*event.Event) { + for _, rec := range recs { + if rec == nil { + continue + } + b, err := util.ToJSONBytes(rec.Fields) + if err != nil { + log.Warnf("merge_to_bulk_lane: skip record (serialize fields): %v", err) + continue + } + select { + case <-ctx.Done(): + return + case docs <- b: + } + } + }) + defer laneCancel() + } + +READ_DOCS: + for { + + if ctx.IsCanceled() { + goto CLEAN_BUFFER + } + + var pop []byte + if processor.config.InputLane != "" { + select { + case <-ctx.Done(): + goto CLEAN_BUFFER + case pop = <-docs: + } + } else { + var err error + pop, _, err = queue.PopTimeout(queue.GetOrInitConfig(processor.config.InputQueue), idleDuration) + if err != nil { + log.Error(err) + panic(err) + } + } + + if len(pop) > 0 { + + var id_part = "" + if len(processor.config.KeyFields) > 0 { + id_part = fmt.Sprintf(", \"_id\":\"%v\"", processor.fingerprintID(pop)) + } else if processor.config.KeyField != "" { + // zero-alloc path query: extracting ONE field + // must not float-parse the whole document (embeddings!) + v, _, _, err := jsonparser.Get(pop, strings.Split(processor.config.KeyField, ".")...) + if err != nil { + log.Warnf("merge_to_bulk_lane: key_field %q not found: %v", processor.config.KeyField, err) + } else if s, ok := trimJSONString(v); ok { + id_part = fmt.Sprintf(", \"_id\":\"%v\"", s) + } else { + id_part = fmt.Sprintf(", \"_id\":\"%v\"", string(v)) + } + } + + if typeName != "" { + mainBuf.WriteString(fmt.Sprintf("{ \"%s\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" %v } }\n", writeOpType, processor.config.IndexName, typeName, id_part)) + } else { + mainBuf.WriteString(fmt.Sprintf("{ \"%s\" : { \"_index\" : \"%s\" %v } }\n", writeOpType, processor.config.IndexName, id_part)) + } + + util.WalkBytesAndReplace(pop, util.NEWLINE, util.SPACE) + + // Data stream (create) documents must carry a top-level date + // @timestamp; the otel envelope keeps the timestamp at the + // envelope level and the record body under "payload", which + // neither reaches the doc top level. Normalize envelope docs + // for data stream semantics; bare docs pass through as-is. + if writeOpType == "create" { + pop = normalizeDataStreamDoc(pop) + } + + mainBuf.Write(pop) + mainBuf.WriteString("\n") + } + + //submit no matter the size of bulk after idle timeout + if time.Since(lastCommit) > idleDuration && mainBuf.Len() > 0 { + if global.Env().IsDebug { + log.Trace("hit idle timeout, ", idleDuration.String()) + } + goto CLEAN_BUFFER + } + + if mainBuf.Len() > (bulkSizeInByte) { + if global.Env().IsDebug { + log.Trace("hit buffer size, ", mainBuf.Len()) + } + goto CLEAN_BUFFER + } + + } + +CLEAN_BUFFER: + + lastCommit = time.Now() + + if mainBuf.Len() > 0 { + mainBuf.WriteByte('\n') + + // the record outlives this reusable buffer — hand over an + // exact-size copy, then reset + bulk := make([]byte, mainBuf.Len()) + copy(bulk, mainBuf.Bytes()) + mainBuf.Reset() + + rec := newBulkRecord(bulk, "bulk-"+util.GetUUID()) + // Re-acquire per flush: the consumer task (bulk_indexing_lane) + // DELETES the lane on exit; GetOrCreate re-creates it when absent + // or closed. A stale cached handle would emit-fail "closed" + // forever across the consumer's stop/restart cycle — re-acquiring + // here makes that cycle self-healing (bulks buffer in the fresh + // lane while the consumer is down, then drain on its restart). + processor.outputLane = lane.GetOrCreate(processor.laneCfg) + // Emit (not EmitAck): there is no commit gate on this host to + // attach receipts to, and Emit is the variant that reports the + // closed-lane failure the salvage path below needs + if err := processor.outputLane.Emit(rec); err != nil { + // closed lane (sender task stopped first): salvage the merged + // payload on the failure queue so a replay path exists + log.Errorf("merge_to_bulk_lane: emit to lane %q failed: %v", processor.config.OutputLane, err) + stats.Increment("merge_to_bulk_lane", "salvaged") + if err := queue.Push(processor.failureQ, bulk); err != nil { + log.Errorf("merge_to_bulk_lane: push to failure queue %q failed: %v", processor.failureQ.Name, err) + } + } else { + stats.Increment("merge_to_bulk_lane", "emitted") + stats.IncrementBy("merge_to_bulk_lane", "emitted_bytes", int64(len(bulk))) + } + + log.Trace("clean buffer") + } + + if ctx.IsCanceled() { + return + } + goto READ_DOCS +} + +// fingerprintID derives a deterministic document id: the configured dot-path +// fields are looked up in the queue message, joined and hashed (FNV-1a 64). +// When NONE of the fields resolve, the raw message bytes are hashed instead — +// the fallback keeps distinct documents apart for sources lacking e.g. file +// metadata. Deterministic ids make at-least-once replays idempotent when +// combined with write_op_type=create: duplicated deliveries turn into +// ES version conflicts instead of duplicate documents. +// +// NOTE the fields must be stable across redeliveries: per-attempt stamps like +// ingest timestamps change between retries and defeat dedup; content-derived +// or source-location fields (file path/offset) do not. +func (processor *MergeToBulkLaneProcessor) fingerprintID(pop []byte) string { + h := fnv.New64a() + + // zero-alloc path queries: fingerprinting must not + // float-parse the whole document (embeddings ride these messages) + matched := false + for _, f := range processor.config.KeyFields { + h.Write([]byte{'|'}) + v, _, _, err := jsonparser.Get(pop, strings.Split(f, ".")...) + if err != nil { + // keep positional meaning for missing fields + h.Write([]byte("\x00")) + continue + } + matched = true + h.Write(v) // raw JSON token: any scalar type hashes deterministically + } + if !matched { + // none of the key fields exist on this message: fall back to a + // content hash so unrelated messages don't collapse into one id + h.Reset() + h.Write(pop) + } + return fmt.Sprintf("%x", h.Sum64()) +} + +// trimJSONString strips JSON string quotes from a raw token ("" and false +// when the token is not a quoted string). +func trimJSONString(v []byte) (string, bool) { + if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' { + return string(v[1 : len(v)-1]), true + } + return "", false +} + +// normalizeDataStreamDoc turns one otel envelope (the queue's LogEvent JSON: +// {"metadata":{...},"payload":{...},"timestamp":"..."}) into a data stream +// compatible document: +// +// - payload fields are promoted to the top level (message etc. become +// first-class doc fields) +// - a top-level @timestamp is derived from payload.timestamp / +// payload.observed_timestamp / the envelope timestamp / now — data +// streams reject documents without a date @timestamp. @timestamp is the +// document's one true timestamp (house standard): the mirrored top-level +// "timestamp" and any payload "timestamp" are folded into it and removed. +// - a bare document (no "payload" key) keeps its own @timestamp when +// present; only a missing one is stamped with now. +// - metadata.file (agent collection origin) lives in @agent.file_* — the +// former top-level "file" promotion is gone (zero readers). +// +// Documents that are not envelopes (no "payload" key) pass through +// unchanged except for the @timestamp stamp. +func normalizeDataStreamDoc(doc []byte) []byte { + var m util.MapStr + if err := util.FromJSONBytes(doc, &m); err != nil { + return doc + } + + ts := "" + if payload, ok := m["payload"].(map[string]interface{}); ok { + for _, k := range []string{"timestamp", "@timestamp", "observed_timestamp"} { + if v, ok := payload[k].(string); ok && v != "" { + ts = v + break + } + } + } + if ts == "" { + if v, ok := m["timestamp"].(string); ok { + ts = v + } + } + if ts == "" { + if v, ok := m["@timestamp"].(string); ok { // bare doc already carrying one + ts = v + } + } + if ts == "" { + ts = time.Now().UTC().Format(time.RFC3339Nano) + } + + out := util.MapStr{} + if payload, ok := m["payload"].(map[string]interface{}); ok { + for k, v := range payload { + out[k] = v + } + } else { + for k, v := range m { + out[k] = v + } + } + out["@timestamp"] = ts + delete(out, "timestamp") // @timestamp is the single source of truth + + if meta, ok := m["metadata"].(map[string]interface{}); ok { + if rt, ok := meta["log_type"].(string); ok && rt != "" { + out["log_type"] = rt + } + // per-pattern label (e.g. server|deprecation|slowlog|gc) — virtual + // streams split on this field. + if lk, ok := meta["log_kind"].(string); ok && lk != "" { + out["log_kind"] = lk + } + } + return util.MustToJSONBytes(out) +} diff --git a/plugins/elastic/lane_bulk/payload.go b/plugins/elastic/lane_bulk/payload.go new file mode 100644 index 00000000..bd1cd8ac --- /dev/null +++ b/plugins/elastic/lane_bulk/payload.go @@ -0,0 +1,121 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello#infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +// Package lane_bulk provides the in-memory counterpart of the +// indexing_merge + bulk_indexing pair: merged bulk payloads travel a lane +// (zero serialization, bounded memory, optional spill-to-disk degradation) +// instead of round-tripping the output disk queue. +// +// merge_to_bulk_lane input disk queue → merged bulk records → lane +// bulk_indexing_lane lane → Elasticsearch bulk send +// +// The two processors are additive: the legacy queue-based pair keeps +// running unchanged; pipelines opt into the memory path per task. +package lane_bulk + +import ( + "fmt" + "time" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// PayloadField is the record field carrying one merged bulk NDJSON payload. +const PayloadField = "bulk" + +// codecName names the record codec for merged bulk payloads. +const codecName = "bulk" + +// bulkPayloadCodec keeps lane records and queue payloads byte-identical: +// Encode returns the wrapped bulk NDJSON as-is, so a lane configured with +// `overflow: spill:` degrades records onto a disk queue that the +// legacy bulk_indexing processor can consume directly (fast/slow paths run +// the equivalent "send to ES" step, satisfying the spill equivalence rule). +type bulkPayloadCodec struct{} + +func (bulkPayloadCodec) Name() string { return codecName } +func (bulkPayloadCodec) Namespace() string { return codecName } + +func (bulkPayloadCodec) Decode(data []byte) (*event.Event, error) { + if len(data) == 0 { + return nil, fmt.Errorf("bulk codec: empty payload") + } + rec := &event.Event{ + Timestamp: time.Now().UTC(), + Fields: util.MapStr{PayloadField: data}, + Metadata: event.EventMetadata{Labels: util.MapStr{pipeline.CodecLabel: codecName}}, + } + return rec, nil +} + +func (bulkPayloadCodec) Encode(rec *event.Event) ([]byte, error) { + if rec == nil { + return nil, fmt.Errorf("bulk codec: nil record") + } + switch v := rec.Fields[PayloadField].(type) { + case []byte: + return v, nil + case string: + return []byte(v), nil + default: + return nil, fmt.Errorf("bulk codec: record field [%s] is not a payload", PayloadField) + } +} + +func init() { + pipeline.MustRegisterRecordCodec(bulkPayloadCodec{}) +} + +// newBulkRecord wraps one merged bulk payload as a lane record. The bytes +// must be caller-owned (not alias a reusable buffer): the record outlives +// the flush that produced it. The codec label routes spill encoding and +// keeps queue round-trips symmetric. +func newBulkRecord(bulk []byte, traceID string) *event.Event { + rec := &event.Event{ + Timestamp: time.Now().UTC(), + Fields: util.MapStr{PayloadField: bulk}, + Metadata: event.EventMetadata{Labels: util.MapStr{pipeline.CodecLabel: codecName}}, + } + pipeline.SetTraceID(rec, traceID) + return rec +} + +// bulkPayloadOf extracts the merged bulk bytes from a lane record. +func bulkPayloadOf(rec *event.Event) ([]byte, bool) { + if rec == nil { + return nil, false + } + switch v := rec.Fields[PayloadField].(type) { + case []byte: + return v, len(v) > 0 + case string: + return []byte(v), v != "" + default: + return nil, false + } +} From 29164e90c8810bf4683508c1f901ffc2ca87c429 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 10:58:46 +0800 Subject: [PATCH 29/65] docs: processor reference pages for the lane bulk pipeline New pages: bulk_indexing_lane, merge_to_bulk_lane, lane_consumer, if, output, tap; index updates for processors, conditions and API references. --- .../content.en/docs/development/api/_index.md | 1 + .../docs/references/conditions/_index.md | 1 + docs/content.en/docs/references/pipeline.md | 33 +++++++++++- .../docs/references/processors/_index.md | 5 ++ .../processors/bulk_indexing_lane.md | 34 +++++++++++++ .../docs/references/processors/for_each.md | 1 + .../docs/references/processors/if.md | 51 +++++++++++++++++++ .../references/processors/lane_consumer.md | 42 +++++++++++++++ .../processors/merge_to_bulk_lane.md | 47 +++++++++++++++++ .../docs/references/processors/output.md | 49 ++++++++++++++++++ .../docs/references/processors/tap.md | 30 +++++++++++ 11 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 docs/content.en/docs/references/processors/bulk_indexing_lane.md create mode 100644 docs/content.en/docs/references/processors/if.md create mode 100644 docs/content.en/docs/references/processors/lane_consumer.md create mode 100644 docs/content.en/docs/references/processors/merge_to_bulk_lane.md create mode 100644 docs/content.en/docs/references/processors/output.md create mode 100644 docs/content.en/docs/references/processors/tap.md diff --git a/docs/content.en/docs/development/api/_index.md b/docs/content.en/docs/development/api/_index.md index c5a32cbf..6dce9000 100644 --- a/docs/content.en/docs/development/api/_index.md +++ b/docs/content.en/docs/development/api/_index.md @@ -1,6 +1,7 @@ --- weight: 40 title: "API Development" +bookCollapseSection: true --- # API Development diff --git a/docs/content.en/docs/references/conditions/_index.md b/docs/content.en/docs/references/conditions/_index.md index a100de49..5987de9b 100644 --- a/docs/content.en/docs/references/conditions/_index.md +++ b/docs/content.en/docs/references/conditions/_index.md @@ -1,6 +1,7 @@ --- title: "Conditions" weight: 45 +bookCollapseSection: true --- # Conditions diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md index 2e982d58..3bf11a16 100644 --- a/docs/content.en/docs/references/pipeline.md +++ b/docs/content.en/docs/references/pipeline.md @@ -505,7 +505,7 @@ import _ "your_app/plugins/health" ## Built-in Processors -The framework ships with a growing set of built-in processors registered via `RegisterProcessorPlugin` / `RegisterDomainProcessor` — framework primitives (`echo`, `dag`, `for_each`), queue bridges (`consumer`, `queue_output`), Elasticsearch indexing (`bulk_indexing`, `json_indexing`, `indexing_merge`, `merge_to_bulk`), output transports (`otlp_export`), general-purpose helpers (`http`, `smtp`, `replay`), and a wide catalog of record processors for parsing, transforming, enriching, routing and governing records inside `for_each` sub-chains (`dissect`, `grok`, `mutate`, `geoip`, `throttle`, `script`, ...). +The framework ships with a growing set of built-in processors registered via `RegisterProcessorPlugin` / `RegisterDomainProcessor` — framework primitives (`echo`, `dag`, `for_each`), queue bridges (`consumer`, `queue_output`, `lane_consumer`), Elasticsearch indexing (`bulk_indexing`, `json_indexing`, `indexing_merge`, `merge_to_bulk`, and the in-memory tail `merge_to_bulk_lane` + `bulk_indexing_lane`), unified output (`output` — exactly one of queue/lane), output transports (`otlp_export`), general-purpose helpers (`http`, `smtp`, `replay`), and a wide catalog of record processors for parsing, transforming, enriching, routing and governing records inside `for_each` sub-chains (`dissect`, `grok`, `mutate`, `geoip`, `throttle`, `script`, ...). Every processor has its own reference page — description, configuration parameters with types and defaults, and an example — under the **Processor Reference** section. Record processors operate on the current record inside `for_each` sub-chains (most are no-ops outside a record scope); they are registered by domain category, the bare name is the YAML key, and `category:name` is an explicit alias. @@ -579,6 +579,17 @@ The gate is only as strong as constructor validation. Two rules, enforced by the 1. **Return errors, never panic** — a config problem is not a runtime fault; `panic` in a constructor crashes the host that compiles the chain. 2. **Either require your inputs or default them safely** — every processor is constructed with an empty config in CI; one that succeeds must land in the audit's documented allowlist (44 entries, each annotated with why empty-accept is legitimate), otherwise it must fail with a clear required-field error. Variable templates such as `$[[env.HOSTS]]` remain non-empty strings at validation time and do not conflict with required-field checks. +## Stream indexing tail: lanes vs disk bridge + +Compiled stream pipelines (LogPilot → Gateway) choose between two tail shapes via the stream spec `sink.transport`: + +| `sink.transport` | Shape | Durability | +|------------------|-------|------------| +| `""` / `"lane"` (default) | `consumer(staging)` → `for_each(parallel)` → `output{lane}` → `merge_to_bulk_lane{input_lane}` → `bulk_indexing_lane` → ES. The staging queue is the **only** disk write. | At-most-once: in-flight records are lost if the gateway exits. | +| `"disk"` | `consumer(staging)` → `output{queue: indexing}` → `indexing_merge` → `bulk` queue → `bulk_indexing` → ES. | At-least-once: bulk payloads survive a gateway restart. | + +Both tails process records in parallel (`for_each.parallel`, `worker_size`); log workloads are order-insensitive. + ## Pipeline Configuration API Beyond YAML files, pipelines are managed as first-class resources (`PipelineConfigV2`) over HTTP: @@ -670,6 +681,24 @@ A pipeline whose config fails to compile **on the instance** (e.g. the instance' | `POST /pipeline/dry-run` | Unified dry-run: execute a processor chain against sample input with side-effect capture, field diffs, and scope/contract checks. | | `GET /pipeline/processors` | Processor catalog (`?grouped=1`). | | `GET /pipeline/codecs` | Registered record codecs. | -| `GET /pipeline/lanes` | In-process lane channels and their depth. | +| `GET /pipeline/lanes` | In-process lane channels: depth, emitted/delivered counters, drops, backpressure waits. | +| `GET /pipeline/task/:id/stages?stage=` | Per-stage metrics for one stage. | +| `DELETE /pipeline/task/:id` | Stop and remove a task (cancels consumers first, waits for release). | + +### Queue consumer management + +Queue consumers (offset registrations) are manageable per queue — stale consumer groups from removed pipelines are the typical cleanup target: + +| Method & Path | Description | +|---------------|-------------| +| `GET /queue/:id/consumer/:consumer_id/offset` | Read a consumer's committed offset. | +| `PUT /queue/:id/consumer/:consumer_id/offset?offset=,` | Reset the offset to an explicit segment/position. Rewinds re-ingest (duplicates); jumps skip backlog. An actively committing consumer overwrites manual resets at its next commit cycle — pause the pipeline first for a sticky reset. | +| `DELETE /queue/:id/consumer/:consumer_id` | Remove the consumer registration and its offset. Active consumers re-register at `0,0` (from head); stale registrations clear permanently. | + +Notes: + +- Address the **queue by its registry UUID** (the key in `/queue/stats`) — consumer/offset lookups walk the per-queue KV bucket and do not resolve queue names. +- The consumer ID is the `id` field of the entry in `/queue/stats` (`?use_key=true&consumers=true`). +- All three are surfaced in the Console queue browser per consumer (reset-to-earliest / reset-to-latest / delete). `GET /pipeline/cluster/overview` aggregates fleet-wide pipeline health for control-plane dashboards. diff --git a/docs/content.en/docs/references/processors/_index.md b/docs/content.en/docs/references/processors/_index.md index 8787a9bb..ee3b0748 100644 --- a/docs/content.en/docs/references/processors/_index.md +++ b/docs/content.en/docs/references/processors/_index.md @@ -1,6 +1,7 @@ --- title: "Processor Reference" weight: 16 +bookCollapseSection: true --- # Processor Reference @@ -22,6 +23,7 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea |-----------|-------------| | [`consumer`](consumer/) | Consumes a queue (by name or label selector) and exposes the batch to the pipeline. | | [`queue_output`](queue_output/) | Appends the processed batch onto a target queue — the two-stage pipeline bridge. | +| [`lane_consumer`](lane_consumer/) | Consumes an in-memory lane in competing micro-batches, runs a record sub-chain, optional durable bridge. | ## Elasticsearch @@ -31,12 +33,15 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea | [`json_indexing`](json_indexing/) | Indexes JSON documents into Elasticsearch. | | [`indexing_merge`](indexing_merge/) | Merges small documents into combined indexing requests before writing. | | [`merge_to_bulk`](merge_to_bulk/) | Merges events into Elasticsearch bulk requests. | +| [`merge_to_bulk_lane`](merge_to_bulk_lane/) | Aggregates queue/lane input into bulk payloads written to an in-memory lane — the memory tail of the indexing pipeline. | +| [`bulk_indexing_lane`](bulk_indexing_lane/) | Ships combined bulk payloads from an in-memory lane to Elasticsearch. | ## Output transport | Processor | Description | |-----------|-------------| | [`otlp_export`](otlp_export/) | Ships the processed batch to any OTLP/gRPC collector. | +| [`output`](output/) | Unified output step: writes to exactly one target — a durable queue (batch scope) or an in-memory lane (record scope). | ## General diff --git a/docs/content.en/docs/references/processors/bulk_indexing_lane.md b/docs/content.en/docs/references/processors/bulk_indexing_lane.md new file mode 100644 index 00000000..9de984be --- /dev/null +++ b/docs/content.en/docs/references/processors/bulk_indexing_lane.md @@ -0,0 +1,34 @@ +--- +title: "bulk_indexing_lane" +weight: 306 +--- + +# bulk_indexing_lane + +| Category | Scope | +|----------|-------| +| Elasticsearch | pipeline (consumer-form) | + +Consumes combined bulk payloads from an in-memory lane and ships them to Elasticsearch. Paired with [`merge_to_bulk_lane`](merge_to_bulk_lane/) — the send half of the all-memory indexing tail. One batch's payloads are sent concurrently across `worker_size` workers; the batch function waits for every send before returning, keeping receipt settlement exact. + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `lane` | string | required | Lane carrying combined bulk payloads (from `merge_to_bulk_lane.output_lane`). | +| `elasticsearch` | string | required | Cluster reference (id resolved Gateway-side; no credentials inlined). | +| `worker_size` | int | | Send one batch's payloads concurrently across workers. | +| `bulk` | object | | Bulk processor tuning: `batch_size_in_mb`, `batch_size_in_docs`, `retry_delay_in_seconds`, `max_reject_retry_times`, `request_timeout_in_second`, dead-letter queues. | +| `failure_queue` | string | | Dead-letter queue for payloads that exhausted retries. | +| `max_retries` | int | | Send retry attempts before dead-lettering. | +| `retry_delay_in_ms` | int | | Delay between send retries. | + +## Example + +```yaml +processor: + - bulk_indexing_lane: + lane: bulk-lane- + elasticsearch: cls-1 + worker_size: 4 +``` diff --git a/docs/content.en/docs/references/processors/for_each.md b/docs/content.en/docs/references/processors/for_each.md index 352962ce..76ae06bd 100644 --- a/docs/content.en/docs/references/processors/for_each.md +++ b/docs/content.en/docs/references/processors/for_each.md @@ -14,6 +14,7 @@ weight: 103 | Field | Type | Default | Description | |-------|------|---------|-------------| | `message_field` | string | "messages" | Context key holding the message batch. | +| `parallel` | int | 1 | Runs the record sub-chain with N workers concurrently. Log workloads are order-insensitive — with a serial per-record cost of ~330µs, `parallel: 4` measured ~2× end-to-end throughput. Processors must be safe for concurrent `Process` calls on **distinct** records (they mutate the record, never shared state); every in-repo record processor satisfies this. Cancellation still propagates from the parent context. | | `codec` | string | | Payload codec used to decode/encode records (default `otel`). | | `on_failure` | string | | Sub-chain failure strategy: `ignore`, `tag` or `fail`. | | `failure_tag` | string | "_processing_failed" | Tag appended to the record when `on_failure` is `tag`. | diff --git a/docs/content.en/docs/references/processors/if.md b/docs/content.en/docs/references/processors/if.md new file mode 100644 index 00000000..c5a3615f --- /dev/null +++ b/docs/content.en/docs/references/processors/if.md @@ -0,0 +1,51 @@ +--- +title: "if" +weight: 207 +--- + +# if + +| Category | Scope | +|----------|-------| +| Routing | record (sub-chain host) | + +Conditional branch: when the condition matches, records run the `then` sub-chain; otherwise the `else` sub-chain. After the branch completes, the event returns to the main chain. The condition body is written directly under the `if` key; `then`/`else` are sibling keys holding processor objects. + +Conditions use the [conditions DSL](../conditions/) — field comparisons, patterns, ranges, network membership, and logical `and`/`or`/`not` combinations, nestable arbitrarily. + +## Structure + +```yaml +- if: + # any condition operator, see Conditions + then: + - ... # runs when the condition matches + else: + - ... # optional; runs when it does not +``` + +| Key | Description | +|-----|-------------| +| `` | Condition operator at the `if` key (`equals`, `regexp`, `range`, `and`, ...). | +| `then` | Sub-chain for matching records. An empty `then` fails compile validation. | +| `else` | Optional sub-chain for non-matching records. | +| `when` | Per-processor conditional execution is the same mechanism declared on any processor (evaluates the condition, skips the step when it does not match). | + +## Example + +```yaml +- if: + equals: + log_level: error + then: + - drop_event: + else: + - mutate: + add: + routed: non-error +``` + +## Notes + +- Branch execution returns to the main chain afterwards; a branch `terminal` processor (e.g. [`drop_event`](drop_event/)) ends the event instead. +- Sub-chain processors must be safe for concurrent `Process` calls when the surrounding [`for_each`](for_each/) runs with `parallel > 1` (each processor only touches its own record). diff --git a/docs/content.en/docs/references/processors/lane_consumer.md b/docs/content.en/docs/references/processors/lane_consumer.md new file mode 100644 index 00000000..b461b407 --- /dev/null +++ b/docs/content.en/docs/references/processors/lane_consumer.md @@ -0,0 +1,42 @@ +--- +title: "lane_consumer" +weight: 211 +--- + +# lane_consumer + +| Category | Scope | +|----------|-------| +| Queue | pipeline (batch host) | + +Consumes an in-memory lane in competing micro-batches and executes a record sub-chain per batch, stage-wise in written order. Records arrive as live pointers (no codec decode). Surviving records are optionally encoded once into an `output_queue` durable bridge — the memory-to-disk counterpart of `output{lane}`. + +Use it to parallelize or re-shape the tail of a lane pipeline: several `lane_consumer` steps (or multiple workers) compete on the same lane, each record is delivered to exactly one of them. + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `lane` | string | required | Lane to consume. Must exist (`lane:` stanza or created by an upstream `output{lane}`). | +| `batch_max` | int | 64 | Records per micro-batch (flush on size). | +| `batch_flush_ms` | int | 100 | Flush an idle partial batch after this window. | +| `on_failure` | string | "tag" | Sub-chain failure strategy: `tag` (default), `dead_letter:`, or `ignore`. | +| `failure_tag` | string | "_processing_failed" | Tag appended to the record when `on_failure` is `tag`. | +| `output_queue` | string | | Optional durable bridge: surviving records encode once into this queue. | +| `output_codec` | string | | Override the codec used for the `output_queue` encode. | +| `processor` | list (processor configs) | required | Record-scope sub-chain executed per batch, stage-wise in written order. | + +## Example + +Parallel aggregation tail: two workers compete on the raw lane, each aggregates bulk payloads and hands them to the send lane (paired with [`bulk_indexing_lane`](bulk_indexing_lane/)): + +```yaml +processor: + - lane_consumer: + lane: ingest-lane- + batch_max: 64 + batch_flush_ms: 100 + processor: + - merge_to_bulk: + output_queue: { name: bulk- } +``` diff --git a/docs/content.en/docs/references/processors/merge_to_bulk_lane.md b/docs/content.en/docs/references/processors/merge_to_bulk_lane.md new file mode 100644 index 00000000..3017939c --- /dev/null +++ b/docs/content.en/docs/references/processors/merge_to_bulk_lane.md @@ -0,0 +1,47 @@ +--- +title: "merge_to_bulk_lane" +weight: 305 +--- + +# merge_to_bulk_lane + +| Category | Scope | +|----------|-------| +| Elasticsearch | pipeline (consumer-form) | + +Reads documents from an `input_queue` (disk) **or** an `input_lane` (memory), aggregates them into combined Elasticsearch bulk payloads, and writes the payloads into an in-memory lane. Paired with [`bulk_indexing_lane`](bulk_indexing_lane/) — together they replace the disk-queue bridge (`indexing_merge` + `bulk` queue + `bulk_indexing`) with an all-memory tail: the staging queue is the only disk write. In-flight payloads are lost if the node exits; the lane applies block-on-full backpressure instead of unbounded buffering. + +Exactly one of `input_queue` / `input_lane` must be set. The output lane is auto-created (block-on-full, capacity 64) when no `lane:` stanza defines it — define an explicit stanza to tune the memory budget (capacity × `bulk_size_in_mb` is the worst case). + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `input_queue` | string | | Disk queue to read documents from. | +| `input_lane` | string | | Lane to read records from — each record's fields are serialized to a document. Exactly one of `input_queue` / `input_lane`. | +| `output_lane` | string | required | Lane receiving the combined bulk payloads. | +| `index_name` | string | required | Target index / data stream name written into the bulk action line. | +| `type_name` | string | | `_type` for clusters that still require it (defaults from the target cluster's doc type). | +| `write_op_type` | string | "index" | Bulk action (`index` / `create`). Data streams only accept `create`. | +| `key_field` / `key_fields` | string / list | | Deterministic `_id`: fingerprint of the given field(s) makes replays land on version conflicts instead of duplicates. Without a file source, falls back to hashing the whole document. | +| `bulk_size_in_mb` | int | 10 | Flush the aggregate payload at this size. | +| `bulk_size_in_kb` | int | | Size trigger in KB (overrides MB). | +| `idle_timeout_in_seconds` | int | 5 | Flush a partial payload after this idle window. | +| `worker_size` | int | 1 | Competing aggregation workers on the input (lane input: workers subscribe in parallel; queue input: workers pop concurrently). | +| `elasticsearch` | string | | Cluster reference — only used to default `type_name`; the merge never talks to Elasticsearch. | +| `failure_queue` | string | `_failure` | Dead-letter queue for failed payloads. | + +## Example + +Lane-to-lane aggregation tail (memory-only between staging and the send lane): + +```yaml +processor: + - merge_to_bulk_lane: + input_lane: ingest-lane- + output_lane: bulk-lane- + index_name: nginx2 + write_op_type: create + worker_size: 2 + elasticsearch: cls-1 +``` diff --git a/docs/content.en/docs/references/processors/output.md b/docs/content.en/docs/references/processors/output.md new file mode 100644 index 00000000..0d116cc5 --- /dev/null +++ b/docs/content.en/docs/references/processors/output.md @@ -0,0 +1,49 @@ +--- +title: "output" +weight: 210 +--- + +# output + +| Category | Scope | +|----------|-------| +| Output transport | batch (queue target) · record (lane target) | + +Unified output step: writes the processed batch or record to **exactly one** target — a durable queue or an in-memory lane. Config unpacking rejects zero or multiple targets. `queue_output` is a registered alias for the queue form. + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `queue` | string or object | | Target queue: bare name, or `{ name, label }` object. Queue and lane are mutually exclusive. | +| `lane` | string | | Target lane name. Record scope only — the lane carries live event pointers. Auto-created (block-on-full, default capacity) when no `lane:` stanza defines it. | +| `ack` | bool | | Lane target only: attach an acknowledgment receipt to the context's AckSet, so the emitting consumer's commit gate waits for settlement (cross-lane at-least-once). | +| `codec` | string | | Queue target: override the codec resolution. | +| `message_field` | string | "messages" | Batch scope (queue target): context key holding the batch. | + +## Scope rules + +| Target | Scope | Notes | +|--------|-------|-------| +| `queue` | batch (top level) | Encodes the batch through the 3-level codec resolution. | +| `lane` | record (inside `for_each` / `lane_consumer`) | Placing it at the consumer top level errors per batch with "requires record scope". | + +## Example + +Lane hand-off — the main pipeline writes processed records into an in-memory lane instead of a second disk queue (staging stays the only disk write): + +```yaml +processor: + - consumer: + queue_selector: + keys: ["staging-"] + message_field: messages + processor: + - for_each: + message_field: messages + parallel: 4 + processor: + - grok: { field: message, patterns: ["..."] } + - output: + lane: ingest-lane- +``` diff --git a/docs/content.en/docs/references/processors/tap.md b/docs/content.en/docs/references/processors/tap.md new file mode 100644 index 00000000..652e9e91 --- /dev/null +++ b/docs/content.en/docs/references/processors/tap.md @@ -0,0 +1,30 @@ +--- +title: "tap" +weight: 212 +--- + +# tap + +| Category | Scope | +|----------|-------| +| General | record | + +Side-channel sampler: copies a sampled fraction of records to a debug queue or lane. The main flow is never affected — it is an in-flight **copy**, not a divert. Sanctioned side-channel for debug indices, shadow runs and observation sampling. Exactly one target (`queue` or `lane`). + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `sample_ratio` | float | 1.0 | Fraction of records copied, within (0, 1]. `1.0` copies everything. | +| `queue` | object | | Side target queue (`{ name }`). Mutually exclusive with `lane`. | +| `lane` | string | | Side target lane. Mutually exclusive with `queue`. | + +## Example + +Copy 10% of records into a debug lane: + +```yaml +- tap: + sample_ratio: 0.1 + lane: debug-samples +``` From 400c77d5aaf1a5c77b56ac73e69145b02e670c9e Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 11:29:42 +0800 Subject: [PATCH 30/65] fix(ci): keep the empty-config audit importable without the enterprise overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestEmptyConfigAudit blank-imported every plugins/enterprise/processors/* package to register the full shipping catalog, but plugins/enterprise/ is a gitignored overlay checkout (framework-enterprise-plugins) that never exists in CI — go mod tidy failed there trying to resolve the imports. The audit now covers the OSS catalog this repo tracks (modules/pipeline built-ins plus the elastic/http/queue/replay/smtp plugins); the OSS registry passes unchanged: exactly the six audited default-able processors (bulk_indexing, for_each, merge_to_bulk, output, replay, tap) accept an empty config. Enterprise processors get the same audit from a twin test that lives in the enterprise-plugins overlay repo, so the full catalog stays covered wherever the overlay is present. --- modules/pipeline/empty_config_audit_test.go | 86 ++++----------------- 1 file changed, 13 insertions(+), 73 deletions(-) diff --git a/modules/pipeline/empty_config_audit_test.go b/modules/pipeline/empty_config_audit_test.go index 027e99e1..319063b1 100644 --- a/modules/pipeline/empty_config_audit_test.go +++ b/modules/pipeline/empty_config_audit_test.go @@ -11,52 +11,16 @@ import ( "infini.sh/framework/core/config" "infini.sh/framework/core/pipeline" - // register the processor catalog the shipping binaries carry - // (direct imports: the plugins umbrella pulls optional deps like clashx) + // register the OSS processor catalog this repo ships (direct imports: + // the plugins umbrella pulls optional deps like clashx). Enterprise + // processors live in the separate enterprise-plugins overlay repo and + // are audited by the twin test in that repo — importing them here would + // break CI checkouts where the overlay is absent. _ "infini.sh/framework/plugins/elastic/bulk_indexing" _ "infini.sh/framework/plugins/elastic/indexing_merge" _ "infini.sh/framework/plugins/elastic/json_indexing" _ "infini.sh/framework/plugins/elastic/lane_bulk" _ "infini.sh/framework/plugins/elastic/merge_to_bulk" - _ "infini.sh/framework/plugins/enterprise/otlp/export" - _ "infini.sh/framework/plugins/enterprise/processors/decode_duration" - _ "infini.sh/framework/plugins/enterprise/processors/fingerprint" - _ "infini.sh/framework/plugins/enterprise/processors/string" - _ "infini.sh/framework/plugins/enterprise/processors/xml" - _ "infini.sh/framework/plugins/enterprise/processors/dateformat" - _ "infini.sh/framework/plugins/enterprise/processors/kv" - _ "infini.sh/framework/plugins/enterprise/processors/redact" - _ "infini.sh/framework/plugins/enterprise/processors/sample" - _ "infini.sh/framework/plugins/enterprise/processors/web_log" - _ "infini.sh/framework/plugins/enterprise/processors/cidr" - _ "infini.sh/framework/plugins/enterprise/processors/communityid" - _ "infini.sh/framework/plugins/enterprise/processors/drop_event" - _ "infini.sh/framework/plugins/enterprise/processors/enrich_es" - _ "infini.sh/framework/plugins/enterprise/processors/script" - _ "infini.sh/framework/plugins/enterprise/processors/clone" - _ "infini.sh/framework/plugins/enterprise/processors/dissect" - _ "infini.sh/framework/plugins/enterprise/processors/mutate" - _ "infini.sh/framework/plugins/enterprise/processors/registered_domain" - _ "infini.sh/framework/plugins/enterprise/processors/urldecode" - _ "infini.sh/framework/plugins/enterprise/processors/add_locale" - _ "infini.sh/framework/plugins/enterprise/processors/drop_fields" - _ "infini.sh/framework/plugins/enterprise/processors/field_standardize" - _ "infini.sh/framework/plugins/enterprise/processors/geoip" - _ "infini.sh/framework/plugins/enterprise/processors/syslogdecode" - _ "infini.sh/framework/plugins/enterprise/processors/url" - _ "infini.sh/framework/plugins/enterprise/processors/context_enrich" - _ "infini.sh/framework/plugins/enterprise/processors/csv" - _ "infini.sh/framework/plugins/enterprise/processors/date" - _ "infini.sh/framework/plugins/enterprise/processors/decode_base64_field" - _ "infini.sh/framework/plugins/enterprise/processors/decompress_gzip_field" - _ "infini.sh/framework/plugins/enterprise/processors/extract_array" - _ "infini.sh/framework/plugins/enterprise/processors/grok" - _ "infini.sh/framework/plugins/enterprise/processors/json" - _ "infini.sh/framework/plugins/enterprise/processors/otel_normalize" - _ "infini.sh/framework/plugins/enterprise/processors/pattern_tagger" - _ "infini.sh/framework/plugins/enterprise/processors/pizza_bulk" - _ "infini.sh/framework/plugins/enterprise/processors/throttle" - _ "infini.sh/framework/plugins/enterprise/processors/useragent" _ "infini.sh/framework/plugins/http" _ "infini.sh/framework/plugins/queue/consumer" _ "infini.sh/framework/plugins/queue/queue_output" @@ -72,39 +36,15 @@ import ( // processor that lands here must either gain required-field validation // or be consciously added to the allowlist. func TestEmptyConfigAudit(t *testing.T) { - // audited: every parameter has a working default + // audited: every parameter has a working default (OSS catalog only — + // enterprise verdicts live in the twin test in the enterprise repo) allowlist := map[string]bool{ - "add_locale": true, // locale auto-detect - "bulk_indexing": true, // detect_active_queue auto-discovery mode - "clone": true, // clone whole record - "community_id": true, // standard seed defaults - "context_enrich": true, // promote-all default fields - "csv": true, // field defaults to otel message - "date": true, // field defaults to otel timestamp - "date_format": true, // source defaults to record timestamp - "decode_duration": true, // field defaults to "duration" - "decompress_gzip_field": true, - "drop_event": true, // unconditional drop - "drop_fields": true, // no-op until fields set — harmless - "fingerprint": true, // hashes the whole record - "for_each": true, // host — inner chain required, checked separately - "json": true, // parse default field - "kv": true, // field + defaults - "merge_to_bulk": true, - "mutate": true, // no-op until operations set — harmless - "otel_normalize": true, // normalization rules built-in - "output": true, // queue defaults to event routing - "pizza_bulk": true, - "registered_domain": true, // field defaults - "replay": true, // localhost:9200 test-utility default - "sample": true, // ratio defaults 0.5, range-checked - "tap": true, // taps everything by default - "throttle": true, - "url": true, // field defaults to uri // 100 events/1s/tag defaults, all validated - "urldecode": true, // field defaults - "useragent": true, // field defaults - "web_log": true, // parser defaults - "field_standardize": true, // mode default + "bulk_indexing": true, // detect_active_queue auto-discovery mode + "for_each": true, // host — inner chain required, checked separately + "merge_to_bulk": true, + "output": true, // queue defaults to event routing + "replay": true, // localhost:9200 test-utility default + "tap": true, // taps everything by default } ctors := pipeline.ProcessorConstructors() From 971273992e8eb02c46e17d48d4c98baeadce8bf7 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 12:19:24 +0800 Subject: [PATCH 31/65] ci: run unit tests with the enterprise plugins overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New enterprise-test job alongside the OSS unit test job: it checks out the framework-enterprise-plugins overlay into plugins/enterprise (the branch matching the PR when it exists there, default branch otherwise, via the shared CI_TOKEN secret) plus the infini.sh/license module, wires the license replace in (the umbrella packages need it; it stays out of go.mod on purpose), and runs the same OFFLINE_BUILD make tidy test — so enterprise packages and their tests, including the twin empty-config audit, run on every PR. Verified locally end-to-end: tidy, full-tree build, all test binaries, and the enterprise/pipeline suites pass with the overlay mounted. --- .github/workflows/unit_test.yml | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 366ba36d..a551193f 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -49,4 +49,68 @@ jobs: # for unit test cd $WORK echo Testing code at $PWD ... + OFFLINE_BUILD=true CI=true make tidy test + + enterprise-test: + timeout-minutes: 60 + runs-on: ubuntu-latest + env: + GO_VERSION: 1.23.4 + steps: + - name: Checkout current repository + uses: actions/checkout@v4 + with: + path: framework + + - name: Set up go toolchain + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: false + cache: true + + # enterprise 插件 overlay (私有仓库): 优先取与本 PR 同名的分支, + # 不存在时回退默认分支; license 模块是 overlay 的编译依赖。 + - name: Checkout enterprise plugins overlay + env: + CI_GH_TOKEN: ${{ secrets.CI_TOKEN }} + OVERLAY_REF: ${{ github.head_ref }} + run: | + set -e + REPO="https://x-access-token:${CI_GH_TOKEN}@github.com/infinilabs/framework-enterprise-plugins.git" + if [ -n "$OVERLAY_REF" ] && git ls-remote --exit-code --heads "$REPO" "refs/heads/${OVERLAY_REF}" >/dev/null 2>&1; then + git clone --depth 1 --branch "${OVERLAY_REF}" "$REPO" framework/plugins/enterprise + else + git clone --depth 1 "$REPO" framework/plugins/enterprise + fi + git -C framework/plugins/enterprise log --oneline -1 + + - name: Checkout license module + uses: actions/checkout@v4 + with: + token: ${{ secrets.CI_TOKEN }} + repository: infinilabs/license + ref: main + path: license + + - name: Check go toolchain + run: go version + + - name: Unit test with enterprise overlay + env: + GOFLAGS: -tags=ci + run: | + echo Home path is $HOME + export WORKBASE=$HOME/go/src/infini.sh + export WORK=$WORKBASE/framework + + # for test workspace + mkdir -p $HOME/go/src/ + ln -s $GITHUB_WORKSPACE $WORKBASE + + # overlay 的 umbrella 包依赖私有模块 infini.sh/license (不进 go.mod) + cd $WORK + go mod edit -replace infini.sh/license=../license + + echo Testing code at $PWD (with enterprise overlay) ... OFFLINE_BUILD=true CI=true make tidy test \ No newline at end of file From 2fb8dee0b10bd2c2f6cf79c9f047cd9cc48489d6 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 12:23:46 +0800 Subject: [PATCH 32/65] ci: fall back to DOCS_DEPLOYMENT_TOKEN until a dedicated CI_TOKEN exists --- .github/workflows/unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index a551193f..77983b33 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -73,7 +73,7 @@ jobs: # 不存在时回退默认分支; license 模块是 overlay 的编译依赖。 - name: Checkout enterprise plugins overlay env: - CI_GH_TOKEN: ${{ secrets.CI_TOKEN }} + CI_GH_TOKEN: ${{ secrets.CI_TOKEN || secrets.DOCS_DEPLOYMENT_TOKEN }} OVERLAY_REF: ${{ github.head_ref }} run: | set -e @@ -88,7 +88,7 @@ jobs: - name: Checkout license module uses: actions/checkout@v4 with: - token: ${{ secrets.CI_TOKEN }} + token: ${{ secrets.CI_TOKEN || secrets.DOCS_DEPLOYMENT_TOKEN }} repository: infinilabs/license ref: main path: license From 86776ec27504c309f832caddb1bb4621d8e7d4d2 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 12:27:37 +0800 Subject: [PATCH 33/65] =?UTF-8?q?ci:=20quote=20the=20echo=20line=20?= =?UTF-8?q?=E2=80=94=20unquoted=20parens=20broke=20bash=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 77983b33..44888f48 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -112,5 +112,5 @@ jobs: cd $WORK go mod edit -replace infini.sh/license=../license - echo Testing code at $PWD (with enterprise overlay) ... + echo "Testing code at $PWD (with enterprise overlay) ..." OFFLINE_BUILD=true CI=true make tidy test \ No newline at end of file From 0cc5c3cbc725469627143f687f4628a7fbc185a3 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 12:46:48 +0800 Subject: [PATCH 34/65] docs: reference page for the case processor and index rows for if/case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit case (compile-time routing, added in this PR) had a pipeline.md section but no per-construct page and no row in the processor reference index — if was missing from the index too. The page documents the nested field/branches/default structure, values (typed in-style membership on field) and when arms (both = AND), first-match-wins with optional default and pass-through otherwise, compile-time validation, scope inheritance, and the case:N / case:default / case:no-match flow trace. --- .../docs/references/processors/_index.md | 2 + .../docs/references/processors/case.md | 100 ++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 docs/content.en/docs/references/processors/case.md diff --git a/docs/content.en/docs/references/processors/_index.md b/docs/content.en/docs/references/processors/_index.md index ee3b0748..ac585428 100644 --- a/docs/content.en/docs/references/processors/_index.md +++ b/docs/content.en/docs/references/processors/_index.md @@ -16,6 +16,8 @@ Record-scope processors operate on the current record inside [`for_each`](for_ea | [`echo`](echo/) | Logs a configured message. | | [`dag`](dag/) | Executes a DAG of processors with parallel and dependency-based execution. | | [`for_each`](for_each/) | Splits a batch into records, runs a sub-chain per record, re-encodes the batch. | +| [`if`](if/) | Conditional branch: matching records run the `then` sub-chain, the rest the optional `else`. | +| [`case`](case/) | Ordered multi-way dispatch: the first matching branch runs, with an optional `default` chain. | ## Queue diff --git a/docs/content.en/docs/references/processors/case.md b/docs/content.en/docs/references/processors/case.md new file mode 100644 index 00000000..7f196c8b --- /dev/null +++ b/docs/content.en/docs/references/processors/case.md @@ -0,0 +1,100 @@ +--- +title: "case" +weight: 208 +--- + +# case + +| Category | Scope | +|----------|-------| +| Routing | record (sub-chain host) | + +Multi-way dispatch: the block declares an ordered list of `branches`, the **first** branch whose match holds runs its `then` sub-chain, and exactly one arm executes. An optional `default` chain handles records no branch matched. It replaces the `if`/`then`/`else` ladder once the same input has to route to three or more destinations. Unlike [`if`](if/), everything — `field`, `branches`, `default` — is nested under the `case` key. + +A branch matches on an enumerated value set (`values`, compared against the block's `field`), on a full [condition](../conditions/) (`when`), or on both (AND). Conditions use the same DSL as `if`. + +## Structure + +```yaml +- case: + field: # optional; required when any branch uses `values` + branches: + - values: [, ] # match: field value equals any entry + then: + - ... + - when: + # match: any condition operator + then: + - ... + default: # optional; runs when no branch matched + - ... +``` + +| Key | Description | +|-----|-------------| +| `field` | Field read for `values` matching (record field inside a per-record sub-chain, `_ctx.`-prefixed context field at pipeline level). Required when any branch declares `values`; omit it for a pure condition ladder. | +| `branches` | Ordered list of arms; at least one is required. Evaluation stops at the first match. | +| `branches[].values` | Match when the `field` value equals any entry — the same typed membership test as the [`in`](../conditions/in/) condition. | +| `branches[].when` | Match when the condition holds (`equals`, `regexp`, `range`, `and`, ...). | +| `branches[].then` | Sub-chain for a matching record — a processor or a list of processors, required on every branch. | +| `default` | Optional sub-chain for records no branch matched. Without it, unmatched records pass through unchanged. | + +Each branch needs `values` or `when`; declaring both means both must hold. Compile validation rejects a `case` with no branches, a branch without `then`, and a `values` branch in a block that declares no `field`. + +## Example + +Route by severity, with a prefix match for the `warn*` variants and a catch-all default: + +```yaml +- case: + field: log_level + branches: + - values: [error, fatal] + then: + - mutate: + add: + route: alerting + - when: + prefix: + log_level: warn + then: + - mutate: + add: + route: warnings + default: + - mutate: + add: + route: archive +``` + +Branches without `values` do not need `field`, so the block also works as an ordered condition ladder: + +```yaml +- case: + branches: + - when: + and: + - equals: + env: prod + - regexp: + file.path: '\.json$' + then: + - json: + field: message + - when: + exists: + - file.path + then: + - dissect: + field: message + pattern: "%{ts} %{level} %{msg}" +``` + +## Notes + +- Exactly one arm runs per record; after it completes the event returns to the main chain. A `terminal` processor inside an arm (e.g. [`drop_event`](drop_event/)) ends the event instead. +- No match and no `default` is not an error — the record continues down the main chain untouched. +- Value resolution follows `if`: inside a per-record sub-chain ([`for_each`](for_each/), [`lane_consumer`](lane_consumer/)) arms see the current record's fields; at pipeline level they read the context through the `_ctx.` prefix. +- Arm chains compile under the enclosing scope, so record-scope contracts keep applying inside nested chains and batch-only processors stay rejected there, exactly as for `if`. +- The arm that ran is recorded in the flow trace as `case:1`, `case:2`, ..., `case:default`, or `case:no-match`. +- Sub-chain processors must be safe for concurrent `Process` calls when the surrounding `for_each` runs with `parallel > 1`. From 55b5f51fe311adffdd92755b96c0c1d79ba29432 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 17:41:24 +0800 Subject: [PATCH 35/65] docs: keep code comments in English Translate the Chinese comments that rode along in the PR's commits (workflow notes, ErrEmptyQueue, SPA cache policy, pipeline task teardown, queue consumer management, lane input config, configs sync tests, timestamp/mirror promotion notes) and swap the Chinese example display name in the instance Alias doc for an English one. No behavior change. --- .github/workflows/unit_test.yml | 8 +++++--- core/model/instance.go | 2 +- core/queue/api.go | 5 +++-- core/vfs/fs.go | 14 ++++++++------ modules/configs/config/config_test.go | 8 +++++--- modules/configs/server/server_test.go | 12 ++++++++---- modules/pipeline/tasks.go | 11 ++++++----- modules/queue/api.go | 6 ++++-- .../elastic/indexing_merge/indexing_merge_test.go | 11 +++++++---- plugins/elastic/lane_bulk/lane_bulk_test.go | 7 ++++--- plugins/elastic/lane_bulk/merge_to_bulk_lane.go | 13 ++++++++----- 11 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 44888f48..dcddcb6d 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -69,8 +69,10 @@ jobs: check-latest: false cache: true - # enterprise 插件 overlay (私有仓库): 优先取与本 PR 同名的分支, - # 不存在时回退默认分支; license 模块是 overlay 的编译依赖。 + # Enterprise plugins overlay (private repo): prefer the branch + # matching this PR's branch name, fall back to the default branch + # when it does not exist there; the license module is a build + # dependency of the overlay. - name: Checkout enterprise plugins overlay env: CI_GH_TOKEN: ${{ secrets.CI_TOKEN || secrets.DOCS_DEPLOYMENT_TOKEN }} @@ -108,7 +110,7 @@ jobs: mkdir -p $HOME/go/src/ ln -s $GITHUB_WORKSPACE $WORKBASE - # overlay 的 umbrella 包依赖私有模块 infini.sh/license (不进 go.mod) + # the overlay's umbrella packages need the private infini.sh/license module (kept out of go.mod) cd $WORK go mod edit -replace infini.sh/license=../license diff --git a/core/model/instance.go b/core/model/instance.go index 7f652d85..f0aa74ab 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -65,7 +65,7 @@ type Instance struct { // registration/heartbeat upserts preserve the stored value. Groups []string `json:"groups,omitempty" elastic_mapping:"groups:{type:keyword}"` - // Alias is a SERVER-OWNED display name (UI-managed, e.g. "边缘网关-生产"): + // Alias is a SERVER-OWNED display name (UI-managed, e.g. "edge-gateway-prod"): // shown in the management UI in place of the auto-generated instance // name. Like Groups, heartbeat upserts preserve the stored value. Alias string `json:"alias,omitempty" elastic_mapping:"alias:{type:keyword,copy_to:search_text}"` diff --git a/core/queue/api.go b/core/queue/api.go index 3f3fd41e..bc13e7f1 100755 --- a/core/queue/api.go +++ b/core/queue/api.go @@ -38,8 +38,9 @@ import ( "infini.sh/framework/core/util" ) -// ErrEmptyQueue 队列尚无任何数据 (无写段) — 浏览/取样场景按"无消息"处理, -// 不作为错误。DiskQueue 高级处理器在 AcquireConsumer 时返回。 +// ErrEmptyQueue: the queue holds no data yet (no written segment) — +// browse/sample flows treat it as "no messages", not an error. Returned by +// the DiskQueue advanced processor on AcquireConsumer. var ErrEmptyQueue = errors.New("empty queue") type QueueAPI interface { diff --git a/core/vfs/fs.go b/core/vfs/fs.go index 4d6c59bf..69ae366c 100755 --- a/core/vfs/fs.go +++ b/core/vfs/fs.go @@ -604,14 +604,16 @@ func serveFile(w http.ResponseWriter, r *http.Request, fs http.FileSystem, name serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) } -// vfsAssetHashRE 匹配前端构建产物文件名里的内容哈希段（如 -// index-DPak5aSO.js、pizza_engine_bg-B2pSYpQe.wasm）。 +// vfsAssetHashRE matches the content-hash segment in built frontend asset +// names (index-DPak5aSO.js, pizza_engine_bg-B2pSYpQe.wasm). var vfsAssetHashRE = regexp.MustCompile(`-[A-Za-z0-9_]{8,}\.[A-Za-z0-9]+$`) -// spaCacheControl SPA 静态资源缓存策略：/assets/ 下带内容哈希的构建产物 -// 内容永不原地变更，标记 immutable 让浏览器跳过重复校验（大 bundle 场景 -// 二次访问收益明显）；HTML 入口必须始终再验证，保证发版后能拉到新 hash -// 的资源清单。返回空串表示不干预（沿用 Last-Modified 协商缓存）。 +// spaCacheControl is the cache policy for SPA static assets: hashed build +// outputs under /assets/ never change in place, so they are marked +// immutable to let browsers skip revalidation (a big win on repeat visits +// with large bundles); the HTML entry must always revalidate so a release +// picks up the asset manifest with the new hashes. An empty return means +// no intervention (Last-Modified negotiation as usual). func spaCacheControl(p string) string { if strings.HasPrefix(p, "/assets/") && vfsAssetHashRE.MatchString(p) { return "public, max-age=31536000, immutable" diff --git a/modules/configs/config/config_test.go b/modules/configs/config/config_test.go index 3d189095..eb38875c 100644 --- a/modules/configs/config/config_test.go +++ b/modules/configs/config/config_test.go @@ -78,15 +78,17 @@ func TestSaveConfigStr(t *testing.T) { } } -// TestStripManagedMarkersSymmetry — 客户端剥离标记行后的哈希必须与服务端 -// (原始行内容 TrimSpace) 一致, 否则每个同步周期都判定内容变化、重写文件。 +// TestStripManagedMarkersSymmetry — the hash computed after the client +// strips the marker lines must equal the server-side one (raw lines, +// TrimSpace'd); otherwise every sync cycle would see a content change and +// rewrite the file. func TestStripManagedMarkersSymmetry(t *testing.T) { row := "{\"pipeline\":[{\"name\":\"x\"}]}" stamped := row + "\n#MANAGED_CONFIG_VERSION: 15\n#MANAGED: true" if got := stripManagedMarkers(stamped); got != row { t.Fatalf("strip = %q, want %q", got, row) } - // 标记粘行 (历史叠加): 整行剔除 + // marker glued onto the row (historic stacking): the whole line goes glued := row + "\n#MANAGED: trueMANAGED_CONFIG_VERSION: 15" if got := stripManagedMarkers(glued); got != row { t.Fatalf("strip glued = %q, want %q", got, row) diff --git a/modules/configs/server/server_test.go b/modules/configs/server/server_test.go index 2834ee4a..7d093868 100644 --- a/modules/configs/server/server_test.go +++ b/modules/configs/server/server_test.go @@ -119,7 +119,9 @@ func TestConfigsHash(t *testing.T) { if h1 != h2 { t.Fatal("hash must be stable") } - // 顺序无关不应成立? 协议要求两侧列表一致 — 同序序列化, 顺序变化视为变更 + // order-insensitivity must NOT hold: the protocol requires both sides + // to carry the same list — same-order marshaling, so a reorder counts + // as a change reordered := []common.ConfigFile{files[1], files[0]} if ConfigsHash(reordered) == h1 { t.Log("note: hash is order-sensitive (both sides marshal the same list)") @@ -226,9 +228,11 @@ func TestHashTokenOneWay(t *testing.T) { } } -// TestDiffConfigs_HashAware — 内容哈希比较 (新客户端): 跨行版本号不可比 -// (实例变体 v1 vs 客户端持有的基础行 v12), 哈希不同即更新; 哈希相同则即使 -// 版本号更小也不重发。旧客户端 (无哈希) 回退版本比较。 +// TestDiffConfigs_HashAware — content-hash comparison (new clients): +// version numbers are not comparable across rows (an instance variant at +// v1 vs the base row at v12 held by the client); a different hash means +// update, and an equal hash suppresses the resend even when the version is +// smaller. Legacy clients (no hash) fall back to version comparison. func TestDiffConfigs_HashAware(t *testing.T) { base := cfg("s.yml", 12) base.Hash = "hash-base" diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index c1824dac..9b5161fa 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -172,11 +172,12 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req * id := ps.ByName("id") _, exists := module.contexts.Load(id) if exists { - // 先取消再删表: 管线 goroutine 阻塞在 Process 内部 (consumer 工 - // 作循环等 IsCanceled / lane 处理器等 ctx.Done), 只清映射表不取消 - // 会让 goroutine 永远活着 — 任务注册表停留在 STARTED (僵尸"运行 - // 中")。与 auto_reload 路径同款: 取消并等释放 (≤1min), 超时也强制 - // 删表。 + // Cancel before deleting the registry entry: the pipeline goroutine + // is blocked inside Process (consumer worker loops on IsCanceled, + // lane processors on ctx.Done) — clearing the map alone would leave + // the goroutine alive forever and the task registry stuck at + // STARTED (a zombie "running" task). Same as the auto_reload path: + // cancel and wait for release (≤1min), force the delete on timeout. module.stopAndWaitForRelease([]string{id}, time.Minute) module.deleteTask(id) module.WriteAckOKJSON(w) diff --git a/modules/queue/api.go b/modules/queue/api.go index 5f56f516..22bf0473 100644 --- a/modules/queue/api.go +++ b/modules/queue/api.go @@ -74,7 +74,8 @@ func init() { //reset consumer offset api.HandleAPIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset) api.HandleUIMethod(api.PUT, "/queue/:id/consumer/:consumer_id/offset", module.QueueResetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueUpdate)) - // 消费者管理: 位点读取与消费者删除 (残留消费组/位点清理, Console 同款)。 + // Consumer management: offset read and consumer delete (clearing stale + // consumer groups/offsets, same as Console). api.HandleAPIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset) api.HandleUIMethod(api.GET, "/queue/:id/consumer/:consumer_id/offset", module.QueueGetConsumerOffset, api.RequireLogin(), api.RequirePermission(security.PermissionSystemQueueRead)) api.HandleAPIMethod(api.DELETE, "/queue/:id/consumer/:consumer_id", module.QueueDeleteConsumerByID) @@ -337,7 +338,8 @@ func (module *API) QueueExplore(w http.ResponseWriter, req *http.Request, ps htt result := util.MapStr{} status := 200 if err != nil { - // 空队列 (尚无写入) 是浏览/取样的常态, 按"无消息"返回而非 500 + // an empty queue (nothing written yet) is normal for + // browse/sample: answer "no messages" instead of a 500 if stderrors.Is(err, queue1.ErrEmptyQueue) { result["empty"] = true err = nil diff --git a/plugins/elastic/indexing_merge/indexing_merge_test.go b/plugins/elastic/indexing_merge/indexing_merge_test.go index 12da9df4..c920d1c9 100644 --- a/plugins/elastic/indexing_merge/indexing_merge_test.go +++ b/plugins/elastic/indexing_merge/indexing_merge_test.go @@ -31,8 +31,10 @@ func TestNormalizeDataStreamDoc(t *testing.T) { t.Fatalf("payload key should not survive: %v", doc) } - // @timestamp: payload 无显式 timestamp 时 observed_timestamp 优先于信封级。 - // @timestamp 是唯一真实时间戳 — legacy "timestamp" mirror 不再写出。 + // @timestamp: when the payload carries no explicit timestamp, + // observed_timestamp wins over the envelope-level one. @timestamp is + // the only real timestamp — the legacy "timestamp" mirror is no longer + // written. if doc["@timestamp"] != "2026-08-23T14:38:32.982744Z" { t.Fatalf("@timestamp = %v", doc["@timestamp"]) } @@ -40,8 +42,9 @@ func TestNormalizeDataStreamDoc(t *testing.T) { t.Fatalf("timestamp mirror must not be written: %v", doc["timestamp"]) } - // metadata promotion: file origin 不再提升为顶层 "file" (采集溯源由 - // @agent.file_* 承载), 仅 log_type/log_kind 标签提升。 + // metadata promotion: file origin no longer promotes to a top-level + // "file" (collection provenance lives on @agent.file_*); only the + // log_type/log_kind labels promote. if _, ok := doc["file"]; ok { t.Fatalf("top-level file promotion is gone: %v", doc["file"]) } diff --git a/plugins/elastic/lane_bulk/lane_bulk_test.go b/plugins/elastic/lane_bulk/lane_bulk_test.go index 5a1f38bc..d4b9e163 100644 --- a/plugins/elastic/lane_bulk/lane_bulk_test.go +++ b/plugins/elastic/lane_bulk/lane_bulk_test.go @@ -221,12 +221,13 @@ func TestNormalizeDataStreamDoc(t *testing.T) { if m["@timestamp"] != "2026-10-15T00:00:00Z" { t.Errorf("@timestamp = %v", m["@timestamp"]) } - // @timestamp 是唯一真实时间戳: payload 里的 timestamp 折叠进 - // @timestamp 后删除, 不再写顶层 mirror。 + // @timestamp is the only real timestamp: the payload's timestamp folds + // into @timestamp and is dropped — no top-level mirror is written. if _, ok := m["timestamp"]; ok { t.Errorf("timestamp mirror must not survive: %v", m["timestamp"]) } - // file origin 不再提升为顶层 "file" (采集溯源由 @agent.file_* 承载) + // file origin no longer promotes to a top-level "file" + // (collection provenance lives on @agent.file_*) if _, ok := m["file"]; ok { t.Errorf("top-level file promotion is gone: %v", m["file"]) } diff --git a/plugins/elastic/lane_bulk/merge_to_bulk_lane.go b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go index dcfa996c..a1ff6d06 100644 --- a/plugins/elastic/lane_bulk/merge_to_bulk_lane.go +++ b/plugins/elastic/lane_bulk/merge_to_bulk_lane.go @@ -89,9 +89,11 @@ func (processor *MergeToBulkLaneProcessor) Name() string { type MergeConfig struct { InputQueue string `config:"input_queue"` - // InputLane 内存输入 (与 InputQueue 二选一): 记录为 event.Event, - // 序列化其 Fields 为文档字节后走同一聚合路径 — 上游 output{lane} - // 直写, 全程内存 (staging 队列是唯一落盘点)。 + // InputLane is the in-memory input (mutually exclusive with InputQueue): + // records arrive as event.Event, their Fields are serialized into + // document bytes and take the same aggregation path — written directly + // by an upstream output{lane}, in memory end to end (the staging queue + // is the only disk landing point). InputLane string `config:"input_lane"` OutputLane string `config:"output_lane"` IndexName string `config:"index_name"` @@ -247,8 +249,9 @@ func (processor *MergeToBulkLaneProcessor) newBulkWorker(ctx *pipeline.Context, var lastCommit time.Time = time.Now() - // 输入源: input_lane 时订阅 lane, 序列化 Fields 为文档字节; - // input_queue 时直接 PopTimeout 磁盘队列 (原路径)。 + // Input source: with input_lane, subscribe to the lane and serialize + // Fields into document bytes; with input_queue, PopTimeout the disk + // queue directly (the original path). docs := make(chan []byte, 256) var laneCancel func() if processor.config.InputLane != "" { From 82e5fcc8677762a28cc2f249829e416d88be364d Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 21 Oct 2026 17:56:46 +0800 Subject: [PATCH 36/65] docs: drop the README pipeline section in favor of release notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The developer notes that had accumulated in README.md move out — the deep API detail already lives in the reference docs (pipeline record, validation gate, deployment APIs, processor pages), and the repo README stays minimal again (restored to main's version). Release notes gain entries for everything the October pipeline branch ships: the unified engine foundation (contracts, codecs, lanes, step stats), the compile-time validation gate (also noted under breaking changes), distributed deployment management with the rollback gate, the lane-based bulk indexing pair, the case processor, per-instance config shadowing, disk-queue consumer self-heal, and the output/loopback/agent fixes. --- README.md | 49 +------------------- docs/content.en/docs/release-notes/_index.md | 11 +++++ 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 0ebb0ba3..857d7142 100755 --- a/README.md +++ b/README.md @@ -25,51 +25,4 @@ make build # production build make build-dev # development build (includes -tags dev) ``` -No `GOPATH` manipulation or vendor repository checkout is needed. Standard `go build` works directly. -## Pipeline Engine (v2) — Developer Notes - -The record-processing engine lives in `core/pipeline` (interfaces, registry, -contracts, codecs, observability) and `modules/pipeline` (the PipeModule -runtime host and `for_each`). Key APIs added by the unified-pipeline work: - -### Record codecs (`core/pipeline/codec.go`) - -`RecordCodec` converts queue payload bytes to/from `*event.Event` and now -declares the record family it produces via `Namespace()` ("document", -"attachment", "" for generic). Registration accepts runtime callers -(`RegisterRecordCodec` returns an error; `MustRegisterRecordCodec` keeps the -init-time panic), `RegisterRecordCodecOrReplace` upgrades in place -(version-incremented), and `ListRecordCodecs` feeds discovery APIs. -`modules/pipeline` re-exports the API for compatibility. **Domain codec -modules should import only `core/pipeline`** to keep their dependency -surface light (see `infini.sh/coco-codecs` for the reference layout). - -### Processor contracts (`core/pipeline/contract.go`) - -Processors may implement `ContractProvider` to declare where they may run: - -- `Scope` (batch / record / any-zero-value) and `Namespace` are enforced at - **compile time** by `NewPipelineWithScope(cfg, scope, direct)` — mismatches - error under `ScopeInfo.Strict`, warn otherwise. Plain `NewPipeline` keeps - legacy behavior (batch scope, warnings only). -- `Granularity` (record / batch_capable / batch_only): stage-wise execution - means written order IS execution order; `batch_only` processors must be - direct members of a record host's `processor:` list (nesting under - `if/then` is a compile error). -- `CheckContract(ctx, contract)` is the runtime guard for legacy chains. - -### Stage-wise execution (`modules/pipeline/for_each.go`) - -`for_each` sub-chains execute as alternating segments (record runs / one -batch call) in written order: a batch processor sees the edits of record -processors declared before it; early-segment drops exclude records from all -later segments; clones join the active set and flow through later segments. -A batch-segment failure with `on_failure: tag` tags every live record. - -### Execution observers (`core/pipeline/observer.go`) - -`ExecutionObserver` receives `StepEvent`s (pipeline, step, processor, scope, -duration, outcome, trace id) from the chain loop — one instrumentation point -covering both batch chains and record sub-chains (scope is derived from the -current record). The empty registry is a zero-allocation fast path. Record -trace ids ride `Metadata.Labels["_trace_id"]` via `SetTraceID`/`TraceID`. +No `GOPATH` manipulation or vendor repository checkout is needed. Standard `go build` works directly. \ No newline at end of file diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 7b612786..5a90f083 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -11,7 +11,15 @@ Information about release notes of INFINI Framework is provided here. ## Latest (In development) ### ❌ Breaking changes - `ReadGuard` rules now interpret glob metacharacters. Literal names containing these characters require escaping or singleton character groups. On Unix, literal backslashes also require escaping. #434 +- pipeline configs are now validated at compile time before they are persisted or distributed: unknown processors, missing required fields, constructor errors and scope-violating sub-chains are rejected up front instead of surfacing at runtime, and every registered processor constructor is audited to return an error (never panic) on an empty config. #433 ### 🚀 Features +- feat(pipeline): unified pipeline engine foundation — record contracts declared per processor (`ContractProvider` with `Scope`/`Namespace`/`Granularity`, enforced by `NewPipelineWithScope`, `batch_only` processors restricted to direct members of record hosts), payload-agnostic record codecs with namespace declaration and in-place upgrade registration (`RegisterRecordCodecOrReplace`, `ListRecordCodecs`), in-memory lanes with self-healing `lane.GetOrCreate`, and per-step stats (`Context.StepStat`) for sequential runs feeding document-level timelines. #433 +- feat(pipeline): distributed deployment management — `PipelineConfigV2`/`PipelineDeployment` ORM models and a deployment reconciler that pushes configs to instances through configs-center: instance-targeted selectors with exact per-instance convergence, disabling a deployment propagates to instances, deleting one cascades config recall, name-based status matching, and instance runtime reporting in `POST /configs/_sync`. #433 +- feat(pipeline): pipeline rollback gate — a config distribution that fails on an instance rolls the deployment back instead of leaving instances half-applied, with instance-side failure visibility in the deployment status. #433 +- feat(pipeline): lane-based bulk indexing, the memory-path twin of the disk-queue bulk pipeline — `merge_to_bulk_lane` aggregates queue/lane input into combined bulk payloads written to an in-memory lane, `bulk_indexing_lane` ships them to Elasticsearch and settles the batch receipt on success. #433 +- feat(pipeline): add the `case` processor — ordered multi-way dispatch for enumerated routing: the first matching branch runs (exactly one arm executes) with an optional `default` chain, unmatched records pass through unchanged when no `default` is set; arms match on `values` (typed `in`-style membership against the block's `field`) or on any condition (`when`), combinable with AND, and evaluate against the current record inside per-record sub-chains. #433 +- feat(configs): per-instance config rows shadow group defaults, enabling edge-rule variants per instance without forking the group config; heartbeat upserts preserve server-owned display metadata (`alias`) and group assignments. #433 +- feat(queue): disk-queue consumer self-heal — offsets parked at a missing write head re-open the segment (or keep waiting for it) instead of erroring forever, so consumers closing their lane on exit no longer wedge producers. #433 - feat: support glob rules in `ReadGuard` to authorize matching directories without enumerating the filesystem. Rules support relative paths and path component cleanup, with documented escaping and symbolic-link behavior. #434 - feat: add MCP (Model Context Protocol) server support — auto-expose UI routes as MCP tools via `api.MCPTool()` annotation - feat: add MCP configuration section (`web.mcp.enabled`, `web.mcp.base_path`, `web.mcp.stateless`, `web.mcp.disable_streaming`) @@ -40,6 +48,9 @@ Information about release notes of INFINI Framework is provided here. - perf(pipeline): probe the current record via `Parameters.Peek` — the `GetValue` miss path built a stack-capturing error for every `CurrentRecord` call outside a record scope (µs + 6 allocs per if/then/else evaluation); the bare locked map read restores the no-record path to pre-fix allocation counts #408 ### 🐛 Bug fix +- fix(pipeline): the unified `output` processor accepts a bare queue name or a full config object. #433 +- fix(agent): mint the reverse-channel loopback API token agents were missing, so agent-initiated loopback calls authenticate again. #433 +- fix(agent): skip the deployment reconcile loop and guard ORM access on lightweight agent builds that carry no metadata store. #433 - fix(queue): `disk_queue.max_used_bytes` now counts only `data/queue`, so unrelated application data no longer blocks queue writes or prevents startup. #431 - fix: expand configs.template when loading templated config files #391 - fix: register elasticsearch instance even when version probe fails #393 From c4ebe4a8496dcc3a21abc0210d1f1d92476db66f Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 22 Oct 2026 16:14:34 +0800 Subject: [PATCH 37/65] feat(orm): body-level parameters on the query builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Query builders could only place parameters inside the bool clause, so body-root settings like min_score had no home — nestling them into the clause makes Elasticsearch reject the search with "unknown field [min_score]". QueryBuilder gains BodyParameter(key, val): values recorded there are rendered by the Elasticsearch DSL builder at the body root, next to query/from/size, in BuildQueryDSL — and through the OnTopOfDSL paths, where the explicit parameter also beats a raw request-body key of the same name (plain Parameter keeps feeding the clause unchanged). Needed by coco's document search quality loop, which sets a search-body-level min_score for semantic/hybrid retrieval. --- core/orm/query.go | 23 ++++++++++++++++ modules/elastic/orm/query_dsl.go | 8 ++++++ modules/elastic/orm/query_dsl_test.go | 38 +++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/core/orm/query.go b/core/orm/query.go index 28001cd8..5ce499a8 100644 --- a/core/orm/query.go +++ b/core/orm/query.go @@ -120,6 +120,11 @@ type QueryBuilder struct { allowRequestBodyBytes bool requestBodyBytes []byte Aggs map[string]Aggregation + + // body-level search parameters (min_score, track_total_hits, ...) — + // rendered at the DSL body root by the Elasticsearch builder, see + // BodyParameter + bodyParams map[string]interface{} } func NewQuery() *QueryBuilder { @@ -209,6 +214,24 @@ func (q *QueryBuilder) Parameter(key param.ParaKey, val interface{}) *QueryBuild return q } +// BodyParameter records a top-level search-body parameter (min_score, +// track_total_hits, ...). Unlike Parameter, whose keys feed the bool clause, +// these are rendered by the Elasticsearch DSL builder at the body root next +// to query/from/size, overriding raw request-body keys of the same name. +func (q *QueryBuilder) BodyParameter(key string, val interface{}) *QueryBuilder { + if q.bodyParams == nil { + q.bodyParams = map[string]interface{}{} + } + q.bodyParams[key] = val + return q +} + +// BodyParametersVal returns the recorded body-level parameters (nil when +// none were set). +func (q *QueryBuilder) BodyParametersVal() map[string]interface{} { + return q.bodyParams +} + func (q *QueryBuilder) DefaultQueryField(field ...string) *QueryBuilder { q.defaultQueryFields = append(q.defaultQueryFields, field...) return q diff --git a/modules/elastic/orm/query_dsl.go b/modules/elastic/orm/query_dsl.go index 67d8ed60..325a3020 100644 --- a/modules/elastic/orm/query_dsl.go +++ b/modules/elastic/orm/query_dsl.go @@ -242,6 +242,14 @@ func BuildQueryDSL(q *orm.QueryBuilder) map[string]interface{} { dsl["aggs"] = aggs } + // body-level parameters land at the DSL root next to query/from/size; + // in the OnTopOfDSL paths this runs inside the override map, so an + // explicit BodyParameter also beats a raw request-body key of the same + // name + for k, v := range q.BodyParametersVal() { + dsl[k] = v + } + return dsl } diff --git a/modules/elastic/orm/query_dsl_test.go b/modules/elastic/orm/query_dsl_test.go index 62ac7922..cd0e4922 100644 --- a/modules/elastic/orm/query_dsl_test.go +++ b/modules/elastic/orm/query_dsl_test.go @@ -1652,3 +1652,41 @@ func TestToDSL_NestedQuery_SimpleMatch(t *testing.T) { actual, _ := json.Marshal(dsl) assert.JSONEq(t, expected, string(actual)) } + +func TestBuildQueryDSL_BodyParameter(t *testing.T) { + q := orm.NewQuery(). + Must(orm.MatchQuery("title", "elastic")). + BodyParameter("min_score", 0.35) + + dsl := BuildQueryDSL(q) + + // body-level parameter must sit at the DSL root, next to query — + // not inside the bool clause where ES rejects it with + // "unknown field [min_score]" + // a single must clause flattens to a bare match (flattenBoolClauses) + expected := `{ + "query": { + "match": {"title": {"query": "elastic"}} + }, + "min_score": 0.35 + }` + actual, _ := json.Marshal(dsl) + assert.JSONEq(t, expected, string(actual)) +} + +func TestBuildQueryDSLOnTopOfDSL_BodyParameterOverridesRawBody(t *testing.T) { + q := orm.NewQuery(). + Must(orm.MatchQuery("title", "elastic")). + BodyParameter("min_score", 0.35) + + raw := []byte(`{"query": {"match_all": {}}, "min_score": 0.05, "size": 20}`) + dsl := BuildQueryDSLOnTopOfDSL(q, raw) + + // the explicit BodyParameter beats the raw request-body key + if v, ok := dsl["min_score"].(float64); !ok || v != 0.35 { + t.Fatalf("min_score = %v, want 0.35", dsl["min_score"]) + } + if dsl["size"] != float64(20) { + t.Fatalf("size = %v, want 20 (raw body keys survive unless overridden)", dsl["size"]) + } +} From f2c2ee03088f306d5a5911ac3f2f75f51c95e417 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 22 Oct 2026 16:42:48 +0800 Subject: [PATCH 38/65] fix(pipeline): explicit PipelineConfigV2 mappings; replay error log fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PipelineConfigV2 ORM mappings: Name becomes a keyword field so the pipeline list API can term-query and sort on it; Processors disables indexing — processor configs are arbitrary per-processor JSON and dynamic mapping would risk field-type conflicts between pipelines sharing the index. - replay: req/res Reset moves from execute's defer to the callers — execute logs the request when it fails, and must do so while the request is still intact instead of after the deferred reset emptied it. --- core/pipeline/config.go | 8 ++++++-- plugins/replay/replay.go | 14 ++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core/pipeline/config.go b/core/pipeline/config.go index c539ceda..2471fcf7 100644 --- a/core/pipeline/config.go +++ b/core/pipeline/config.go @@ -44,7 +44,8 @@ type PipelineConfigV2 struct { // Human-readable identifier for this pipeline. // Required when creating a pipeline via the API. - Name string `config:"name" json:"name,omitempty"` + // Keyword so the pipeline list API can term-query and sort on it. + Name string `config:"name" json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` // A disabled pipeline is treated as if it does not exist. Nil means // "use the module default". @@ -98,7 +99,10 @@ type PipelineConfigV2 struct { // Processors is the ordered chain of processor definitions that makes // up the actual work the pipeline performs. Each entry is a single // processor config keyed by processor type. - Processors []map[string]interface{} `config:"processor" json:"processor"` + // Indexing is disabled: processor configs are arbitrary per-processor + // JSON and would otherwise be dynamically mapped, risking field-type + // conflicts between pipelines on the same index. + Processors []map[string]interface{} `config:"processor" json:"processor" elastic_mapping:"processor: { enabled: false }"` // StrictContract turns record-contract mismatches into compile errors // instead of warnings. Absent = legacy warn-only (P8 flips the default). diff --git a/plugins/replay/replay.go b/plugins/replay/replay.go index 0d9855ef..5a760750 100644 --- a/plugins/replay/replay.go +++ b/plugins/replay/replay.go @@ -204,8 +204,12 @@ func ReplayLines(req *fasthttp.Request, res *fasthttp.Response, ctx *pipeline.Co err := execute(req, res, buffer) if err != nil { log.Error(err, req.String()) + req.Reset() + res.Reset() panic(err) } + req.Reset() + res.Reset() buffer.Reset() requestIsSet = false } @@ -256,18 +260,20 @@ func ReplayLines(req *fasthttp.Request, res *fasthttp.Response, ctx *pipeline.Co err := execute(req, res, buffer) if err != nil { log.Error(err, req.String()) + req.Reset() + res.Reset() panic(err) } + req.Reset() + res.Reset() } return count, nil, false } func execute(req *fasthttp.Request, res *fasthttp.Response, buffer *bytebufferpool.ByteBuffer) error { - defer func() { - req.Reset() - res.Reset() - }() + // reset is the caller's job: it logs the request when execute fails, + // and must do so while the request is still intact. if buffer.Len() > 0 { if util.ContainStr(string(req.Header.RequestURI()), "_bulk") { buffer.WriteString(newline) From 2051f0fb1791baba012ea57ed421c6ce740ce0c5 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 23 Oct 2026 10:44:03 +0800 Subject: [PATCH 39/65] feat(pipeline): count telemetry loss instead of dropping silently Firecracker-style loss accounting for the observability buffers: losing telemetry under backpressure is fine, losing track of how much was lost is not. - flight recorder: sampled_out counts OK steps skipped by the 1/N sampling, evicted counts entries pushed out of a full ring (both atomic, cumulative since process start); the flight endpoint returns them in a `lost` block next to the events. - metrics exporter: persist_failures counts stage-snapshot rows a failed orm save dropped; the stages endpoint surfaces it as persist_errors so a non-zero value marks holes in the persisted history. --- modules/pipeline/flight.go | 35 ++++++++++++++++++++++++++++- modules/pipeline/flight_test.go | 33 +++++++++++++++++++++++++++ modules/pipeline/metrics_persist.go | 8 +++++++ modules/pipeline/tasks.go | 14 +++++++++--- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go index 86bc9e15..499191ab 100644 --- a/modules/pipeline/flight.go +++ b/modules/pipeline/flight.go @@ -1,6 +1,6 @@ /* Copyright © INFINI Ltd. All rights reserved. * Web: https://infinilabs.com - * Email: hello#infinilabs.com */ + * Email: hello#infini.ltd */ package pipeline @@ -22,10 +22,17 @@ const flightSample = 100 // It installs globally at PipeModule setup — the empty-registry fast // path applies before installation, and the per-step cost after is one // counter increment plus the sampled append. +// Loss is counted, never silent: sampledOut tracks OK steps skipped by +// the 1/N sampling, evicted tracks entries pushed out of a full ring — +// both surfaced through the flight endpoint so consumers know how much +// of the picture they are not seeing. type flightRecorder struct { mu sync.Mutex rings map[string][]pipeline.StepEvent seq int64 + + sampledOut atomic.Int64 + evicted atomic.Int64 } func newFlightRecorder() *flightRecorder { @@ -37,6 +44,7 @@ func (f *flightRecorder) OnStep(e *pipeline.StepEvent) { if !keep { n := atomic.AddInt64(&f.seq, 1) if n%flightSample != 0 { + f.sampledOut.Add(1) return } } @@ -44,12 +52,37 @@ func (f *flightRecorder) OnStep(e *pipeline.StepEvent) { f.mu.Lock() ring := append(f.rings[e.Pipeline], cp) if len(ring) > flightRing { + f.evicted.Add(int64(len(ring) - flightRing)) ring = ring[len(ring)-flightRing:] } f.rings[e.Pipeline] = ring f.mu.Unlock() } +// FlightLostStats is the loss accounting of the flight recorder, +// cumulative since process start. +type FlightLostStats struct { + // OK steps skipped by the 1/N sampling (by design, still a cost + // consumers should see). + SampledOut int64 `json:"sampled_out"` + // Oldest entries pushed out of a full ring (oldest-first eviction). + Evicted int64 `json:"evicted"` +} + +// flightLostStats reads the recorder's loss counters; zero values when +// no recorder is installed. +func flightLostStats() FlightLostStats { + f := globalFlight.Load() + if f == nil { + return FlightLostStats{} + } + rec := f.(*flightRecorder) + return FlightLostStats{ + SampledOut: rec.sampledOut.Load(), + Evicted: rec.evicted.Load(), + } +} + // flightQuery filters a pipeline's ring: by trace id, errors-only, newest // first, capped at limit. func flightQuery(pipelineName, traceID string, errorsOnly bool, limit int) []pipeline.StepEvent { diff --git a/modules/pipeline/flight_test.go b/modules/pipeline/flight_test.go index 09da912c..6acd0357 100644 --- a/modules/pipeline/flight_test.go +++ b/modules/pipeline/flight_test.go @@ -47,3 +47,36 @@ func TestFlight_RingBounded(t *testing.T) { t.Fatalf("ring = %d, want %d", n, flightRing) } } + +// Loss must be counted, never silent: OK steps skipped by sampling and +// entries evicted from a full ring both land in the lost counters. +func TestFlight_LossCounters(t *testing.T) { + installFlight() + f := globalFlight.Load().(*flightRecorder) + + errsBefore := f.evicted.Load() + sampledBefore := f.sampledOut.Load() + + // 200 OK steps: all but ~2 are sampled out + for i := 0; i < 200; i++ { + f.OnStep(&pipeline.StepEvent{Pipeline: "loss", Outcome: pipeline.OutcomeOK}) + } + sampled := f.sampledOut.Load() - sampledBefore + if sampled < 190 || sampled > 198 { // ~99% of 200 + t.Fatalf("sampledOut = %d, want ~198", sampled) + } + + // errors only: two full rings worth — the older ring overflows + for i := 0; i < flightRing*2; i++ { + f.OnStep(&pipeline.StepEvent{Pipeline: "loss-ring", Outcome: pipeline.OutcomeError}) + } + if evicted := f.evicted.Load() - errsBefore; evicted != flightRing { + t.Fatalf("evicted = %d, want %d", evicted, flightRing) + } + + // stats surface through the package helper (handler reads this) + lost := flightLostStats() + if lost.SampledOut < sampled || lost.Evicted < flightRing { + t.Fatalf("flightLostStats = %+v, want counters >= this test's deltas", lost) + } +} diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index 0107187f..a19e93fd 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -5,6 +5,7 @@ package pipeline import ( + "sync/atomic" "time" log "github.com/cihub/seelog" @@ -28,6 +29,12 @@ type PipelineStageStat struct { Ts time.Time `json:"ts" elastic_mapping:"ts:{type:date}"` } +// metricsPersistFailures counts stage-snapshot rows the exporter failed +// to persist (orm save errors) — cumulative since process start. Losses +// here mean the persisted metric history has holes; surfaced via the +// stages endpoint next to the in-memory stats. +var metricsPersistFailures atomic.Int64 + // metricsPersistConfig reads pipeline_metrics.persistence. type metricsPersistConfig struct { Persistence struct { @@ -87,6 +94,7 @@ func (module *PipeModule) persistStageSnapshot() { } row.ID = util.GetUUID() if err := orm.Save(octx, &row); err != nil { + metricsPersistFailures.Add(1) log.Warnf("metrics persist: save %s/%s failed: %v", name, st.Stage, err) } } diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index 9b5161fa..ab2e4ebb 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -191,16 +191,24 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req * // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_start // stagesPipelineTaskHandler — GET /pipeline/task/:id/stages // Standard per-stage metrics: processor invocations/errors/latency plus -// live lane carriers (depth-derived). +// live lane carriers (depth-derived). persist_errors counts stage +// snapshot rows the metrics exporter failed to write — a non-zero value +// means the persisted history has holes. func (module *PipeModule) stagesPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") - module.WriteJSON(w, util.MapStr{"stages": stageStats(id)}, http.StatusOK) + module.WriteJSON(w, util.MapStr{ + "stages": stageStats(id), + "persist_errors": metricsPersistFailures.Load(), + }, http.StatusOK) } // flightPipelineTaskHandler — GET /pipeline/task/:id/flight // Sampled execution trace of a pipeline: ?trace_id= for one record's // journey, ?errors=1 for recent failures, &limit= (default 50, newest // first). Memory-only (ring); persistence rides the metrics exporter. +// The `lost` block counts what the view does not show: OK steps skipped +// by the 1/N sampling and entries evicted from the full ring (cumulative +// since process start). func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") q := req.URL.Query() @@ -211,7 +219,7 @@ func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req * } } events := flightQuery(id, q.Get("trace_id"), q.Get("errors") == "1", limit) - module.WriteJSON(w, util.MapStr{"events": events}, http.StatusOK) + module.WriteJSON(w, util.MapStr{"events": events, "lost": flightLostStats()}, http.StatusOK) } // pausePipelineTaskHandler — POST /pipeline/task/:id/_pause From 18633cc2561eb83d5752973e87157d3e936de1ee Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 23 Oct 2026 17:46:17 +0800 Subject: [PATCH 40/65] fix(sqlite): must_not over a missing field drops rows under SQL three-valued logic --- modules/sqlite/orm/query_builder.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/sqlite/orm/query_builder.go b/modules/sqlite/orm/query_builder.go index 7c9b04f4..5a65bb28 100644 --- a/modules/sqlite/orm/query_builder.go +++ b/modules/sqlite/orm/query_builder.go @@ -166,11 +166,16 @@ func clauseToSQL(clause *orm.Clause, resolve FieldResolver) (string, []interface } } - // must_not is combined with AND NOT + // must_not is combined with AND NOT. A predicate over a MISSING + // field yields NULL under SQL three-valued logic (NOT NULL = NULL = + // row dropped), but Elasticsearch semantics say a missing field never + // matches the inner term — so must_not passes and the row survives. + // IFNULL folds that NULL to false before the NOT, restoring the + // engine's contract for sparse fields. for _, sub := range clause.MustNotClauses { sql, args := clauseToSQL(sub, resolve) if sql != "" { - parts = append(parts, fmt.Sprintf("NOT (%s)", sql)) + parts = append(parts, fmt.Sprintf("NOT (IFNULL(%s, 0))", sql)) allArgs = append(allArgs, args...) } } From 95576cccee700c0b89a8abaf4ae37e86d3655614 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 01:58:38 +0800 Subject: [PATCH 41/65] fix: access_token native path sets the ORM model so ES-backed managers don't panic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CreateAPIToken/ListTokens/DeleteTokenByID built direct contexts without orm.WithModel; the elastic ORM's data-op hooks resolve the index from the ctx model and panicked 'model is not set'. SQLite-backed products (logpilot) resolve from the object and never saw it — minting an instance manager token on coco (native + elastic) died mid-approve. --- modules/security/access_token/authentication.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go index 4a905755..29560dfb 100644 --- a/modules/security/access_token/authentication.go +++ b/modules/security/access_token/authentication.go @@ -241,6 +241,11 @@ func CreateAPIToken(user *security.UserSessionInfo, tokenName, tokenDesc, typeNa ctx.DirectAccess() ctx.Refresh = orm.WaitForRefresh ctx.PermissionScope(security.PermissionScopePlatform) + // the elastic ORM's data-op hooks resolve the index from the ctx + // model — without this the create panics "model is not set" on any + // ES-backed manager (sqlite resolves from the object, so only + // native+elastic deployments hit it) + orm.WithModel(ctx, &accessToken) if err := orm.Create(ctx, &accessToken); err != nil { return nil, err @@ -618,6 +623,7 @@ func ListTokens() ([]*security.AccessToken, error) { if isNative() { ctx := orm.NewContext().DirectAccess() ctx.PermissionScope(security.PermissionScopePlatform) + orm.WithModel(ctx, &security.AccessToken{}) res, err := orm.SearchV2(ctx, orm.NewQuery().Size(1000)) if err != nil { return nil, err @@ -658,6 +664,7 @@ func DeleteTokenByID(tokenID string) error { if isNative() { ctx := orm.NewContext().DirectAccess() ctx.PermissionScope(security.PermissionScopePlatform) + orm.WithModel(ctx, &security.AccessToken{}) t := security.AccessToken{} t.ID = tokenID return orm.Delete(ctx, &t) From b6a3f0a2cea97d49356c28fed77232e1f705c743 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 02:39:31 +0800 Subject: [PATCH 42/65] fix(ucfg): empty dict reifies to empty map so its key survives unpack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A YAML/JSON value of {} reified to nil, and reflect.ValueOf(nil) is an invalid Value, so reifyMap skipped SetMapIndex and the KEY disappeared from any map[string]interface{} target. Config-file pipelines with empty processor bodies (processor: [{echo: {}}] — the most common shape) lost the processor name at load and failed with 'each processor must have exactly one action, but found 0 actions'; managed instances could never start deployed pipelines. Empty dicts now reify to empty maps, mirroring the existing empty-array behavior ([]interface{}{}). --- core/config/config_test.go | 44 +++++++++++++++++++++++++++++++++++ lib/go-ucfg/types.go | 8 ++++++- lib/go-ucfg/unpack_test.go | 37 +++++++++++++++++++++++++++++ lib/go-ucfg/yaml/yaml_test.go | 8 ++++++- 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/core/config/config_test.go b/core/config/config_test.go index 3f6d990a..e02ebbf4 100755 --- a/core/config/config_test.go +++ b/core/config/config_test.go @@ -971,3 +971,47 @@ func TestLoadFileExpandsConfigTemplatesPlain(t *testing.T) { assert.Equal(t, out.Entry[0].Name, "my_entry_test_entry") assert.Equal(t, out.Entry[0].Network.Binding, "127.0.0.1:8002") } + +// A processor entry with an empty body (`- echo: {}`) must keep its key +// through LoadPath + Unpack. Empty dicts used to reify to nil, which made +// the KEY vanish from map[string]interface{} targets and broke every +// config-file-loaded pipeline (managed instances saw "each processor must +// have exactly one action, but found 0 actions"). +func TestLoadPathKeepsEmptyProcessorKeys(t *testing.T) { + dir := t.TempDir() + yml := "pipeline:\n" + + " - name: t\n" + + " auto_start: false\n" + + " processor:\n" + + " - echo: {}\n" + + " - node_stamp:\n" + + " field: x\n" + if err := os.WriteFile(filepath.Join(dir, "p.yml"), []byte(yml), 0o600); err != nil { + t.Fatal(err) + } + + cfg, err := LoadPath(dir) + if err != nil { + t.Fatal(err) + } + child, err := cfg.Child("pipeline", -1) + if err != nil { + t.Fatal(err) + } + type rawPipeline struct { + Processor []map[string]interface{} `config:"processor"` + } + list := []rawPipeline{} + if err := child.Unpack(&list); err != nil { + t.Fatal(err) + } + if len(list) != 1 || len(list[0].Processor) != 2 { + t.Fatalf("unexpected unpack result: %#v", list) + } + if _, ok := list[0].Processor[0]["echo"]; !ok { + t.Fatalf("empty processor body lost its key: %#v", list[0].Processor[0]) + } + if _, ok := list[0].Processor[1]["node_stamp"]; !ok { + t.Fatalf("node_stamp lost its key: %#v", list[0].Processor[1]) + } +} diff --git a/lib/go-ucfg/types.go b/lib/go-ucfg/types.go index af3315d2..1c833f7d 100755 --- a/lib/go-ucfg/types.go +++ b/lib/go-ucfg/types.go @@ -376,7 +376,13 @@ func (c cfgSub) reify(opts *options) (interface{}, error) { // preserve empty arrays return []interface{}{}, nil case len(fields) == 0 && len(arr) == 0: - return nil, nil + // preserve empty dicts: returning nil here used to drop the KEY + // entirely when the parent unpacked into map[string]interface{} + // (reflect.ValueOf(nil) is invalid, so reifyMap skipped + // SetMapIndex) — pipeline configs like `processor: [{echo: {}}]` + // lost the processor name and failed to load. An empty dict must + // reify to an empty map, mirroring the empty-array case above. + return map[string]interface{}{}, nil case len(fields) > 0 && len(arr) == 0: m := make(map[string]interface{}) for k, v := range fields { diff --git a/lib/go-ucfg/unpack_test.go b/lib/go-ucfg/unpack_test.go index 6acf1074..f7d7dad1 100755 --- a/lib/go-ucfg/unpack_test.go +++ b/lib/go-ucfg/unpack_test.go @@ -237,3 +237,40 @@ func TestReifyUnpackersPtr(t *testing.T) { assertSubConfig(to.C.c) assertSubConfig(to.R.c) } + +// An empty dict value must keep its key when unpacked into a generic +// map[string]interface{}: `processor: [{echo: {}}]` losing `echo` broke +// every config-file-loaded pipeline with an empty processor body. +func TestUnpackPreservesEmptyDictKeys(t *testing.T) { + cfg, err := NewFrom(map[string]interface{}{ + "processor": []interface{}{ + map[string]interface{}{"echo": map[string]interface{}{}}, + map[string]interface{}{"node_stamp": map[string]interface{}{"field": "x"}}, + }, + }) + if err != nil { + t.Fatal(err) + } + + type raw struct { + Processor []map[string]interface{} `config:"processor"` + } + out := raw{} + if err := cfg.Unpack(&out); err != nil { + t.Fatal(err) + } + + if len(out.Processor) != 2 { + t.Fatalf("expected 2 processors, got %d", len(out.Processor)) + } + echoCfg, exists := out.Processor[0]["echo"] + if !exists { + t.Fatalf("empty dict key dropped: %v", out.Processor[0]) + } + if m, ok := echoCfg.(map[string]interface{}); !ok || len(m) != 0 { + t.Fatalf("expected echo to be an empty map, got %#v", echoCfg) + } + if _, exists := out.Processor[1]["node_stamp"]; !exists { + t.Fatalf("node_stamp key dropped: %v", out.Processor[1]) + } +} diff --git a/lib/go-ucfg/yaml/yaml_test.go b/lib/go-ucfg/yaml/yaml_test.go index 4c4ff020..18d29af3 100755 --- a/lib/go-ucfg/yaml/yaml_test.go +++ b/lib/go-ucfg/yaml/yaml_test.go @@ -134,9 +134,15 @@ func TestEmptyCollections(t *testing.T) { to: &struct { A interface{} }{}, + // an empty mapping reifies to an empty map (mirroring `a: []` + // → []interface{}{}), NOT nil: nil made the KEY disappear when + // the parent unpacked into map[string]interface{}, breaking + // configs like `processor: [{echo: {}}]` want: &struct { A interface{} - }{}, + }{ + A: map[string]interface{}{}, + }, }, "struct with empty map into struct with map of interfaces": { input: "a: {}", From f1ace01df2a72ddb9411f57fdbdbfcbb84c15862 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 03:01:53 +0800 Subject: [PATCH 43/65] fix(configs): placeholder version dates no longer kill instance heartbeats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A binary built without version ldflags reports build_date/eol_date as "N/A". On indices whose coco_instance mapping predates the keyword pin, those fields carry a dynamic date mapping and ES rejects the upsert with document_parsing_exception — every heartbeat failed silently (Debug log) while /configs/_sync kept returning 200, so the instance showed offline forever. Three layers: - env.Version fields now pin keyword elastic_mapping (new indices) - upsertInstance blanks non-RFC3339 date placeholders before Save (protects existing date-mapped indices) - heartbeat upsert failure logs at Warn instead of Debug --- core/env/version.go | 18 +++++++++------ modules/configs/server/server.go | 22 +++++++++++++++++- modules/configs/server/server_test.go | 33 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/core/env/version.go b/core/env/version.go index 5fef4a4e..dd965119 100644 --- a/core/env/version.go +++ b/core/env/version.go @@ -28,13 +28,17 @@ package env type Version struct { - VersionNumber string `json:"number,omitempty"` - FrameworkBuildCommitHash string `json:"framework_hash,omitempty"` - FrameworkVendorBuildCommitHash string `json:"vendor_hash,omitempty"` - BuildCommitHash string `json:"build_hash,omitempty"` - BuildDate string `json:"build_date,omitempty"` - BuildNumber string `json:"build_number,omitempty"` - EolDate string `json:"eol_date,omitempty"` + VersionNumber string `json:"number,omitempty" elastic_mapping:"number: { type: keyword }"` + FrameworkBuildCommitHash string `json:"framework_hash,omitempty" elastic_mapping:"framework_hash: { type: keyword }"` + FrameworkVendorBuildCommitHash string `json:"vendor_hash,omitempty" elastic_mapping:"vendor_hash: { type: keyword }"` + BuildCommitHash string `json:"build_hash,omitempty" elastic_mapping:"build_hash: { type: keyword }"` + // BuildDate/EolDate must map to keyword, never a dynamic date: binaries + // built without version ldflags report placeholders ("N/A"), which a + // date mapping rejects with document_parsing_exception — killing + // instance heartbeats on indices whose mapping predates this pin. + BuildDate string `json:"build_date,omitempty" elastic_mapping:"build_date: { type: keyword }"` + BuildNumber string `json:"build_number,omitempty" elastic_mapping:"build_number: { type: keyword }"` + EolDate string `json:"eol_date,omitempty" elastic_mapping:"eol_date: { type: keyword }"` } func (env *Env) GetVersionInfo() Version { diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go index 2c18a253..b18a0c6f 100644 --- a/modules/configs/server/server.go +++ b/modules/configs/server/server.go @@ -436,6 +436,24 @@ func (h *APIHandler) registerInstance(w http.ResponseWriter, req *http.Request, h.WriteJSON(w, resp, http.StatusOK) } +// normalizeVersionDates blanks client-reported version date placeholders +// ("N/A", "unknown", ...) that are not RFC3339 dates. Indices created +// before the keyword mapping pin carry a dynamic `date` mapping for +// version.build_date/eol_date (the first writer sent real ISO dates), and +// ES rejects placeholder strings with document_parsing_exception — which +// silently killed every heartbeat upsert for instances built without +// full version metadata (plain `go build`, no ldflags). +func normalizeVersionDates(app *env.Application) { + for _, d := range []*string{&app.Version.BuildDate, &app.Version.EolDate} { + if *d == "" { + continue + } + if _, err := time.Parse(time.RFC3339, *d); err != nil { + *d = "" + } + } +} + // upsertInstance persists/refreshes the registration. Returns true when // the instance was newly created. func upsertInstance(instance *model.Instance) (bool, error) { @@ -449,6 +467,8 @@ func upsertInstance(instance *model.Instance) (bool, error) { // not-found is the normal first-registration path, not an error exists = err == nil && exists + normalizeVersionDates(&instance.Application) + now := strconv.FormatInt(time.Now().UnixMilli(), 10) if instance.Labels == nil { instance.Labels = map[string]string{} @@ -532,7 +552,7 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt // Heartbeat: refresh the instance record on every sync. if _, err := upsertInstance(&obj.Client); err != nil { - log.Debugf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err) + log.Warnf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err) } // Runtime report: persist the instance's pipeline state for the diff --git a/modules/configs/server/server_test.go b/modules/configs/server/server_test.go index 7d093868..986c52b6 100644 --- a/modules/configs/server/server_test.go +++ b/modules/configs/server/server_test.go @@ -10,6 +10,7 @@ import ( "testing" httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/env" "infini.sh/framework/lib/go-ucfg" "infini.sh/framework/modules/configs/common" ) @@ -266,3 +267,35 @@ func TestDiffConfigs_HashAware(t *testing.T) { } }) } + +// Instances built without version ldflags report placeholder dates ("N/A"). +// An index whose mapping predates the keyword pin maps these fields as +// `date`, and ES rejects the placeholder — every heartbeat upsert failed +// silently (Debug log) while syncs kept returning 200. normalizeVersionDates +// must blank anything that is not an RFC3339 date. +func TestNormalizeVersionDates(t *testing.T) { + app := env.Application{} + app.Version.BuildDate = "N/A" + app.Version.EolDate = "unknown" + app.Version.BuildCommitHash = "abc123" + + normalizeVersionDates(&app) + + if app.Version.BuildDate != "" { + t.Fatalf("placeholder build_date not blanked: %q", app.Version.BuildDate) + } + if app.Version.EolDate != "" { + t.Fatalf("placeholder eol_date not blanked: %q", app.Version.EolDate) + } + if app.Version.BuildCommitHash != "abc123" { + t.Fatalf("non-date field must be untouched: %q", app.Version.BuildCommitHash) + } + + // real dates survive + app.Version.BuildDate = "2026-10-21T12:58:27Z" + app.Version.EolDate = "2027-12-31T10:10:10Z" + normalizeVersionDates(&app) + if app.Version.BuildDate != "2026-10-21T12:58:27Z" || app.Version.EolDate != "2027-12-31T10:10:10Z" { + t.Fatalf("real dates must survive: %+v", app.Version) + } +} From 494c6171819e752fd51e461fa5debf308adb7859 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 15:41:20 +0800 Subject: [PATCH 44/65] revert(pipeline): remove flight and persist loss accounting Drops the sampled-out/evicted flight counters and FlightLostStats, the metrics persist-failure counter, and the lost/persist_errors fields the flight and stages endpoints carried. The 1/N sampling loss is by design; the loss counters did not earn their complexity. --- modules/pipeline/flight.go | 35 +---------------------------- modules/pipeline/flight_test.go | 33 --------------------------- modules/pipeline/metrics_persist.go | 8 ------- modules/pipeline/tasks.go | 14 +++--------- 4 files changed, 4 insertions(+), 86 deletions(-) diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go index 499191ab..86bc9e15 100644 --- a/modules/pipeline/flight.go +++ b/modules/pipeline/flight.go @@ -1,6 +1,6 @@ /* Copyright © INFINI Ltd. All rights reserved. * Web: https://infinilabs.com - * Email: hello#infini.ltd */ + * Email: hello#infinilabs.com */ package pipeline @@ -22,17 +22,10 @@ const flightSample = 100 // It installs globally at PipeModule setup — the empty-registry fast // path applies before installation, and the per-step cost after is one // counter increment plus the sampled append. -// Loss is counted, never silent: sampledOut tracks OK steps skipped by -// the 1/N sampling, evicted tracks entries pushed out of a full ring — -// both surfaced through the flight endpoint so consumers know how much -// of the picture they are not seeing. type flightRecorder struct { mu sync.Mutex rings map[string][]pipeline.StepEvent seq int64 - - sampledOut atomic.Int64 - evicted atomic.Int64 } func newFlightRecorder() *flightRecorder { @@ -44,7 +37,6 @@ func (f *flightRecorder) OnStep(e *pipeline.StepEvent) { if !keep { n := atomic.AddInt64(&f.seq, 1) if n%flightSample != 0 { - f.sampledOut.Add(1) return } } @@ -52,37 +44,12 @@ func (f *flightRecorder) OnStep(e *pipeline.StepEvent) { f.mu.Lock() ring := append(f.rings[e.Pipeline], cp) if len(ring) > flightRing { - f.evicted.Add(int64(len(ring) - flightRing)) ring = ring[len(ring)-flightRing:] } f.rings[e.Pipeline] = ring f.mu.Unlock() } -// FlightLostStats is the loss accounting of the flight recorder, -// cumulative since process start. -type FlightLostStats struct { - // OK steps skipped by the 1/N sampling (by design, still a cost - // consumers should see). - SampledOut int64 `json:"sampled_out"` - // Oldest entries pushed out of a full ring (oldest-first eviction). - Evicted int64 `json:"evicted"` -} - -// flightLostStats reads the recorder's loss counters; zero values when -// no recorder is installed. -func flightLostStats() FlightLostStats { - f := globalFlight.Load() - if f == nil { - return FlightLostStats{} - } - rec := f.(*flightRecorder) - return FlightLostStats{ - SampledOut: rec.sampledOut.Load(), - Evicted: rec.evicted.Load(), - } -} - // flightQuery filters a pipeline's ring: by trace id, errors-only, newest // first, capped at limit. func flightQuery(pipelineName, traceID string, errorsOnly bool, limit int) []pipeline.StepEvent { diff --git a/modules/pipeline/flight_test.go b/modules/pipeline/flight_test.go index 6acd0357..09da912c 100644 --- a/modules/pipeline/flight_test.go +++ b/modules/pipeline/flight_test.go @@ -47,36 +47,3 @@ func TestFlight_RingBounded(t *testing.T) { t.Fatalf("ring = %d, want %d", n, flightRing) } } - -// Loss must be counted, never silent: OK steps skipped by sampling and -// entries evicted from a full ring both land in the lost counters. -func TestFlight_LossCounters(t *testing.T) { - installFlight() - f := globalFlight.Load().(*flightRecorder) - - errsBefore := f.evicted.Load() - sampledBefore := f.sampledOut.Load() - - // 200 OK steps: all but ~2 are sampled out - for i := 0; i < 200; i++ { - f.OnStep(&pipeline.StepEvent{Pipeline: "loss", Outcome: pipeline.OutcomeOK}) - } - sampled := f.sampledOut.Load() - sampledBefore - if sampled < 190 || sampled > 198 { // ~99% of 200 - t.Fatalf("sampledOut = %d, want ~198", sampled) - } - - // errors only: two full rings worth — the older ring overflows - for i := 0; i < flightRing*2; i++ { - f.OnStep(&pipeline.StepEvent{Pipeline: "loss-ring", Outcome: pipeline.OutcomeError}) - } - if evicted := f.evicted.Load() - errsBefore; evicted != flightRing { - t.Fatalf("evicted = %d, want %d", evicted, flightRing) - } - - // stats surface through the package helper (handler reads this) - lost := flightLostStats() - if lost.SampledOut < sampled || lost.Evicted < flightRing { - t.Fatalf("flightLostStats = %+v, want counters >= this test's deltas", lost) - } -} diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index a19e93fd..0107187f 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -5,7 +5,6 @@ package pipeline import ( - "sync/atomic" "time" log "github.com/cihub/seelog" @@ -29,12 +28,6 @@ type PipelineStageStat struct { Ts time.Time `json:"ts" elastic_mapping:"ts:{type:date}"` } -// metricsPersistFailures counts stage-snapshot rows the exporter failed -// to persist (orm save errors) — cumulative since process start. Losses -// here mean the persisted metric history has holes; surfaced via the -// stages endpoint next to the in-memory stats. -var metricsPersistFailures atomic.Int64 - // metricsPersistConfig reads pipeline_metrics.persistence. type metricsPersistConfig struct { Persistence struct { @@ -94,7 +87,6 @@ func (module *PipeModule) persistStageSnapshot() { } row.ID = util.GetUUID() if err := orm.Save(octx, &row); err != nil { - metricsPersistFailures.Add(1) log.Warnf("metrics persist: save %s/%s failed: %v", name, st.Stage, err) } } diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index ab2e4ebb..9b5161fa 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -191,24 +191,16 @@ func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req * // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_start // stagesPipelineTaskHandler — GET /pipeline/task/:id/stages // Standard per-stage metrics: processor invocations/errors/latency plus -// live lane carriers (depth-derived). persist_errors counts stage -// snapshot rows the metrics exporter failed to write — a non-zero value -// means the persisted history has holes. +// live lane carriers (depth-derived). func (module *PipeModule) stagesPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") - module.WriteJSON(w, util.MapStr{ - "stages": stageStats(id), - "persist_errors": metricsPersistFailures.Load(), - }, http.StatusOK) + module.WriteJSON(w, util.MapStr{"stages": stageStats(id)}, http.StatusOK) } // flightPipelineTaskHandler — GET /pipeline/task/:id/flight // Sampled execution trace of a pipeline: ?trace_id= for one record's // journey, ?errors=1 for recent failures, &limit= (default 50, newest // first). Memory-only (ring); persistence rides the metrics exporter. -// The `lost` block counts what the view does not show: OK steps skipped -// by the 1/N sampling and entries evicted from the full ring (cumulative -// since process start). func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") q := req.URL.Query() @@ -219,7 +211,7 @@ func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req * } } events := flightQuery(id, q.Get("trace_id"), q.Get("errors") == "1", limit) - module.WriteJSON(w, util.MapStr{"events": events, "lost": flightLostStats()}, http.StatusOK) + module.WriteJSON(w, util.MapStr{"events": events}, http.StatusOK) } // pausePipelineTaskHandler — POST /pipeline/task/:id/_pause From 866b35e9d0a83c588dc7b10aefd97ee84edb3589 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 16:48:12 +0800 Subject: [PATCH 45/65] fix(pipeline): contract enforcement actually reaches every processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two silent gaps made the strict-contract machinery a no-op for real product pipelines: - validatePlacement only consulted the constructed instance's ContractProvider method, but every product processor declares its contract through RegisterProcessorContract — registry declarations never drove placement. The registry is now the fallback source, so record-scoped processors sitting at a batch root are finally caught. - for_each and lane_consumer compiled their sub-chains without the enclosing scope's Strict flag, so even opted-in strict compiles only warned inside record sub-chains. New ScopeAwareProcessor interface: after construction the compile hands hosts its ScopeInfo and they recompile their sub-chain strict (if/else and case already threaded theirs through WithScope constructors). Tests: strict compile rejects a batch-scoped processor inside for_each (instance-declared and registry-declared variants); non-strict stays warn-only; ValidatePipelineConfig surfaces the nested mismatch in its warnings — the signal the pipeline-audit pre-flight consumes. --- core/pipeline/contract.go | 29 ++++-- core/pipeline/processor.go | 11 +++ modules/pipeline/for_each.go | 20 +++++ modules/pipeline/lane_consumer.go | 20 +++++ modules/pipeline/scope_strict_test.go | 122 ++++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 modules/pipeline/scope_strict_test.go diff --git a/core/pipeline/contract.go b/core/pipeline/contract.go index 6afb181d..3455bae5 100644 --- a/core/pipeline/contract.go +++ b/core/pipeline/contract.go @@ -64,15 +64,32 @@ type ContractProvider interface { RecordContract() RecordContract } +// ScopeAwareProcessor is implemented by hosts that compile nested chains +// of their own (for_each, lane_consumer). The registry constructor takes +// no scope, so those hosts compile their sub-chains warn-only; after +// construction the enclosing compile hands them its ScopeInfo so +// strictness reaches the nested chain too. ApplyScope must be +// idempotent — a host may be constructed and scoped more than once. +type ScopeAwareProcessor interface { + ApplyScope(scope ScopeInfo) error +} + // validatePlacement checks one constructed processor against the compile -// scope. Returns a descriptive error; the caller decides error-vs-warning -// (strict mode) — this function stays pure. +// scope. The contract comes from the instance (ContractProvider) or from +// the registry (RegisterProcessorContract — the path every product +// processor takes); without the registry fallback the placement check +// would silently pass for all of them. Returns a descriptive error; the +// caller decides error-vs-warning (strict mode) — this function stays pure. func validatePlacement(p Processor, stepIdx int, scope ScopeInfo, direct bool) error { - cp, ok := p.(ContractProvider) - if !ok { - return nil + var c RecordContract + if cp, ok := p.(ContractProvider); ok { + c = cp.RecordContract() + } else { + c = RecordContractOf(p.Name()) + } + if c.Scope == ScopeAny && c.Namespace == "" && c.Granularity == GranularityRecord && len(c.RequiredFields) == 0 { + return nil // unconstrained } - c := cp.RecordContract() where := fmt.Sprintf("processor [%s] at step %d", p.Name(), stepIdx+1) if c.Scope == ScopeRecord && scope.Kind != ScopeRecord { diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go index dd471753..5deca46c 100755 --- a/core/pipeline/processor.go +++ b/core/pipeline/processor.go @@ -204,6 +204,17 @@ func newPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (* } log.Warnf("contract (non-strict): %v", err) } + // hosts with nested chains (for_each, lane_consumer) compile + // them warn-only at construction; hand them the enclosing + // scope so a strict compile reaches the sub-chain as well + if sa, isScopeAware := p.(ScopeAwareProcessor); isScopeAware { + if err := sa.ApplyScope(scope); err != nil { + if scope.Strict { + return nil, err + } + log.Warnf("contract (non-strict): %v", err) + } + } procs.AddProcessor(p) } else { return nil, errors.Errorf("invalid processor: [%v]", plugin.Name()) diff --git a/modules/pipeline/for_each.go b/modules/pipeline/for_each.go index 39808b23..205ac4be 100644 --- a/modules/pipeline/for_each.go +++ b/modules/pipeline/for_each.go @@ -165,6 +165,26 @@ func recordCodecNames() []string { func (p *ForEachProcessor) Name() string { return "for_each" } +// ApplyScope recompiles the sub-chain under the enclosing compile scope: +// the registry constructor takes no scope, so the initial compile is +// warn-only — a strict parent demands the strict view of the sub-chain +// (pipeline.ScopeAwareProcessor; idempotent by replacement). +func (p *ForEachProcessor) ApplyScope(scope pipeline.ScopeInfo) error { + if !scope.Strict { + return nil + } + sub, err := pipeline.NewPipelineWithScope(p.cfg.Processors, pipeline.ScopeInfo{ + Kind: pipeline.ScopeRecord, + Namespace: p.codec.Namespace(), + Strict: true, + }, true) + if err != nil { + return fmt.Errorf("failed to compile for_each sub-chain: %s", err) + } + p.sub = sub + return nil +} + func (p *ForEachProcessor) Process(c *pipeline.Context) error { v := c.Get(param.ParaKey(p.cfg.MessageField)) msgs, ok := v.([]queue.Message) diff --git a/modules/pipeline/lane_consumer.go b/modules/pipeline/lane_consumer.go index 3386421f..d9d8f4e5 100644 --- a/modules/pipeline/lane_consumer.go +++ b/modules/pipeline/lane_consumer.go @@ -95,6 +95,26 @@ func NewLaneConsumerProcessor(c *config.Config) (pipeline.Processor, error) { func (p *LaneConsumerProcessor) Name() string { return laneConsumerName } +// ApplyScope recompiles the sub-chain under the enclosing compile scope: +// the registry constructor takes no scope, so the initial compile is +// warn-only — a strict parent demands the strict view of the sub-chain +// (pipeline.ScopeAwareProcessor; idempotent by replacement). +func (p *LaneConsumerProcessor) ApplyScope(scope pipeline.ScopeInfo) error { + if !scope.Strict { + return nil + } + sub, err := pipeline.NewPipelineWithScope(p.cfg.Processors, pipeline.ScopeInfo{ + Kind: pipeline.ScopeRecord, + Namespace: p.lane.Config().Namespace, + Strict: true, + }, true) + if err != nil { + return fmt.Errorf("failed to compile %s sub-chain: %s", laneConsumerName, err) + } + p.sub = sub + return nil +} + // Process subscribes (once) and blocks until the pipeline task cancels — // the host drives the lane for its whole lifetime. func (p *LaneConsumerProcessor) Process(ctx *pipeline.Context) error { diff --git a/modules/pipeline/scope_strict_test.go b/modules/pipeline/scope_strict_test.go new file mode 100644 index 00000000..65b50c31 --- /dev/null +++ b/modules/pipeline/scope_strict_test.go @@ -0,0 +1,122 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" +) + +// batchScopedFixture declares a ScopeBatch contract — legal at a chain +// root, illegal inside a record sub-chain. It exists to prove strict +// compile reaches through for_each into the nested chain. +type batchScopedFixture struct{} + +func (batchScopedFixture) Name() string { return "test_batch_scoped" } + +func (batchScopedFixture) Process(*pipeline.Context) error { return nil } + +func (batchScopedFixture) RecordContract() pipeline.RecordContract { + return pipeline.RecordContract{Scope: pipeline.ScopeBatch} +} + +func init() { + // for_each registers during module setup, which these unit tests skip + pipeline.RegisterProcessorPlugin("for_each", NewForEachProcessor) + pipeline.RegisterProcessorPlugin("test_batch_scoped", + func(*config.Config) (pipeline.Processor, error) { return batchScopedFixture{}, nil }) + pipeline.RegisterProcessorPlugin("test_registry_scoped", + func(*config.Config) (pipeline.Processor, error) { return registryContractFixture{}, nil }) + // registry-declared contract, instance implements nothing — the path + // every product processor takes (RegisterProcessorContract) + pipeline.RegisterProcessorContract("test_registry_scoped", + pipeline.RecordContract{Scope: pipeline.ScopeBatch}) +} + +// registryContractFixture carries NO ContractProvider method; its +// contract lives only in the registry. +type registryContractFixture struct{} + +func (registryContractFixture) Name() string { return "test_registry_scoped" } + +func (registryContractFixture) Process(*pipeline.Context) error { return nil } + +func forEachWithBatchScopedStep(t *testing.T) *config.Config { + t.Helper() + c, err := config.NewConfigFrom(util.MapStr{ + "for_each": util.MapStr{ + "processor": []interface{}{util.MapStr{"test_batch_scoped": util.MapStr{}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestForEach_SubChainStrictnessThreaded(t *testing.T) { + step := forEachWithBatchScopedStep(t) + + // non-strict: the nested placement mismatch only warns, the chain + // compiles — this is the legacy behavior the default flip keeps + if _, err := pipeline.NewPipeline([]*config.Config{step}); err != nil { + t.Fatalf("non-strict compile should accept the nested mismatch: %v", err) + } + + // strict: the violation inside the sub-chain must surface as a + // compile error (threaded through ApplyScope) + _, err := pipeline.NewPipelineWithScope([]*config.Config{step}, + pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false) + if err == nil || !strings.Contains(err.Error(), "requires batch scope") { + t.Fatalf("strict compile must reject the nested violation, got: %v", err) + } +} + +func TestValidatePipelineConfig_NestedContractWarns(t *testing.T) { + // the validation gate feeds the audit tool: without scope threading + // a nested mismatch never reached its warnings list + cfg := &pipeline.PipelineConfigV2{ + Name: "strict-threading-probe", + Processors: []map[string]interface{}{ + {"for_each": util.MapStr{ + "processor": []interface{}{util.MapStr{"test_batch_scoped": util.MapStr{}}}, + }}, + }, + } + errs, warns := ValidatePipelineConfig(cfg) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(warns) == 0 || !strings.Contains(warns[0].Message, "requires batch scope") { + t.Fatalf("nested contract mismatch must warn, got warns=%v", warns) + } +} + +func TestValidatePlacement_RegistryDeclaredContract(t *testing.T) { + // registry-declared contracts (the product-processor path) must drive + // placement too — the instance-only assertion silently passed them + c, err := config.NewConfigFrom(util.MapStr{ + "for_each": util.MapStr{ + "processor": []interface{}{util.MapStr{"test_registry_scoped": util.MapStr{}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + step := []*config.Config{c} + + if _, err := pipeline.NewPipeline(step); err != nil { + t.Fatalf("non-strict compile should accept the mismatch: %v", err) + } + _, err = pipeline.NewPipelineWithScope(step, + pipeline.ScopeInfo{Kind: pipeline.ScopeBatch, Strict: true}, false) + if err == nil || !strings.Contains(err.Error(), "requires batch scope") { + t.Fatalf("strict compile must honor registry-declared contracts, got: %v", err) + } +} From 1fea4beade160857286119e820b5e886f2b0bad8 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 20:06:10 +0800 Subject: [PATCH 46/65] feat(pipeline): record event pool, release wiring, P4 benchmarks (#67 #68 #75) AcquireEvent/ReleaseEvent recycle *Event across record-mode batches: maps are cleared in place under an entry cap, double release is rejected atomically, and ReleaseEvent no-ops on codec/clone-allocated events so for_each can release every record unconditionally on all exit paths (including drops and failures). The P4 bench suite lands the gates: 1MB document K=4 record chain (pooled vs fresh), and the 1KB CPU-chain parallel gate (parallel:4 measures 3.8x on the 8-core M2, clearing the >=3x target). Also fixes a pre-existing -race failure that blocked the gate: debug logs passed the mutable *Processors to seelog's lazy async formatter; eagerly materialize procs.String() at the three call sites. --- core/event/event.go | 4 + core/event/pool.go | 135 +++++++++++++++++ core/event/pool_test.go | 120 +++++++++++++++ core/pipeline/filter.go | 4 +- core/pipeline/processor.go | 2 +- modules/pipeline/for_each.go | 8 + modules/pipeline/for_each_bench_test.go | 192 ++++++++++++++++++++++++ modules/pipeline/for_each_pool_test.go | 109 ++++++++++++++ 8 files changed, 571 insertions(+), 3 deletions(-) create mode 100644 core/event/pool.go create mode 100644 core/event/pool_test.go create mode 100644 modules/pipeline/for_each_bench_test.go create mode 100644 modules/pipeline/for_each_pool_test.go diff --git a/core/event/event.go b/core/event/event.go index 87a89f32..1d6861dd 100644 --- a/core/event/event.go +++ b/core/event/event.go @@ -47,6 +47,10 @@ type Event struct { Meta util.MapStr `json:"-"` Private interface{} `json:"-"` // for beats private use TimeSeries bool `json:"-"` // true if the event contains timeseries data + + // pooled links to the record-pool wrapper while the event came from + // AcquireEvent; managed by pool.go, never serialized. + pooled *pooledEvent } type EventMetadata struct { diff --git a/core/event/pool.go b/core/event/pool.go new file mode 100644 index 00000000..dc136afc --- /dev/null +++ b/core/event/pool.go @@ -0,0 +1,135 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello#infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* ©INFINI, All Rights Reserved. + * mail: contact#infini.ltd */ + +package event + +import ( + "sync" + "sync/atomic" + "time" +) + +// The record pool recycles *Event across record-mode batches: decode-heavy +// hosts (for_each) otherwise allocate a fresh Event, its field maps and the +// whole decoded tree per record only to drop them after encode. +// +// Contract: +// - AcquireEvent returns a zeroed event; Decode-style producers write into +// it as if it were freshly allocated. +// - ReleaseEvent returns the event to the pool. It must be called only +// after the record has been fully encoded (or dropped) AND no downstream +// processor retains a reference — the host owns the record lifetime. +// - ReleaseEvent is a no-op on events that did not come from the pool +// (codec- or clone-allocated), so a host can release every record +// unconditionally. +// - A double release is rejected atomically (second call returns false) +// instead of pooling the same event twice. +// +// Reset policy: scalar fields are zeroed; the hot maps (Fields, Metadata +// labels, Meta) are cleared in place — their bucket arrays are reused — +// unless they grew past maxRecycledEntries, in which case they are dropped +// so one giant record cannot pin memory in the pool forever. + +// maxRecycledEntries caps in-place map reuse; bigger maps are released to +// the GC instead of living in the pool. +const maxRecycledEntries = 128 + +type pooledEvent struct { + live atomic.Bool + ev Event +} + +var recordPool = sync.Pool{New: func() interface{} { return &pooledEvent{} }} + +var ( + poolAcquires atomic.Int64 + poolReleases atomic.Int64 +) + +// AcquireEvent returns an event from the record pool (zeroed). +func AcquireEvent() *Event { + pe := recordPool.Get().(*pooledEvent) + pe.live.Store(true) + pe.ev.pooled = pe + poolAcquires.Add(1) + return &pe.ev +} + +// ReleaseEvent recycles a pooled event; returns true when the event went +// back to the pool, false for non-pooled events or a rejected double +// release. +func ReleaseEvent(e *Event) bool { + if e == nil || e.pooled == nil { + return false + } + pe := e.pooled + if !pe.live.CompareAndSwap(true, false) { + return false + } + e.resetForPool() + e.pooled = nil + poolReleases.Add(1) + recordPool.Put(pe) + return true +} + +// PoolStats reports cumulative acquire/release counts; the difference is +// the number of pooled events currently in use (leak detection). +func PoolStats() (acquired, released int64) { + return poolAcquires.Load(), poolReleases.Load() +} + +// resetForPool clears an event for reuse; see the pool doc for the +// map-retention policy. +func (e *Event) resetForPool() { + e.Fields = recycleMap(e.Fields) + e.Metadata.Labels = recycleMap(e.Metadata.Labels) + e.Meta = recycleMap(e.Meta) + e.Metadata.Category = "" + e.Metadata.Name = "" + e.Metadata.Version = "" + e.Metadata.Datatype = "" + e.Agent = nil + e.Private = nil + e.QueueName = "" + e.TimeSeries = false + e.Timestamp = time.Time{} +} + +// recycleMap clears a map in place when it is small enough to be worth +// keeping, else drops it. +func recycleMap(m map[string]interface{}) map[string]interface{} { + if m == nil { + return nil + } + if len(m) > maxRecycledEntries { + return nil + } + for k := range m { + delete(m, k) + } + return m +} diff --git a/core/event/pool_test.go b/core/event/pool_test.go new file mode 100644 index 00000000..b1d53948 --- /dev/null +++ b/core/event/pool_test.go @@ -0,0 +1,120 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package event + +import ( + "sync" + "testing" + "time" + + "infini.sh/framework/core/util" +) + +func TestPoolAcquireReleaseBalance(t *testing.T) { + a0, r0 := PoolStats() + e := AcquireEvent() + e.Fields = util.MapStr{"k": "v"} + if !ReleaseEvent(e) { + t.Fatal("releasing a pooled event must report true") + } + a1, r1 := PoolStats() + if a1-a0 != 1 || r1-r0 != 1 { + t.Fatalf("stats must balance: acquired +%d released +%d", a1-a0, r1-r0) + } +} + +func TestPoolReleaseNonPooledIsNoop(t *testing.T) { + if ReleaseEvent(&Event{}) { + t.Fatal("a plain event must not enter the pool") + } + if ReleaseEvent(nil) { + t.Fatal("nil must not enter the pool") + } +} + +func TestPoolDoubleReleaseRejected(t *testing.T) { + _, r0 := PoolStats() + e := AcquireEvent() + if !ReleaseEvent(e) { + t.Fatal("first release must succeed") + } + if ReleaseEvent(e) { + t.Fatal("second release of the same event must be rejected") + } + _, r1 := PoolStats() + if r1-r0 != 1 { + t.Fatalf("double release must not inflate the release count: +%d", r1-r0) + } +} + +func TestPoolResetClearsEverything(t *testing.T) { + e := AcquireEvent() + e.Fields = util.MapStr{"document": map[string]interface{}{"id": "x"}} + e.Metadata = EventMetadata{ + Labels: util.MapStr{"datasource": "d1"}, + Category: "cat", Name: "n", Version: "v", Datatype: "dt", + } + e.Meta = util.MapStr{"m": "1"} + e.Agent = &AgentMeta{AgentID: "a"} + e.Private = struct{}{} + e.QueueName = "q" + e.TimeSeries = true + e.Timestamp = time.Now() + + e.resetForPool() + + if len(e.Fields) != 0 || len(e.Metadata.Labels) != 0 || len(e.Meta) != 0 { + t.Fatal("maps must be emptied") + } + if e.Agent != nil || e.Private != nil || e.QueueName != "" || e.TimeSeries { + t.Fatal("scalar/pointer state must be zeroed") + } + if !e.Timestamp.IsZero() || e.Metadata.Category != "" || e.Metadata.Name != "" || + e.Metadata.Version != "" || e.Metadata.Datatype != "" { + t.Fatal("metadata and timestamp must be zeroed") + } + ReleaseEvent(e) +} + +func TestPoolMapRecyclePolicy(t *testing.T) { + small := util.MapStr{} + for i := 0; i < maxRecycledEntries; i++ { + small["k"+string(rune('a'+i%26))+string(rune('0'+i/26))] = i + } + if got := recycleMap(small); got == nil || len(got) != 0 { + t.Fatal("small maps must be cleared and kept") + } + big := util.MapStr{} + for i := 0; i <= maxRecycledEntries; i++ { + big[string(rune('a'+i%26))+string(rune('0'+i/26))] = i + } + if got := recycleMap(big); got != nil { + t.Fatal("oversized maps must be dropped, not recycled") + } +} + +// TestPoolConcurrentAcquireRelease is the -race gate: concurrent owners +// recycle disjoint events and the counters balance at the end. +func TestPoolConcurrentAcquireRelease(t *testing.T) { + const workers = 8 + const rounds = 200 + _, r0 := PoolStats() + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + e := AcquireEvent() + e.Fields = util.MapStr{"message": "x"} + ReleaseEvent(e) + } + }() + } + wg.Wait() + _, r1 := PoolStats() + if got := r1 - r0; got != workers*rounds { + t.Fatalf("every acquire must be released exactly once: released %d of %d", got, workers*rounds) + } +} diff --git a/core/pipeline/filter.go b/core/pipeline/filter.go index 3a59dfba..bfd1a2e8 100644 --- a/core/pipeline/filter.go +++ b/core/pipeline/filter.go @@ -100,7 +100,7 @@ func NewFilter(cfg []*config.Config) (*Filters, error) { } if len(procs.List) > 0 { - log.Debugf("generated new filters: %v", procs) + log.Debugf("generated new filters: %s", procs.String()) } return procs, nil } @@ -150,7 +150,7 @@ func (procs *Filters) Close() error { func (procs *Filters) Filter(ctx *fasthttp.RequestCtx) { if procs == nil { - log.Errorf("invalid filter: %v", procs) + log.Errorf("invalid filter: %s", procs.String()) return } for _, p := range procs.List { diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go index 5deca46c..9819a09c 100755 --- a/core/pipeline/processor.go +++ b/core/pipeline/processor.go @@ -222,7 +222,7 @@ func newPipelineWithScope(cfg []*config.Config, scope ScopeInfo, direct bool) (* } if len(procs.List) > 0 { - log.Debugf("generated new processors: %v", procs) + log.Debugf("generated new processors: %s", procs.String()) } return procs, nil } diff --git a/modules/pipeline/for_each.go b/modules/pipeline/for_each.go index 205ac4be..5e357498 100644 --- a/modules/pipeline/for_each.go +++ b/modules/pipeline/for_each.go @@ -233,6 +233,14 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { for j, rec := range records { active = append(active, forEachActive{rec: rec, msgIdx: decodedIdx[j]}) } + // release pooled records on every exit path — by now each record has + // been encoded (or dropped) and clones have joined active; non-pooled + // records (codec- or clone-allocated) make ReleaseEvent a no-op + defer func() { + for i := range active { + event.ReleaseEvent(active[i].rec) + } + }() for _, seg := range segments { if c.IsCanceled() || !c.ShouldContinue() { diff --git a/modules/pipeline/for_each_bench_test.go b/modules/pipeline/for_each_bench_test.go new file mode 100644 index 00000000..99d4f0a0 --- /dev/null +++ b/modules/pipeline/for_each_bench_test.go @@ -0,0 +1,192 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package pipeline + +import ( + "encoding/json" + "hash/fnv" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/param" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// P4 benchmark suite (#75): record-mode data movement through for_each. +// +// BenchmarkForEach_1MB_K4_* — 1MB document tree, K=4 record processors: +// pooled vs fresh allocation (the pool gate) +// BenchmarkForEach_1KB_Parall — 1KB events, parallel 1 vs 4 (the ≥3× +// throughput gate on CPU-shaped chains) +// +// The codec decodes the whole JSON tree into the record Fields — the +// document-codec shape — so the numbers track the real data-movement cost, +// not framework scaffolding. + +// benchTreeCodec decodes a JSON envelope into the record fields tree. +type benchTreeCodec struct{ pooled bool } + +func (benchTreeCodec) Name() string { return "bench_tree" } +func (benchTreeCodec) Namespace() string { return "" } +func (c benchTreeCodec) Decode(data []byte) (*event.Event, error) { + var tree util.MapStr + if err := json.Unmarshal(data, &tree); err != nil { + return nil, err + } + var e *event.Event + if c.pooled { + e = event.AcquireEvent() + } else { + e = &event.Event{} + } + e.Fields = tree + return e, nil +} +func (benchTreeCodec) Encode(rec *event.Event) ([]byte, error) { + return json.Marshal(rec.Fields) +} + +// benchHasher burns CPU proportional to the content — the CPU-shaped +// chain the parallel gate measures (a no-op chain scales nothing). +type benchHasher struct{} + +func (benchHasher) Name() string { return "test_bench_hasher" } +func (benchHasher) Process(c *pipeline.Context) error { + rec, _ := pipeline.CurrentRecord(c) + if rec == nil { + return nil + } + doc, _ := rec.Fields["document"].(map[string]interface{}) + if doc == nil { + return nil + } + content, _ := doc["content"].(string) + h := fnv.New64a() + for i := 0; i < 256; i++ { // burn enough CPU that the chain, not the codec, dominates + h.Reset() + h.Write([]byte(content)) + } + doc["hash"] = h.Sum64() + return nil +} + +// benchPayload builds a document-shaped payload of roughly size bytes. +func benchPayload(size int) []byte { + content := make([]byte, 0, size) + unit := []byte("The quick brown fox jumps over the lazy dog. ") + for len(content) < size { + content = append(content, unit...) + } + body := map[string]interface{}{ + "document": map[string]interface{}{ + "id": "bench-doc", + "title": "Benchmark Document", + "content": string(content[:size]), + "source": map[string]interface{}{"id": "bench-src", "type": "test"}, + }, + } + b, _ := json.Marshal(body) + return b +} + +func init() { + pipeline.RegisterProcessorPlugin("test_bench_hasher", func(*config.Config) (pipeline.Processor, error) { return benchHasher{}, nil }) +} + +func buildForBench(b *testing.B, extra map[string]interface{}) *ForEachProcessor { + b.Helper() + cfgMap := util.MapStr{ + "codec": "bench_tree", + "processor": []interface{}{ + util.MapStr{benchProc: util.MapStr{}}, + util.MapStr{benchProc: util.MapStr{}}, + util.MapStr{benchProc: util.MapStr{}}, + util.MapStr{benchProc: util.MapStr{}}, + }, + } + for k, v := range extra { + cfgMap[k] = v + } + c, err := config.NewConfigFrom(cfgMap) + if err != nil { + b.Fatal(err) + } + p, err := NewForEachProcessor(c) + if err != nil { + b.Fatalf("NewForEachProcessor: %v", err) + } + return p.(*ForEachProcessor) +} + +func runForEachBench(b *testing.B, p *ForEachProcessor, payload []byte, parallel int) { + b.Helper() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + msgs := []queue.Message{{Data: payload, Size: len(payload), Offset: queue.AcquireOffset(0, int64(i))}} + ctx := &pipeline.Context{Context: b.Context()} + ctx.Set(param.ParaKey("messages"), msgs) + if err := p.Process(ctx); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkForEach_1MB_K4_Pooled(b *testing.B) { + if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil { + b.Fatal(err) + } + runForEachBench(b, buildForBench(b, nil), benchPayload(1<<20), 1) +} + +func BenchmarkForEach_1MB_K4_Fresh(b *testing.B) { + if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: false}); err != nil { + b.Fatal(err) + } + runForEachBench(b, buildForBench(b, nil), benchPayload(1<<20), 1) +} + +// benchProc selects the K=4 chain shape: the data-movement benches use the +// cheap mutator, the parallel gates use the CPU burner. +var benchProc = "test_upper" + +// runForEachBatchBench drives a 32-record batch — parallel workers need +// records to steal; a single-record batch cannot scale. +func runForEachBatchBench(b *testing.B, p *ForEachProcessor, payload []byte) { + b.Helper() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + msgs := make([]queue.Message, 0, 32) + for j := 0; j < 32; j++ { + msgs = append(msgs, queue.Message{Data: payload, Size: len(payload), Offset: queue.AcquireOffset(0, int64(i*32+j))}) + } + ctx := &pipeline.Context{Context: b.Context()} + ctx.Set(param.ParaKey("messages"), msgs) + if err := p.Process(ctx); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkForEach_1KB_CPUChain_Parallel1(b *testing.B) { + if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil { + b.Fatal(err) + } + benchProc = "test_bench_hasher" + defer func() { benchProc = "test_upper" }() + runForEachBatchBench(b, buildForBench(b, map[string]interface{}{"parallel": 1}), benchPayload(1<<10)) +} + +func BenchmarkForEach_1KB_CPUChain_Parallel4(b *testing.B) { + if err := RegisterRecordCodecOrReplace(benchTreeCodec{pooled: true}); err != nil { + b.Fatal(err) + } + benchProc = "test_bench_hasher" + defer func() { benchProc = "test_upper" }() + runForEachBatchBench(b, buildForBench(b, map[string]interface{}{"parallel": 4}), benchPayload(1<<10)) +} diff --git a/modules/pipeline/for_each_pool_test.go b/modules/pipeline/for_each_pool_test.go new file mode 100644 index 00000000..86636848 --- /dev/null +++ b/modules/pipeline/for_each_pool_test.go @@ -0,0 +1,109 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package pipeline + +import ( + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/param" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +// pooledRawCodec is rawCodec with the record coming from the event pool — +// the production shape of coco's document/attachment codecs. +type pooledRawCodec struct{} + +func (pooledRawCodec) Name() string { return "raw_pooled" } +func (pooledRawCodec) Namespace() string { return "" } +func (pooledRawCodec) Decode(data []byte) (*event.Event, error) { + e := event.AcquireEvent() + e.Fields = util.MapStr{"message": string(data)} + return e, nil +} +func (pooledRawCodec) Encode(rec *event.Event) ([]byte, error) { + msg, _ := rec.Fields["message"].(string) + return []byte(msg), nil +} + +// dropper marks every record dropped so the wiring test covers the drop +// path — dropped records must be released too. +type dropper struct{} + +func (dropper) Name() string { return "test_dropper" } +func (dropper) Process(c *pipeline.Context) error { + if rec, ok := pipeline.CurrentRecord(c); ok { + pipeline.MarkDropped(rec) + } + return nil +} + +func init() { + pipeline.RegisterProcessorPlugin("test_dropper", func(*config.Config) (pipeline.Processor, error) { return dropper{}, nil }) +} + +// TestForEach_PooledRecordsReleased: every pooled record must be back in +// the pool by the time Process returns — originals, transformed records +// and dropped records alike (leak detection gate for the pool wiring). +func TestForEach_PooledRecordsReleased(t *testing.T) { + if err := RegisterRecordCodecOrReplace(pooledRawCodec{}); err != nil { + t.Fatal(err) + } + p := buildForEach(t, map[string]interface{}{"codec": "raw_pooled"}) + + _, r0 := event.PoolStats() + for run := 0; run < 3; run++ { + msgs := []queue.Message{ + {Data: []byte("keep-1"), Size: 6, Offset: queue.AcquireOffset(0, 1)}, + {Data: []byte("keep-2"), Size: 6, Offset: queue.AcquireOffset(0, 2)}, + } + ctx := &pipeline.Context{Context: t.Context()} + ctx.Set(param.ParaKey("messages"), msgs) + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + if string(msgs[0].Data) != "PROCESSED:keep-1" { + t.Fatalf("record not transformed: %s", msgs[0].Data) + } + } + _, r1 := event.PoolStats() + if got := r1 - r0; got != 6 { + t.Fatalf("all 6 pooled records must be released, got %d", got) + } +} + +func TestForEach_PooledDroppedRecordsReleased(t *testing.T) { + if err := RegisterRecordCodecOrReplace(pooledRawCodec{}); err != nil { + t.Fatal(err) + } + c, err := config.NewConfigFrom(util.MapStr{ + "codec": "raw_pooled", + "processor": []interface{}{util.MapStr{"test_dropper": util.MapStr{}}}, + }) + if err != nil { + t.Fatal(err) + } + p, err := NewForEachProcessor(c) + if err != nil { + t.Fatal(err) + } + + _, r0 := event.PoolStats() + msgs := []queue.Message{{Data: []byte("gone"), Size: 4, Offset: queue.AcquireOffset(0, 1)}} + ctx := &pipeline.Context{Context: t.Context()} + ctx.Set(param.ParaKey("messages"), msgs) + if err := p.(*ForEachProcessor).Process(ctx); err != nil { + t.Fatal(err) + } + if msgs[0].Data != nil || msgs[0].Size != 0 { + t.Fatalf("dropped record must leave an empty slot: %q", msgs[0].Data) + } + _, r1 := event.PoolStats() + if got := r1 - r0; got != 1 { + t.Fatalf("dropped pooled record must be released, got %d", got) + } +} From e1845cc1db0291b6bfee59856fdf7a1f8f842703 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 21:05:02 +0800 Subject: [PATCH 47/65] =?UTF-8?q?fix(event):=20pooled=20back-reference=20a?= =?UTF-8?q?s=20interface{}=20=E2=80=94=20typed=20cycle=20blew=20the=20refl?= =?UTF-8?q?ection=20walkers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Event.pooled was *pooledEvent while pooledEvent embeds Event: a recursive type that sent util.GetTagsByTagName (no cycle guard) into infinite recursion during model schema registration — coco crashed at startup with a stack overflow. Hold the back-reference as interface{} (the walker does not descend interface fields); ReleaseEvent type-asserts. --- core/event/event.go | 7 +++++-- core/event/pool.go | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/event/event.go b/core/event/event.go index 1d6861dd..9a394923 100644 --- a/core/event/event.go +++ b/core/event/event.go @@ -49,8 +49,11 @@ type Event struct { TimeSeries bool `json:"-"` // true if the event contains timeseries data // pooled links to the record-pool wrapper while the event came from - // AcquireEvent; managed by pool.go, never serialized. - pooled *pooledEvent + // AcquireEvent; managed by pool.go, never serialized. Held as + // interface{} on purpose: a typed *pooledEvent back-reference would + // create a recursive type (Event -> pooledEvent -> Event) and blow + // the reflection walkers (GetTagsByTagName) with a stack overflow. + pooled interface{} } type EventMetadata struct { diff --git a/core/event/pool.go b/core/event/pool.go index dc136afc..2217fecf 100644 --- a/core/event/pool.go +++ b/core/event/pool.go @@ -82,10 +82,13 @@ func AcquireEvent() *Event { // back to the pool, false for non-pooled events or a rejected double // release. func ReleaseEvent(e *Event) bool { - if e == nil || e.pooled == nil { + if e == nil { + return false + } + pe, ok := e.pooled.(*pooledEvent) + if !ok { return false } - pe := e.pooled if !pe.live.CompareAndSwap(true, false) { return false } From fd47c67345661164617ac55ab20afc14096c7d0d Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 21:34:49 +0800 Subject: [PATCH 48/65] feat(pipeline): unified POST /pipeline/ai-generate with ChainGenerator registry (#91) Chain generation is product logic (LLM backend, prompts, catalog); the endpoint is not. Products register a ChainGenerator at init; the unified endpoint validates the request shape (documents must be JSON objects, mode generate|refine, refine requires current_chain), resolves the generator (empty name resolves the single registration, refuses when ambiguous), returns 501 when no generator exists, 502 on generator failure. Validation/replay stays with the caller's /pipeline/dry-run follow-up. --- modules/pipeline/ai_generate.go | 169 +++++++++++++++++++++++++++ modules/pipeline/ai_generate_test.go | 138 ++++++++++++++++++++++ modules/pipeline/module.go | 3 + 3 files changed, 310 insertions(+) create mode 100644 modules/pipeline/ai_generate.go create mode 100644 modules/pipeline/ai_generate_test.go diff --git a/modules/pipeline/ai_generate.go b/modules/pipeline/ai_generate.go new file mode 100644 index 00000000..5f198d47 --- /dev/null +++ b/modules/pipeline/ai_generate.go @@ -0,0 +1,169 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package pipeline + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "sync" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" +) + +// Chain generation is product logic (LLM backend, prompts, processor +// catalog wording); the endpoint is not. Products register a +// ChainGenerator at init and the unified POST /pipeline/ai-generate stays +// product-agnostic — the same shape as the unified dry-run endpoint. + +// ChainGenerateRequest is the product-agnostic input: parsed sample +// documents, a natural-language requirement and (refine mode) the chain +// to edit. +type ChainGenerateRequest struct { + Context context.Context + Documents []util.MapStr + Requirements string + // Mode: "generate" (default) | "refine" + Mode string + CurrentChain []map[string]interface{} +} + +// ChainGenerator turns samples plus a requirement into a processor chain +// (entries in the same shape dry-run takes: single-key processor maps or +// if/then/else). Generators return raw entries; validation and replay +// stay with the caller (the unified dry-run endpoint). +type ChainGenerator interface { + Name() string + Generate(req ChainGenerateRequest) ([]map[string]interface{}, error) +} + +var ( + chainGenMu sync.RWMutex + chainGens = map[string]ChainGenerator{} +) + +// RegisterChainGeneratorOrReplace registers (or replaces) a generator by +// name; the empty name is reserved for the framework. +func RegisterChainGeneratorOrReplace(g ChainGenerator) error { + if g == nil || g.Name() == "" { + return fmt.Errorf("chain generator must have a non-empty name") + } + chainGenMu.Lock() + defer chainGenMu.Unlock() + chainGens[g.Name()] = g + return nil +} + +// GetChainGenerator resolves a generator by name. +func GetChainGenerator(name string) (ChainGenerator, bool) { + chainGenMu.RLock() + defer chainGenMu.RUnlock() + g, ok := chainGens[name] + return g, ok +} + +// ChainGeneratorNames lists registered generators (sorted). +func ChainGeneratorNames() []string { + chainGenMu.RLock() + defer chainGenMu.RUnlock() + names := make([]string, 0, len(chainGens)) + for n := range chainGens { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// postAIGenerateHandler — POST /pipeline/ai-generate. +// body: {generator?, documents: [raw-JSON-string...], requirements?, +// mode?, current_chain?} → {chain} or {chain, generators} on 501/400. +// 501 when no generator is registered; 400 for bad input or an ambiguous +// empty generator name with multiple registrations; 502 when the +// generator fails. Validation/replay is the caller's follow-up call to +// /pipeline/dry-run. +func (module *PipeModule) postAIGenerateHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + var body struct { + Generator string `json:"generator"` + Documents []string `json:"documents"` + Requirements string `json:"requirements"` + Mode string `json:"mode"` + CurrentChain []map[string]interface{} `json:"current_chain"` + } + if err := module.DecodeJSON(req, &body); err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if len(body.Documents) == 0 { + module.WriteError(w, "documents (non-empty string array of sample document JSON) is required", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(body.Generator) + if name == "" { + names := ChainGeneratorNames() + switch len(names) { + case 0: + module.WriteError(w, "no chain generator registered in this build", http.StatusNotImplemented) + return + case 1: + name = names[0] + default: + module.WriteError(w, "multiple chain generators registered, specify one of: "+strings.Join(names, ", "), http.StatusBadRequest) + return + } + } + gen, ok := GetChainGenerator(name) + if !ok { + module.WriteError(w, "unknown chain generator "+name+" (registered: "+strings.Join(ChainGeneratorNames(), ", ")+")", http.StatusBadRequest) + return + } + + mode := body.Mode + if mode == "" { + mode = "generate" + } + if mode != "generate" && mode != "refine" { + module.WriteError(w, "mode must be generate or refine", http.StatusBadRequest) + return + } + if mode == "refine" && len(body.CurrentChain) == 0 { + module.WriteError(w, "refine requires current_chain", http.StatusBadRequest) + return + } + + // sample documents must be JSON objects — generators promise real docs + docs := make([]util.MapStr, 0, len(body.Documents)) + for i, raw := range body.Documents { + var doc util.MapStr + if err := util.FromJSONBytes([]byte(raw), &doc); err != nil { + module.WriteError(w, fmt.Sprintf("documents[%d] is not a valid JSON object", i), http.StatusBadRequest) + return + } + docs = append(docs, doc) + } + + chain, err := gen.Generate(ChainGenerateRequest{ + Context: req.Context(), + Documents: docs, + Requirements: body.Requirements, + Mode: mode, + CurrentChain: body.CurrentChain, + }) + if err != nil { + module.WriteError(w, truncateGenError(err.Error()), http.StatusBadGateway) + return + } + module.WriteJSON(w, util.MapStr{"chain": chain, "generator": name}, http.StatusOK) +} + +func truncateGenError(s string) string { + const cap = 300 + if len(s) <= cap { + return s + } + return s[:cap] + "…" +} diff --git a/modules/pipeline/ai_generate_test.go b/modules/pipeline/ai_generate_test.go new file mode 100644 index 00000000..de4bf0b0 --- /dev/null +++ b/modules/pipeline/ai_generate_test.go @@ -0,0 +1,138 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package pipeline + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" +) + +// stubGenerator is the ChainGenerator test double. +type stubGenerator struct { + name string + err error +} + +func (s stubGenerator) Name() string { return s.name } +func (s stubGenerator) Generate(req ChainGenerateRequest) ([]map[string]interface{}, error) { + if s.err != nil { + return nil, s.err + } + return []map[string]interface{}{{"test_upper": util.MapStr{}}}, nil +} + +func TestChainGeneratorRegistry(t *testing.T) { + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_a"}); err != nil { + t.Fatal(err) + } + if err := RegisterChainGeneratorOrReplace(nil); err == nil { + t.Fatal("nil generator must be rejected") + } + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: ""}); err == nil { + t.Fatal("empty name must be rejected") + } + if g, ok := GetChainGenerator("test_gen_a"); !ok || g.Name() != "test_gen_a" { + t.Fatal("registered generator must resolve") + } + if _, ok := GetChainGenerator("missing"); ok { + t.Fatal("unregistered name must not resolve") + } + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_a", err: fmt.Errorf("v2")}); err != nil { + t.Fatal(err) + } + if g, _ := GetChainGenerator("test_gen_a"); g.(stubGenerator).err == nil { + t.Fatal("replace must overwrite") + } +} + +func callAIGenerate(t *testing.T, body string) *httptest.ResponseRecorder { + t.Helper() + m := &PipeModule{} + req := httptest.NewRequest(http.MethodPost, "/pipeline/ai-generate", strings.NewReader(body)) + w := httptest.NewRecorder() + m.postAIGenerateHandler(w, req, httprouter.Params{}) + return w +} + +func TestAIGenerateHandler_UnimplementedWithoutGenerator(t *testing.T) { + // no generator carries this unique name; simulate a bare build by + // asking for a name that cannot exist + w := callAIGenerate(t, `{"generator":"__none__","documents":["{}"]}`) + if w.Code != http.StatusBadRequest { + t.Fatalf("unknown generator must be 400, got %d", w.Code) + } + if !strings.Contains(w.Body.String(), "unknown chain generator") { + t.Fatalf("unexpected body: %s", w.Body.String()) + } +} + +func TestAIGenerateHandler_HappyPathAndValidation(t *testing.T) { + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai"}); err != nil { + t.Fatal(err) + } + + w := callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{\"title\":\"a\"}"],"requirements":"add tags"}`) + if w.Code != http.StatusOK { + t.Fatalf("generate must succeed, got %d: %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), `"test_upper"`) || !strings.Contains(w.Body.String(), `"generator":"test_gen_ai"`) { + t.Fatalf("unexpected body: %s", w.Body.String()) + } + + // empty documents → 400 + if w := callAIGenerate(t, `{"documents":[]}`); w.Code != http.StatusBadRequest { + t.Fatalf("empty documents must be 400, got %d", w.Code) + } + // bad document JSON → 400 + if w := callAIGenerate(t, `{"documents":["not-json"]}`); w.Code != http.StatusBadRequest { + t.Fatalf("invalid document JSON must be 400, got %d", w.Code) + } + // bad mode → 400 + if w := callAIGenerate(t, `{"documents":["{}"],"mode":"nope"}`); w.Code != http.StatusBadRequest { + t.Fatalf("bad mode must be 400, got %d", w.Code) + } + // refine without current_chain → 400 + if w := callAIGenerate(t, `{"documents":["{}"],"mode":"refine"}`); w.Code != http.StatusBadRequest { + t.Fatalf("refine without chain must be 400, got %d", w.Code) + } + // generator failure → 502 + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai", err: fmt.Errorf("llm down")}); err != nil { + t.Fatal(err) + } + if w := callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{}"]}`); w.Code != http.StatusBadGateway { + t.Fatalf("generator error must be 502, got %d", w.Code) + } + // restore the working stub + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_ai"}); err != nil { + t.Fatal(err) + } +} + +// TestAIGenerateHandler_AmbiguousDefault: with a second generator the +// empty-name default must refuse instead of guessing. +func TestAIGenerateHandler_AmbiguousDefault(t *testing.T) { + if err := RegisterChainGeneratorOrReplace(stubGenerator{name: "test_gen_b"}); err != nil { + t.Fatal(err) + } + defer func() { + chainGenMu.Lock() + delete(chainGens, "test_gen_b") + chainGenMu.Unlock() + }() + w := callAIGenerate(t, `{"documents":["{}"]}`) + if w.Code != http.StatusBadRequest { + t.Fatalf("ambiguous default must be 400, got %d: %s", w.Code, w.Body.String()) + } + // explicit name still works + w = callAIGenerate(t, `{"generator":"test_gen_ai","documents":["{}"]}`) + if w.Code != http.StatusOK { + t.Fatalf("explicit generator must work, got %d", w.Code) + } +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index e6a19010..61db002d 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -250,6 +250,9 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler) api.HandleUIMethod(api.POST, "/pipeline/dry-run", module.postDryRunHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.POST, "/pipeline/ai-generate", module.postAIGenerateHandler) + api.HandleUIMethod(api.POST, "/pipeline/ai-generate", module.postAIGenerateHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler) api.HandleUIMethod(api.GET, "/pipeline/task/:id/stages", module.stagesPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) From 437d3c1319f770befc202a556e350a8187696841 Mon Sep 17 00:00:00 2001 From: medcl Date: Sat, 24 Oct 2026 22:23:28 +0800 Subject: [PATCH 49/65] feat(pipeline): canvas_layout field on PipelineConfigV2 for UI node positions Presentation-state-only mapping persisted alongside the config; keys use the "n" convention because ucfg drops purely numeric map keys (reads them as array indices). Not indexed, omitempty for legacy configs. --- core/pipeline/config.go | 11 +++++ core/pipeline/config_layout_test.go | 70 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 core/pipeline/config_layout_test.go diff --git a/core/pipeline/config.go b/core/pipeline/config.go index 2471fcf7..cfc9fd46 100644 --- a/core/pipeline/config.go +++ b/core/pipeline/config.go @@ -111,6 +111,17 @@ type PipelineConfigV2 struct { // Labels are arbitrary metadata attached to the pipeline. Labels map[string]interface{} `config:"labels" json:"labels"` + // CanvasLayout persists the orchestration UI's node positions: an + // arbitrary, UI-owned mapping (processor-entry key → {x, y}). Pure + // presentation state — never read by execution; stored with the + // config so any client editing the pipeline restores the same + // arrangement. Not indexed (arbitrary JSON shapes). + // + // Keys must NOT be purely numeric: ucfg reads digit-only keys as + // array indices and silently drops them from map unpacking, so "0" + // unpacks to nothing. Clients use a prefixed form such as "n0", "n1". + CanvasLayout map[string]interface{} `config:"canvas_layout" json:"canvas_layout,omitempty" elastic_mapping:"canvas_layout: { enabled: false }"` + // Transient, if false, marks that this pipeline config shpuld be persisted. // // In the current implementation, only the pipeline configs created with the diff --git a/core/pipeline/config_layout_test.go b/core/pipeline/config_layout_test.go new file mode 100644 index 00000000..649e153a --- /dev/null +++ b/core/pipeline/config_layout_test.go @@ -0,0 +1,70 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infini.ltd */ + +package pipeline + +import ( + "encoding/json" + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/util" +) + +// TestPipelineConfigV2CanvasLayoutRoundTrip (#109): the orchestration +// UI's node positions ride with the config through config unpacking and +// JSON serialization — presentation state only, but it must survive both +// directions untouched. +// +// Keys follow the "n" convention documented on the field: ucfg +// reads purely numeric map keys as array indices and drops them, so "0" +// would unpack to nothing. +func TestPipelineConfigV2CanvasLayoutRoundTrip(t *testing.T) { + cfgMap := util.MapStr{ + "name": "layout-probe", + "processor": []interface{}{ + util.MapStr{"echo": util.MapStr{"message": "hi"}}, + }, + "canvas_layout": util.MapStr{ + "n0": util.MapStr{"x": 120.5, "y": 40}, + "n1": util.MapStr{"x": 340, "y": 40}, + }, + } + c, err := config.NewConfigFrom(cfgMap) + if err != nil { + t.Fatal(err) + } + var cfg PipelineConfigV2 + if err := c.Unpack(&cfg); err != nil { + t.Fatal(err) + } + if cfg.CanvasLayout == nil { + t.Fatal("canvas_layout must unpack onto the config") + } + n0, _ := cfg.CanvasLayout["n0"].(map[string]interface{}) + if n0 == nil || n0["x"] != 120.5 { + t.Fatalf("layout node n0 lost: %v", cfg.CanvasLayout["n0"]) + } + + // JSON round-trip (API + persistence shape) + b, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + var back map[string]interface{} + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + layout, _ := back["canvas_layout"].(map[string]interface{}) + if len(layout) != 2 { + t.Fatalf("canvas_layout must survive JSON: %v", back["canvas_layout"]) + } + + // absent stays absent (omitempty) — legacy configs are unchanged + var legacy PipelineConfigV2 + lb, _ := json.Marshal(legacy) + if strings.Contains(string(lb), "canvas_layout") { + t.Fatalf("empty layout must not serialize: %s", lb) + } +} From 3806459fcc59b76f51e1bc861a3065c29f919f00 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 01:05:38 +0800 Subject: [PATCH 50/65] docs: lane reference with queue-vs-lane selection guide, pipeline glossary (P8 #118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - references/lane.md: in-process record channel — capacity/overflow policies (block/drop/spill and their hidden costs), dynamic lanes, lane_consumer, unified output step, cross-lane commit correctness (durable-first / ack receipts / inline), and the queue-vs-lane selection guide distilled from the design spec's appendix. - references/glossary.md: canonical terminology for message/batch/ record, scopes, contracts, channels, and the control plane. - README: compact terminology table linking to the full glossary. - references index lists both pages. Also fixes the docs build under Hugo >= 0.139: the release-notes entry for #415 contained hugo's internal placeholder prefix verbatim (HAHUGOSHORTCODE-...), which the placeholder replacer misparses as an unterminated shortcode token ('illegal state in content; shortcode token missing end delim'). Reworded the changelog line. (The theme's .Site.IsMultiLingual deprecation also fails 0.139 builds; the vendored theme is gitignored, patched locally.) Verified: hugo 0.139.3 build exits 0, both new pages render. --- README.md | 17 +++ docs/content.en/docs/references/_index.md | 2 + docs/content.en/docs/references/glossary.md | 50 +++++++++ docs/content.en/docs/references/lane.md | 111 +++++++++++++++++++ docs/content.en/docs/release-notes/_index.md | 2 +- 5 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 docs/content.en/docs/references/glossary.md create mode 100644 docs/content.en/docs/references/lane.md diff --git a/README.md b/README.md index 857d7142..7bb0c592 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,22 @@ # INFINI Framework +## Pipeline Terminology (glossary) + +The data-processing subsystems use a fixed vocabulary. The short version: + +| Term | Meaning | +|------|---------| +| **message** | A queue's unit of storage — opaque bytes. Queues never interpret payloads. | +| **batch** | The pipeline's native input: a slice of messages, the *batch scope* all historical processors operate on. | +| **record** | A typed, decoded event inside a record host's sub-chain — the *record scope*; mutable in place, never leaks downstream. | +| **record host** | A splitter processor that turns a batch into records and back: `for_each`, `lane_consumer`, or a custom host. | +| **codec** | Pluggable byte↔record converter (`otel` default, `coco_document`, ...). | +| **namespace** | A record family (`document`, `attachment`) used in compile-time contract checks. | +| **queue** | Durable at-least-once **byte** channel; cross-process safe. | +| **lane** | In-process at-most-once **record** channel (zero serialization, pointer hand-off). | + +Full definitions and the queue-vs-lane selection guide live in the docs: [Glossary](docs/content.en/docs/references/glossary.md), [Pipeline Record Processing](docs/content.en/docs/references/pipeline_record.md), [Lane](docs/content.en/docs/references/lane.md). + ## Requirements - Go 1.21+ diff --git a/docs/content.en/docs/references/_index.md b/docs/content.en/docs/references/_index.md index 308de1b4..fe6aa715 100644 --- a/docs/content.en/docs/references/_index.md +++ b/docs/content.en/docs/references/_index.md @@ -17,6 +17,8 @@ Comprehensive reference documentation for the INFINI Framework's core systems an - [Task Scheduling](task/) — Interval-based, cron-based, and transient task execution - [Pipeline Record Processing](pipeline_record/) — The record-processing convention: batch splitting, pluggable payload codecs, failure strategies, and batch-aware processors - [Queue](queue/) — Pluggable message queue abstraction with disk, memory, Kafka, and Redis backends +- [Lane](lane/) — In-process zero-serialization record channel: bounded capacity, overflow policies, `lane_consumer`, and the queue-vs-lane selection guide +- [Glossary](glossary/) — Canonical terminology for the pipeline, record-processing, and lane subsystems - [Key-Value Store](kv/) — Pluggable KV storage with Badger, Elasticsearch, and file-based backends - [Statistics](stats/) — Metrics collection with counters, gauges, timings, and StatsD integration - [Conditions](conditions/) — Declarative condition evaluation with logical operators for pipeline control flow diff --git a/docs/content.en/docs/references/glossary.md b/docs/content.en/docs/references/glossary.md new file mode 100644 index 00000000..c13e383c --- /dev/null +++ b/docs/content.en/docs/references/glossary.md @@ -0,0 +1,50 @@ +--- +title: "Glossary" +weight: 28 +--- + +# Glossary + +Canonical terminology for the pipeline, record-processing, and lane subsystems. When a term appears in configuration, code, or UI, this is what it means. + +## Messages, Batches, Records + +| Term | Meaning | +|------|---------| +| **message** | A queue's unit of storage: an opaque byte payload (`queue.Message`). Queues do not interpret messages. | +| **batch** | The pipeline's native input: a slice of messages delivered to a chain. All pipeline processors historically operate here (the *batch scope*). | +| **record** | A typed, decoded event (`*event.Event`) produced inside a record host. Fields, metadata, and timestamps are mutable in place. Records exist only inside a host's scope — they never leak to the outer batch chain. | +| **record host** (splitter host) | A processor that splits a batch into records, runs a sub-chain over them, and re-encodes: the built-in `for_each`, `lane_consumer`, or a custom host following the same skeleton. | +| **codec** (`RecordCodec`) | Pluggable encoder/decoder between message bytes and records (`otel` default, `coco_document`, `coco_attachment`, ...). Registered by name; selected with `codec:` on `for_each`, resolved three-level for output queues. | +| **namespace** | A record family identifier (`document`, `attachment`, ...) declared by codecs and lane configs; the compile-time currency of `RecordContract.Namespace`. | + +## Scopes and Contracts + +| Term | Meaning | +|------|---------| +| **batch scope** | Execution context at the chain root: processors see raw messages and pipeline parameters, not records. | +| **record scope** | Execution context inside a record host's sub-chain: `CurrentRecord` is bound, record field paths (`document.content`) are addressable in conditions. | +| **RecordContract** | A processor's compile-time declaration: where it may run (`Scope`), which record family it needs (`Namespace`), which paths it asserts (`RequiredFields`), and its `Granularity`. Unconstrained processors pass every placement. | +| **granularity** | Which execution channels a processor supports: `record` (per-record path only), `batch_capable` (both, batch preferred), `batch_only` (cross-record aggregation; must sit directly in a record host's sub-chain). | +| **strict_contract** | Pipeline config flag: placement violations become compile errors instead of warnings. The `POST /pipeline/_validate` gate reports them either way. | +| **failure tag** | A record-level error marker appended by `on_failure: tag`; accumulated tags persist into the record's `tags` field before re-encoding so downstream stages can route on them. | + +## Channels + +| Term | Meaning | +|------|---------| +| **queue** | Durable, at-least-once byte channel between producers and consumers (disk, memory, Kafka, Redis backends). Cross-process safe. | +| **lane** | In-process, zero-serialization record channel: payloads are `*event.Event` pointers. Bounded, at-most-once past a successful `Emit`. See [Lane](../lane/). | +| **overflow policy** | A full lane's behavior: `block` (backpressure, default), `drop` (best-effort), `spill:` (degrade to disk — only for equivalent fast/slow paths). | +| **durable bridge** (`output_queue`) | A `lane_consumer`'s optional exit: surviving records encode once into a disk queue, making the lane flow's outcome persistent and wire-compatible with existing consumers. | +| **AckSet** | Context-borne receipt set for `output{lane:…, ack: true}` steps; the emitting consumer's commit gate waits for settlement before committing offsets (cross-lane at-least-once). | + +## Control Plane and Debugging + +| Term | Meaning | +|------|---------| +| **processor catalog** | Discovery view of all registered processors (scope, contract, config schema) — `GET /pipeline/processors`. Drives canvas UIs and schema forms. | +| **dry-run** | Server-side chain execution against a sample with side effects replaced by captures: step snapshots, field diffs, per-step timings. `POST /pipeline/dry-run`. | +| **placement** | Compile-time check that a processor's scope/granularity matches its position (e.g. a `batch_only` processor must sit directly in a record host's sub-chain). | +| **dynamic lane** | A lane created at runtime (`lane.Create` / `GetOrCreate`) with an `owner` for cascade cleanup — e.g. one per compiled stream pipeline, never shared. | +| **canvas_layout** | Product-owned pipeline config field holding canvas node coordinates (`map[string]{x,y}`, content-fingerprint keys). Pure presentation state; execution never reads it. | diff --git a/docs/content.en/docs/references/lane.md b/docs/content.en/docs/references/lane.md new file mode 100644 index 00000000..2f63cb26 --- /dev/null +++ b/docs/content.en/docs/references/lane.md @@ -0,0 +1,111 @@ +--- +title: "Lane" +weight: 27 +--- + +# Lane — In-Process Record Channel + +A **lane** is the in-process, zero-serialization counterpart of the [queue](../queue/): payloads are `*event.Event` **pointers** — the record *is* the message. A producer emits a record into a lane; a `lane_consumer` host drains it into a record-processing sub-chain. Nothing is encoded or decoded in transit. + +``` +producer ──Emit(*event.Event)──▶ [lane: bounded, in-flight] ──▶ lane_consumer + │ batch{max, flush_ms} │ + │ ▼ + overflow policy record sub-chain + (block/drop/spill) (same execution + contract as + for_each) +``` + +Use the record-processing model's [conventions](../pipeline_record/) inside the sub-chain — `CurrentRecord`, `MarkDropped`, failure tags all behave identically to `for_each`. + +## Declaring Lanes + +```yaml +lane: + - name: mem_processing + namespace: document # record-family contract for compile checks + capacity: 4096 # bounded; in-flight records + overflow: block # block | drop | spill: + - name: debug_samples + overflow: drop +``` + +Configured lanes live in the `lane` section; **dynamic lanes** are created at runtime with `lane.Create(cfg)` / `lane.GetOrCreate(name, cfg)` and carry an `owner` for cascade cleanup (e.g. one lane per compiled stream pipeline, never shared — sharing a lane between consumers is a semantic error). Dynamic lanes default to a small capacity (512) and are lazy: an idle lane costs nothing until first emit. A global `total_budget` caps aggregate in-flight records across dynamic lanes; when the budget is hit each lane applies its own overflow policy. + +## Overflow Policies + +| Policy | Behavior | When it is correct | +|--------|----------|-------------------| +| `block` (default) | `Emit` waits; backpressure reaches the producer | Whenever the producer can safely slow down (task goroutines, HTTP handlers answering 429). The honest default — data is never risked. | +| `drop` | Discard and count (`drops` metric) | Best-effort telemetry only: tap sampling, debug indexes. | +| `spill:` | Degrade the record to the disk queue | **Only** when the fast and slow paths run *equivalent* chains — spill means "this record takes the slow version of the same path". | + +Two hidden costs of `spill`: a required fast-path step (e.g. redaction) is bypassed for spilled records — a correctness bug if the paths differ; and it can mask backpressure until the spill queue grows unbounded. If the producer cannot block **and** the data must be durable, the honest answer is queue-first (persist, then ACK), not spill. + +Spilled records **never flow back into the lane**. The slow path's consumer picks them up on its regular polling cycle, so: a record's path is decided once (`TryEmit` failure is an atomic fork — no double delivery); spilled records survive crashes while in-lane records do not; and completion order across the two paths is not guaranteed — use `block` (lane is FIFO) when ordering matters. + +## lane_consumer + +```yaml +processor: + - lane_consumer: + lane: mem_processing + batch_max: 200 # flush on size + batch_flush_ms: 1000 # flush on idle + on_failure: tag # tag (default) | dead_letter: | ignore + failure_tag: _processing_failed + output_queue: indexing_documents # optional durable bridge (encode once) + processor: [...] # record sub-chain +``` + +The execution contract matches `for_each` exactly (stage-wise, order as written, drop/clone/failure-tag semantics identical). Two differences: the entry is zero-decode (records are constructed by the emitting side, not decoded from bytes), and the exit encodes **only** when `output_queue` is configured — `queue.Message.Data = codec.Encode(record)`, wire-compatible with every existing consumer of that codec. + +Codec resolution for `output_queue` is three-level: the record's `_codec` hint → the lane's `namespace` primary codec → the default `otel`. + +## The Unified `output` Step + +Emit into a lane from inside a chain with the [unified output](../pipeline/) step — exactly one target field, enforced at config unpacking: + +```yaml +- output: + lane: mem_processing + ack: true # attach a receipt to the context's AckSet +# or: +- output: + queue: indexing_documents +``` + +In **batch scope** a queue target pushes raw message bytes; a lane target is a compile error (point the consumer's `codec` + `output_lane` bridging at the lane instead). In **record scope** a queue target encodes the current record; a lane target emits the pointer. `queue_output` / `lane_output` remain as aliases. + +## Queue or Lane? — Selection Guide + +**The boundary in one sentence:** a successful `Emit` return is the end of the producer's responsibility — lanes are **at-most-once**. A lane flow is safe when at least one of these holds: + +1. the upstream can recompute (re-collect, upstream retries, data is disposable), or +2. an uncommitted queue offset covers the whole flow (a crash replays it), or +3. the traffic is best-effort by nature (debugging, sampling). + +| Signal | Choose | +|--------|--------| +| Every record must survive crashes | **Queue** — disk persistence, at-least-once redelivery | +| Chain contains non-replayable steps (LLM enrichment, external calls with side effects) | **Queue** (durable path; `for_each` decodes once on each side) | +| Cross-process or cross-machine delivery | **Queue** — lanes are in-process only | +| Light, deterministic pre-processing between an in-process producer and disk | **Lane** (`lane_consumer` + `output_queue` durable bridge; `block` overflow) | +| Webhook-style intake where the caller retries | **Lane** with `block` (backpressure surfaces as 429/slow response) or `drop` | +| Observation taps, sampled debug feeds | **Lane** with `drop` | +| Fan-out after a durable point | **Lane** with `block` — `output{lane:…}` branches after the offset-covered durable write | + +### Commit Correctness Across Lanes + +`output{lane:…}` is **not** a commit point; a consumer's offset only covers durable writes inside its own sub-chain. Three correct postures, by strength needed: + +| Posture | Shape | Guarantee | +|---------|-------|-----------| +| durable-first (default) | `output{queue:…}` inside the sub-chain (the commit covers it); the lane branch only does optional extras | Correct at zero cost — a crash replays from the queue | +| ack receipts | `output{lane:…, ack: true}` settles into the context **AckSet**; the emitting consumer's commit gate waits for settlement (success → `output_queue`, failure → `dead_letter`, explicit drop — all count), undecided or past the consumer's `ack_timeout_ms` (default 30s) → no commit → the disk queue replays | Cross-lane at-least-once: never lost, possibly duplicated (make sinks idempotent — ES `_id` writes are) | +| inline | When "commit only if processed" must be strict, don't decouple: run `for_each` with the durable write in the sub-chain | Strongest and simplest | + +## Observability + +`GET /pipeline/lanes` returns a live snapshot of every lane: `depth`, `capacity`, in/out counters (`emitted`, `delivered`), `drops`, `spills`, `blocked_waits` (how often `block` overflow actually waited), `owner`, and `closed` — the in-process counterpart of queue-depth observability. diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 495b59a2..57b9651b 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -68,7 +68,7 @@ Information about release notes of INFINI Framework is provided here. - refactor: rate limit the large cluster warning in metric collection #394 - docs: catalog built-in processors and condition operators in the pipeline/conditions references #414 - docs: per-processor and per-condition reference sections — one page for each of the 51 registered processors and 16 condition operators, in a uniform format (category/scope, configuration parameters with types and defaults, examples) #414 -- fixdoc: replace relref shortcodes embedded in link destinations with plain relative page links, silencing the docs build's "Page 'HAHAHUGOSHORTCODE-...' not found" warnings #415 +- fixdoc: replace relref shortcodes embedded in link destinations with plain relative page links, silencing the docs build's broken-relref "Page not found" warnings #415 ## 1.4.2 (2026-06-23) ### ❌ Breaking changes From 9f17a4a30fe3d542563f9be90dc3445de42f832c Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 01:15:42 +0800 Subject: [PATCH 51/65] =?UTF-8?q?feat(pipeline):=20for=5Feach=20adopts=20t?= =?UTF-8?q?he=20bound=20record=20in=20dry-run=20=E2=80=94=20sub-chains=20r?= =?UTF-8?q?eplay=20for=20real?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dry-run replays bind one decoded record to the context and carry no message batch, so for_each's batch lookup missed and the host silently no-opped: every chain with a for_each step replayed as 'no effect', which hid the migration-equivalence question and made canvas dry-runs of wrapped chains meaningless. In dry-run mode with no batch and a bound record, for_each now adopts that record as its single input (the zero-decode typed entry lane_consumer uses at runtime) and runs the sub-chain stage-wise on it. Adopted records: serial execution (deterministic diffs), no encode pass, no pool release — the harness owns the record. Outside dry-run the historical no-batch skip stands. Process is restructured around runOnRecords(records, msgs) shared by the batch path and the adoption path. Tests: adoption mutates and restores the bound record; non-dry-run no-batch stays a no-op; full for_each suite green. --- core/pipeline/ackset.go | 10 ++++ modules/pipeline/for_each.go | 72 ++++++++++++++++++------ modules/pipeline/for_each_dryrun_test.go | 59 +++++++++++++++++++ 3 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 modules/pipeline/for_each_dryrun_test.go diff --git a/core/pipeline/ackset.go b/core/pipeline/ackset.go index e8a369c5..1cfe535f 100644 --- a/core/pipeline/ackset.go +++ b/core/pipeline/ackset.go @@ -213,6 +213,16 @@ func EnableDryRun(ctx *Context, sink *[]DryRunWrite, capture bool) { ctx.Set(dryRunKey, &dryRunState{sink: sink, capture: capture}) } +// InDryRun reports whether this context runs under a dry-run replay +// (EnableDryRun was called), regardless of capture mode. +func InDryRun(ctx *Context) bool { + if ctx == nil { + return false + } + st, _ := ctx.Get(dryRunKey).(*dryRunState) + return st != nil +} + type dryRunState struct { sink *[]DryRunWrite capture bool diff --git a/modules/pipeline/for_each.go b/modules/pipeline/for_each.go index 5e357498..0f194b16 100644 --- a/modules/pipeline/for_each.go +++ b/modules/pipeline/for_each.go @@ -186,13 +186,6 @@ func (p *ForEachProcessor) ApplyScope(scope pipeline.ScopeInfo) error { } func (p *ForEachProcessor) Process(c *pipeline.Context) error { - v := c.Get(param.ParaKey(p.cfg.MessageField)) - msgs, ok := v.([]queue.Message) - if !ok { - log.Debugf("for_each: no batch found under context key [%s], skip", p.cfg.MessageField) - return nil - } - // The record convention is only valid inside the sub-chain; save and // restore the previous values so the batch's last record (and its // failure tags) does not leak to processors downstream of for_each, @@ -204,6 +197,24 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { c.Set(pipeline.FailureTagsKey, previousTags) }() + v := c.Get(param.ParaKey(p.cfg.MessageField)) + msgs, ok := v.([]queue.Message) + if !ok { + // Dry-run replay: the harness binds one decoded record instead of + // a message batch. Adopt it as the single input record — the same + // zero-decode typed entry lane_consumer uses at runtime — so the + // sub-chain really executes and step diffs show its effect. There + // is no message to encode back into; the harness owns the record + // (runOnRecords skips release and encode for adopted records). + if pipeline.InDryRun(c) { + if rec, has := pipeline.CurrentRecord(c); has { + return p.runOnRecords(c, []*event.Event{rec}, nil, nil) + } + } + log.Debugf("for_each: no batch found under context key [%s], skip", p.cfg.MessageField) + return nil + } + // Decode pass: build the record slice; undecodable payloads keep their // original bytes and are passed through untouched (payload transparency // for mixed batches). @@ -221,7 +232,15 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { records = append(records, rec) decodedIdx = append(decodedIdx, i) } + return p.runOnRecords(c, records, msgs, decodedIdx) +} +// runOnRecords executes the sub-chain stage-wise over records. With msgs +// non-nil (normal runtime) originals encode back into their message slot, +// clones append messages, and pooled records are released. With msgs nil +// (dry-run adoption) the caller owns the records: serial execution only +// (deterministic diffs), no encode pass, no release. +func (p *ForEachProcessor) runOnRecords(c *pipeline.Context, records []*event.Event, msgs []queue.Message, decodedIdx []int) error { segments := buildSegments(p.sub.List) // Stage-wise execution: segments run in WRITTEN order (record runs, @@ -231,16 +250,30 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { // segments. active := make([]forEachActive, 0, len(records)) for j, rec := range records { - active = append(active, forEachActive{rec: rec, msgIdx: decodedIdx[j]}) + idx := -1 + if msgs != nil { + idx = decodedIdx[j] + } + active = append(active, forEachActive{rec: rec, msgIdx: idx}) + } + // offsetOf is for logging only; adopted records carry no message. + offsetOf := func(ar *forEachActive) interface{} { + if msgs == nil || ar.msgIdx < 0 { + return -1 + } + return msgs[ar.msgIdx].Offset } // release pooled records on every exit path — by now each record has // been encoded (or dropped) and clones have joined active; non-pooled - // records (codec- or clone-allocated) make ReleaseEvent a no-op - defer func() { - for i := range active { - event.ReleaseEvent(active[i].rec) - } - }() + // records (codec- or clone-allocated) make ReleaseEvent a no-op. + // Adopted dry-run records belong to the harness — never released here. + if msgs != nil { + defer func() { + for i := range active { + event.ReleaseEvent(active[i].rec) + } + }() + } for _, seg := range segments { if c.IsCanceled() || !c.ShouldContinue() { @@ -274,7 +307,8 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { } // record segment: per active record, run the segment's sub-chain - if p.cfg.Parallel > 1 { + // (dry-run adoption stays serial for deterministic diffs) + if p.cfg.Parallel > 1 && msgs != nil { runRecordSegmentParallel(c, seg.chain, active, p.cfg, func(cl *event.Event, idx int) { active = append(active, forEachActive{rec: cl, msgIdx: idx, isClone: true}) }) @@ -302,7 +336,7 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { tagged = true } log.Warnf("for_each: sub-chain error tagged %q on record at offset %v: %v", - p.cfg.FailureTag, msgs[ar.msgIdx].Offset, err) + p.cfg.FailureTag, offsetOf(ar), err) } } mergeFailureTags(ar.rec, tags) @@ -320,6 +354,12 @@ func (p *ForEachProcessor) Process(c *pipeline.Context) error { } } + // Adopted dry-run records have no message to encode back into — the + // harness snapshots the record directly. + if msgs == nil { + return nil + } + // Encode pass: originals write their message slot, clones append new // messages sharing the parent's offset semantics. for i := range active { diff --git a/modules/pipeline/for_each_dryrun_test.go b/modules/pipeline/for_each_dryrun_test.go new file mode 100644 index 00000000..cccb4eb3 --- /dev/null +++ b/modules/pipeline/for_each_dryrun_test.go @@ -0,0 +1,59 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +/* for_each under a dry-run replay: the dryrun harness binds one decoded + * record and no message batch. The host must adopt the bound record (the + * zero-decode typed entry) and actually run the sub-chain on it — before + * this path existed, for_each silently no-opped in every dry-run, so + * migrated legacy chains replayed as "no effect". */ + +import ( + "testing" + + "infini.sh/framework/core/event" + "infini.sh/framework/core/pipeline" +) + +func TestForEach_DryRunAdoptsBoundRecord(t *testing.T) { + p := buildForEach(t, nil) + + rec := event.AcquireEvent() + rec.Fields = map[string]interface{}{"message": "hello"} + ctx := &pipeline.Context{Context: t.Context()} + ctx.Set(pipeline.RecordContextKey, rec) + sink := &[]pipeline.DryRunWrite{} + pipeline.EnableDryRun(ctx, sink, true) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + + if got := rec.Fields["message"]; got != "PROCESSED:hello" { + t.Fatalf("sub-chain did not run on the adopted record: message=%v", got) + } + // the harness's record must still be the bound record afterwards + bound, ok := pipeline.CurrentRecord(ctx) + if !ok || bound != rec { + t.Fatal("bound record not restored after dry-run adoption") + } +} + +// outside dry-run the historical behavior stands: no batch, no-op +func TestForEach_NoBatchOutsideDryRunSkips(t *testing.T) { + p := buildForEach(t, nil) + + rec := event.AcquireEvent() + rec.Fields = map[string]interface{}{"message": "hello"} + ctx := &pipeline.Context{Context: t.Context()} + ctx.Set(pipeline.RecordContextKey, rec) + + if err := p.Process(ctx); err != nil { + t.Fatal(err) + } + if got := rec.Fields["message"]; got != "hello" { + t.Fatalf("non-dry-run run must not touch the bound record: message=%v", got) + } +} From 539e914d4a73c4d52a9111bae5b74eeb7bd35e40 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 01:41:38 +0800 Subject: [PATCH 52/65] refactor(pipeline): dry-run state rides the Go context chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnableDryRun stored its state in the pipeline Context's parameter map, which is invisible to code holding a context DERIVED from ctx.Context — exactly what processor helpers do (utils.EachDocumentRecord callbacks hand down plain contexts), so deep layers (orm, attachment stores) could never see dry-run mode. The state now lives in a context.WithValue chain installed on ctx.Context; InDryRun/DryRunCapture read through it, and new InDryRunCtx/DryRunCaptureCtx accept plain contexts for layers below processors. AcquireContext builds fresh Contexts, so nothing leaks across acquisitions. DryRunWrite.Kind comment widened (attachment captures coming from coco's blob-store guard). Framework suites green; coco's attachment dry-run guard (next commit) consumes DryRunCaptureCtx. --- core/pipeline/ackset.go | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/core/pipeline/ackset.go b/core/pipeline/ackset.go index 1cfe535f..c8171a62 100644 --- a/core/pipeline/ackset.go +++ b/core/pipeline/ackset.go @@ -192,16 +192,20 @@ func receiptSettled(r Receipt) bool { // ─── dry-run interception ──────────────────────────────────────────────────── -// DryRunWrite is one captured side effect (queue/lane write intercepted -// during a dry run — never delivered). +// DryRunWrite is one captured side effect (queue/lane/attachment write +// intercepted during a dry run — never delivered). type DryRunWrite struct { - Kind string // queue | lane + Kind string // queue | lane | attachment (host-specific labels welcome) Target string Records int Preview string } -const dryRunKey = "__dry_run_sink" +// dryRunCtxKey rides the Go context chain: processor helpers derive plain +// contexts from ctx.Context and never see the pipeline wrapper's parameter +// store, so the state must travel with the embedded chain to stay visible +// under orm/attachment layers. +type dryRunCtxKey struct{} // EnableDryRun marks the context as a dry run; when capture is true, // output processors append intercepted writes to sink instead of @@ -210,7 +214,21 @@ func EnableDryRun(ctx *Context, sink *[]DryRunWrite, capture bool) { if ctx == nil { return } - ctx.Set(dryRunKey, &dryRunState{sink: sink, capture: capture}) + st := &dryRunState{sink: sink, capture: capture} + ctx.Context = context.WithValue(ctx.Context, dryRunCtxKey{}, st) +} + +type dryRunState struct { + sink *[]DryRunWrite + capture bool +} + +func dryRunStateFrom(ctx context.Context) *dryRunState { + if ctx == nil { + return nil + } + st, _ := ctx.Value(dryRunCtxKey{}).(*dryRunState) + return st } // InDryRun reports whether this context runs under a dry-run replay @@ -219,13 +237,13 @@ func InDryRun(ctx *Context) bool { if ctx == nil { return false } - st, _ := ctx.Get(dryRunKey).(*dryRunState) - return st != nil + return dryRunStateFrom(ctx.Context) != nil } -type dryRunState struct { - sink *[]DryRunWrite - capture bool +// InDryRunCtx is the context-chain variant for code holding a derived +// plain context (below processors that re-wrap the parent). +func InDryRunCtx(ctx context.Context) bool { + return dryRunStateFrom(ctx) != nil } // DryRunCapture returns the sink when this context runs in capturing @@ -234,7 +252,17 @@ func DryRunCapture(ctx *Context) *[]DryRunWrite { if ctx == nil { return nil } - st, _ := ctx.Get(dryRunKey).(*dryRunState) + return dryRunSinkFrom(ctx.Context) +} + +// DryRunCaptureCtx is the context-chain variant for code deep under a +// pipeline (orm/attachment layers holding derived contexts). +func DryRunCaptureCtx(ctx context.Context) *[]DryRunWrite { + return dryRunSinkFrom(ctx) +} + +func dryRunSinkFrom(ctx context.Context) *[]DryRunWrite { + st := dryRunStateFrom(ctx) if st == nil || !st.capture { return nil } From c02da194fb2a931a4b92c7c72bc92611537dbeff Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 02:00:29 +0800 Subject: [PATCH 53/65] feat(pipeline): ORM pipeline edits hot-reload the local running task (#131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PUT/POST/DELETE /pipelines used to only persist: a locally running task kept its stale compiled chain until the instance restarted. The CRUD handlers now trigger a rebuild that mirrors the fsnotify reload path — drain the in-flight batch (stopAndWaitForRelease), drop the task, and recreate from the new config. Matching is strict on task ID, so deployment-rendered tasks (which keep the ORM pipeline ID) reload while unrelated local YAML tasks stay untouched; transient tasks are skipped. A per-task generation counter (1 = the startup version) rides the runtime snapshot so controllers can observe that a reload landed. Rebuilds run off the request path and are serialized — rapid successive edits converge on the newest config instead of racing LoadOrStore. --- modules/pipeline/hotreload_test.go | 111 +++++++++++++++++++++++++++++ modules/pipeline/module.go | 89 ++++++++++++++++++++++- modules/pipeline/pipeline.go | 11 +++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 modules/pipeline/hotreload_test.go diff --git a/modules/pipeline/hotreload_test.go b/modules/pipeline/hotreload_test.go new file mode 100644 index 00000000..cd76e495 --- /dev/null +++ b/modules/pipeline/hotreload_test.go @@ -0,0 +1,111 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package pipeline + +import ( + "sync" + "testing" + + "infini.sh/framework/core/pipeline" +) + +// A processor name unique to this file — the plugin registry is +// process-global and Setup() (which registers "echo") never runs in tests, +// so a plain RegisterProcessorPlugin here cannot collide. +var hotReloadOnce sync.Once + +func hotReloadTestProcessor() string { + const name = "test_hotreload_echo" + hotReloadOnce.Do(func() { + pipeline.RegisterProcessorPlugin(name, NewEchoProcessor) + }) + return name +} + +func hotCfg(id, procName, message string) pipeline.PipelineConfigV2 { + cfg := pipeline.PipelineConfigV2{ + Name: id, + AutoStart: true, + Processors: []map[string]interface{}{ + {procName: map[string]interface{}{"message": message}}, + }, + } + cfg.ID = id + return cfg +} + +// TestHotReload_SwapsRunningTask covers the ORM CRUD reload path (#131): +// a semantically identical edit is a no-op, a changed config drains and +// rebuilds the task with a generation bump, delete removes the task, and +// unknown IDs / transient tasks are left untouched. +func TestHotReload_SwapsRunningTask(t *testing.T) { + proc := hotReloadTestProcessor() + m := &PipeModule{} + + cfg1 := hotCfg("hr-1", proc, "v1") + if err := m.createPipelineTask(cfg1, false); err != nil { + t.Fatalf("create v1: %v", err) + } + if _, ok := m.contexts.Load("hr-1"); !ok { + t.Fatal("task context missing after create") + } + + // semantically identical edit: no rebuild, no generation bump + same := hotCfg("hr-1", proc, "v1") + m.hotReloadPipeline("hr-1", &same) + if got := m.generationOf("hr-1"); got != 1 { + t.Fatalf("no-op edit bumped generation to %d", got) + } + + // changed config: drain, rebuild, generation increments + cfg2 := hotCfg("hr-1", proc, "v2") + m.hotReloadPipeline("hr-1", &cfg2) + stored, ok := m.configs.Load("hr-1") + if !ok { + t.Fatal("task config missing after hot reload") + } + if !stored.(pipeline.PipelineConfigV2).Equals(cfg2) { + t.Fatalf("task not swapped to the new config: %+v", stored) + } + if _, ok := m.contexts.Load("hr-1"); !ok { + t.Fatal("task context missing after hot reload") + } + if got := m.generationOf("hr-1"); got != 2 { + t.Fatalf("generation = %d, want 2", got) + } + + // the runtime report carries the generation so controllers can see + // the reload landed + snap := m.RuntimeSnapshot() + entry, ok := snap["hr-1"].(map[string]interface{}) + if !ok { + t.Fatalf("runtime snapshot has no entry for hr-1: %+v", snap) + } + if entry["generation"] != 2 { + t.Fatalf("runtime snapshot generation = %v, want 2", entry["generation"]) + } + + // delete: stop and remove, no recreate + m.hotReloadPipeline("hr-1", nil) + if _, ok := m.configs.Load("hr-1"); ok { + t.Fatal("config survives a delete hot reload") + } + if _, ok := m.contexts.Load("hr-1"); ok { + t.Fatal("context survives a delete hot reload") + } + + // unknown ID: no-op, no panic + m.hotReloadPipeline("no-such-pipeline", nil) + + // transient tasks are never reloaded + cfgT := hotCfg("hr-transient", proc, "t") + if err := m.createPipelineTask(cfgT, true); err != nil { + t.Fatalf("create transient: %v", err) + } + m.hotReloadPipeline("hr-transient", nil) + if _, ok := m.configs.Load("hr-transient"); !ok { + t.Fatal("transient task was reloaded") + } +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 61db002d..5df6ac80 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -64,6 +64,10 @@ type PipeModule struct { // instance (e.g. a processor missing from the local catalog) — the // deployment status needs to see WHY, not just that it is absent failures sync.Map // name -> string (compile/load error) + // generations counts the config revisions applied per task (1 = the + // startup version); each successful hot reload increments it and the + // runtime report exposes it so controllers can see a reload landed + generations sync.Map // taskID -> int } // recordFailure stores a pipeline compile/load failure for the runtime @@ -106,9 +110,13 @@ func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { case pipeline.STOPPED, pipeline.FINISHED: state = "STOPPED" } - out[name] = map[string]interface{}{ + entry := map[string]interface{}{ "state": state, } + if taskID, ok := k.(string); ok { + entry["generation"] = module.generationOf(taskID) + } + out[name] = entry return true }) // pipelines that never made it into contexts (config rejected or @@ -623,6 +631,85 @@ const pipelineSingleton = "pipeline_singleton" var creatingLocker = sync.Mutex{} +// hotReloadLocker serializes hot reloads — two rapid API edits to the same +// pipeline would otherwise race their stop/delete/create sequences and +// LoadOrStore could converge on the older config. +var hotReloadLocker = sync.Mutex{} + +// generationOf returns the config revision counter for a task (1 = the +// startup version; each successful hot reload increments it). +func (module *PipeModule) generationOf(taskID string) int { + if v, ok := module.generations.Load(taskID); ok { + if n, ok := v.(int); ok && n > 0 { + return n + } + } + return 1 +} + +// hotReloadPipeline swaps the locally running task for pipelineID to newCfg +// (nil = stop and remove only) — the ORM CRUD path. API edits used to only +// persist: a running task kept its stale compiled chain until the instance +// restarted. Same semantics as the fsnotify reload: drain the in-flight +// batch (stopAndWaitForRelease), then rebuild from the new config. +// Transient tasks are never reloaded. +// +// Matching is strict on task ID — deployment-rendered configs keep the ORM +// pipeline ID, so an edit reaches every instance running the pipeline, +// while unrelated local YAML tasks (their own IDs) stay untouched. +func (module *PipeModule) hotReloadPipeline(pipelineID string, newCfg *pipeline.PipelineConfigV2) { + v, ok := module.configs.Load(pipelineID) + if !ok { + return + } + oldC, ok := v.(pipeline.PipelineConfigV2) + if !ok { + return + } + if oldC.Transient { + log.Debugf("pipeline [%s] is transient, skip hot reload", pipelineID) + return + } + if newCfg != nil && oldC.Equals(*newCfg) && isPipelineEnabled(newCfg.Enabled) { + return // semantically unchanged — nothing to rebuild + } + + hotReloadLocker.Lock() + defer hotReloadLocker.Unlock() + + defer func() { + if !global.Env().IsDebug { + if r := recover(); r != nil { + log.Error("error on hot reloading pipeline,", pipelineID, ",", r) + } + } + }() + + log.Infof("pipeline [%s] config changed via API, reloading task", pipelineID) + module.stopAndWaitForRelease([]string{pipelineID}, time.Minute) + module.deleteTask(pipelineID) + + if newCfg == nil { + return + } + if err := module.createPipelineTask(*newCfg, false); err != nil { + module.recordFailure(newCfg.Name, err) + log.Errorf("pipeline [%s] hot reload: recreate failed: %v", pipelineID, err) + return + } + module.generations.Store(pipelineID, module.generationOf(pipelineID)+1) +} + +// reloadPipelineTaskAsync runs the hot reload off the request path — +// draining the current batch can legitimately take seconds, and the API +// response must not wait for it. +func (module *PipeModule) reloadPipelineTaskAsync(pipelineID string, newCfg *pipeline.PipelineConfigV2) { + if _, ok := module.configs.Load(pipelineID); !ok { + return // no local task runs this ORM row + } + go module.hotReloadPipeline(pipelineID, newCfg) +} + // Helper function to run a pipeline, i.e., create a pipeline task. func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transient bool) error { if module.closed.Load() { diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go index 9ff06c6d..573ea887 100644 --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -43,6 +43,10 @@ func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Requ return } + // a local task may already run under this ID (e.g. a deployment- + // rendered config) — swap it to the freshly created definition + h.reloadPipelineTaskAsync(obj.ID, obj) + h.WriteCreatedOKJSON(w, obj.ID) } @@ -131,6 +135,10 @@ func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Requ return } + // hot-reload any local task running this pipeline — the edit used to + // only persist, leaving a running chain stale until restart + h.reloadPipelineTaskAsync(newConfig.ID, &newConfig) + h.WriteUpdatedOKJSON(w, newConfig.ID) } @@ -170,6 +178,9 @@ func (h *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Requ return } + // stop and remove any local task running this pipeline + h.reloadPipelineTaskAsync(id, nil) + h.WriteDeletedOKJSON(w, id) } From 3d75258c658ec399185adf1ff4c263e95137de77 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 02:13:20 +0800 Subject: [PATCH 54/65] =?UTF-8?q?feat(pipeline):=20fleet=20control=20primi?= =?UTF-8?q?tives=20=E2=80=94=20generation=20guard,=20drain,=20cordon,=20lo?= =?UTF-8?q?ck-takeover=20events=20(#120=20#121=20#122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #120 Control ops (_pause/_resume/_start/_stop and the new _drain) accept an optional ?generation= guard: an instruction applies only when the local task runs exactly that config revision, so an instance still on an older generation (hot reload not landed) never acts on instructions issued for the new one, and vice versa. Mismatches answer 409 carrying both generations; an absent param keeps today's unconditional behavior. #121 Graceful drain: a Drain flag on the pipeline context tells consumers to stop FETCHING — the in-flight batch finishes, its offset commits, then the worker returns (as opposed to CancelTask, which aborts mid-batch for redelivery). Endpoints: POST /pipeline/task/:id/_drain (parks the task STOPPED, resumable) and POST /pipeline/_drain (drain every task, one bounded shared wait). Cordon: POST /pipeline/_cordon / _uncordon (status via GET) — a cordoned instance lets running tasks finish but parks every start path (auto_start, _start, _resume, hot- reload recreations) until uncordoned. Restart clears the drain flag so a restarted consumer does not immediately re-drain. #122 Singleton lock takeovers are now visible: before re-acquiring an expired foreign lease, the task loop probes the previous owner and files a lock-takeover StepEvent (new Outcome) into the flight ring — the alert surface operators watch — plus a warn log. --- core/pipeline/context.go | 24 +++ core/pipeline/observer.go | 6 + modules/pipeline/control_test.go | 264 +++++++++++++++++++++++++++++ modules/pipeline/flight.go | 16 ++ modules/pipeline/module.go | 139 ++++++++++++++- modules/pipeline/tasks.go | 120 +++++++++++++ plugins/queue/consumer/consumer.go | 19 ++- 7 files changed, 579 insertions(+), 9 deletions(-) create mode 100644 modules/pipeline/control_test.go diff --git a/core/pipeline/context.go b/core/pipeline/context.go index e6faa2c7..a462e5a0 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -105,6 +105,10 @@ type Context struct { pause sync.WaitGroup // Set this to true if you want to stop the pipeline, and then, pause (suspend) the goroutine. isQuit bool + // drainRequested asks long-running consumers to stop FETCHING new + // work: finish the in-flight batch, commit its offset, then return — + // as opposed to CancelTask, which aborts mid-batch (redelivery). + drainRequested bool stateLock sync.Mutex // Set this to true if you want to let the goroutine exit, i.e., the kill signal. released bool @@ -370,6 +374,9 @@ func (ctx *Context) Restart() { defer ctx.stateLock.Unlock() ctx.isQuit = false + // a fresh run is no longer draining — otherwise a consumer started + // after _drain would immediately drain again + ctx.drainRequested = false } // IsExit means pipeline has been manually stopped, will not running until Restart @@ -388,6 +395,23 @@ func (ctx *Context) Exit() { ctx.isQuit = true } +// Drain requests a graceful stop: consumers finish the current batch, +// commit its offset, then wind down instead of fetching more work. +func (ctx *Context) Drain() { + ctx.stateLock.Lock() + defer ctx.stateLock.Unlock() + + ctx.drainRequested = true +} + +// IsDraining reports whether a graceful drain was requested. +func (ctx *Context) IsDraining() bool { + ctx.stateLock.Lock() + defer ctx.stateLock.Unlock() + + return ctx.drainRequested +} + func (ctx *Context) CancelTask() { ctx.cancelFunc() } diff --git a/core/pipeline/observer.go b/core/pipeline/observer.go index e937c027..c4188fc2 100644 --- a/core/pipeline/observer.go +++ b/core/pipeline/observer.go @@ -37,6 +37,12 @@ const ( OutcomeError Outcome = "error" OutcomeDropped Outcome = "dropped" OutcomeSkippedContract Outcome = "skipped-contract" + + // OutcomeTakeover marks a singleton lock takeover (a control-plane + // lifecycle event, not a chain step): this instance acquired a lock + // whose previous holder's lease expired. Not an error, but always + // retained by the flight recorder (any non-ok outcome is). + OutcomeTakeover Outcome = "lock-takeover" ) // ExecutionObserver receives StepEvents as chains execute. Observers must diff --git a/modules/pipeline/control_test.go b/modules/pipeline/control_test.go new file mode 100644 index 00000000..e9380da1 --- /dev/null +++ b/modules/pipeline/control_test.go @@ -0,0 +1,264 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/locker" + "infini.sh/framework/core/pipeline" +) + +// ── generation guard (#120) ───────────────────────────────────────────── + +func TestRequestedGeneration(t *testing.T) { + req := httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=2", nil) + if n, ok := requestedGeneration(req); !ok || n != 2 { + t.Fatalf("generation=2 parsed as (%d, %v)", n, ok) + } + req = httptest.NewRequest("POST", "/pipeline/task/x/_pause", nil) + if _, ok := requestedGeneration(req); ok { + t.Fatal("absent param must be unguarded") + } + req = httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=abc", nil) + if _, ok := requestedGeneration(req); ok { + t.Fatal("malformed param must be unguarded") + } + req = httptest.NewRequest("POST", "/pipeline/task/x/_pause?generation=0", nil) + if _, ok := requestedGeneration(req); ok { + t.Fatal("non-positive param must be unguarded") + } +} + +// ── drain (#121) ──────────────────────────────────────────────────────── + +func drainableCfg(id, procName string) pipeline.PipelineConfigV2 { + cfg := hotCfg(id, procName, "x") + cfg.KeepRunning = true + cfg.RetryDelayInMs = 1 + return cfg +} + +func TestDrainTask_ParksStoppedAndResumes(t *testing.T) { + proc := hotReloadTestProcessor() + m := &PipeModule{} + defer func() { + m.stopAndWaitForRelease([]string{"drain-1"}, time.Minute) + m.deleteTask("drain-1") + }() + + if err := m.createPipelineTask(drainableCfg("drain-1", proc), false); err != nil { + t.Fatalf("create: %v", err) + } + + if !m.drainTask("drain-1") { + t.Fatal("drainTask reported task missing") + } + state := m.waitTaskSettled("drain-1", 5*time.Second) + if state == pipeline.STARTED || state == pipeline.STARTING || state == pipeline.STOPPING { + t.Fatalf("task not settled after drain, state=%v", state) + } + // drained task stays registered (resumable), unlike delete + if _, ok := m.contexts.Load("drain-1"); !ok { + t.Fatal("drained task was removed from the registry") + } + + // restart clears the drain flag so the task runs again + if !m.startTask("drain-1") { + t.Fatal("startTask on drained task failed") + } + deadline := time.Now().Add(3 * time.Second) + for m.taskRunningState("drain-1") == pipeline.STOPPED { + if time.Now().After(deadline) { + t.Fatal("task did not resume after start") + } + time.Sleep(20 * time.Millisecond) + } + v, _ := m.contexts.Load("drain-1") + if ctx, ok := v.(*pipeline.Context); !ok || ctx.IsDraining() { + t.Fatal("restart did not clear the drain flag") + } +} + +func TestDrainAllTasks_SettlesEveryTask(t *testing.T) { + proc := hotReloadTestProcessor() + m := &PipeModule{} + for _, id := range []string{"drainall-1", "drainall-2"} { + if err := m.createPipelineTask(drainableCfg(id, proc), false); err != nil { + t.Fatalf("create %s: %v", id, err) + } + defer func(id string) { + m.stopAndWaitForRelease([]string{id}, time.Minute) + m.deleteTask(id) + }(id) + } + + states := m.drainAllTasks(5 * time.Second) + if len(states) != 2 { + t.Fatalf("drainAllTasks saw %d tasks, want 2: %v", len(states), states) + } + for id, s := range states { + if s == string(pipeline.STARTED) || s == string(pipeline.STARTING) || s == string(pipeline.STOPPING) { + t.Fatalf("task %s not settled: %v", id, s) + } + } +} + +// ── cordon (#121) ─────────────────────────────────────────────────────── + +func TestCordon_BlocksTaskStarts(t *testing.T) { + proc := hotReloadTestProcessor() + m := &PipeModule{} + defer func() { + m.stopAndWaitForRelease([]string{"cordon-1"}, time.Minute) + m.deleteTask("cordon-1") + }() + + m.cordoned.Store(true) + if err := m.createPipelineTask(drainableCfg("cordon-1", proc), false); err != nil { + t.Fatalf("create under cordon: %v", err) + } + // auto_start task on a cordoned instance parks STOPPED instead of running + deadline := time.Now().Add(3 * time.Second) + for { + s := m.taskRunningState("cordon-1") + if s == pipeline.STOPPED { + break + } + if time.Now().After(deadline) { + t.Fatalf("cordoned task did not park, state=%v", s) + } + time.Sleep(20 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + if s := m.taskRunningState("cordon-1"); s != pipeline.STOPPED { + t.Fatalf("cordoned task left the parked state: %v", s) + } + + m.cordoned.Store(false) + if !m.startTask("cordon-1") { + t.Fatal("startTask after uncordon failed") + } + deadline = time.Now().Add(3 * time.Second) + for m.taskRunningState("cordon-1") == pipeline.STOPPED { + if time.Now().After(deadline) { + t.Fatal("task did not resume after uncordon") + } + time.Sleep(20 * time.Millisecond) + } +} + +// ── lock takeover (#122) ──────────────────────────────────────────────── + +// controlMemKV is the minimal in-memory KVStore backing the locker probes. +type controlMemKV struct { + mu sync.Mutex + kv map[string][]byte +} + +func (m *controlMemKV) Open() error { return nil } +func (m *controlMemKV) Close() error { return nil } + +func (m *controlMemKV) key(bucket string, key []byte) string { + return bucket + ":" + string(key) +} + +func (m *controlMemKV) GetValue(bucket string, key []byte) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.kv[m.key(bucket, key)] + if !ok { + return nil, nil + } + return v, nil +} + +func (m *controlMemKV) GetCompressedValue(bucket string, key []byte) ([]byte, error) { + return m.GetValue(bucket, key) +} + +func (m *controlMemKV) AddValue(bucket string, key []byte, value []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + m.kv[m.key(bucket, key)] = value + return nil +} + +func (m *controlMemKV) AddValueCompress(bucket string, key []byte, value []byte) error { + return m.AddValue(bucket, key, value) +} + +func (m *controlMemKV) ExistsKey(bucket string, key []byte) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.kv[m.key(bucket, key)] + return ok, nil +} + +func (m *controlMemKV) DeleteKey(bucket string, key []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.kv, m.key(bucket, key)) + return nil +} + +var controlKVOnce sync.Once + +// placeLockRow writes a raw locker row ("clientID/") under the +// locker's private kv bucket; the bucket literal mirrors core/locker's +// parentBucket. +func placeLockRow(t *testing.T, name, clientID string, ts time.Time) { + v := clientID + "/" + strconv.FormatInt(ts.Unix(), 10) + if err := kv.AddValue("dis_locker", locker.GetKey(pipelineSingleton, name), []byte(v)); err != nil { + t.Fatalf("place lock row: %v", err) + } +} + +func TestProbeLockTakeover(t *testing.T) { + controlKVOnce.Do(func() { + kv.Register("control-test-mem", &controlMemKV{kv: map[string][]byte{}}) + }) + + const name = "takeover-probe-1" + ttl := 60 * time.Second + expired := time.Now().Add(-2 * ttl) + + if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" { + t.Fatalf("no lock placed, probe returned %q", got) + } + + placeLockRow(t, name, "node-A", expired) + if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "node-A" { + t.Fatalf("expired foreign lock: probe returned %q, want node-A", got) + } + + placeLockRow(t, name, "node-B", time.Now()) + if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" { + t.Fatalf("fresh foreign lock must not report takeover, got %q", got) + } + + placeLockRow(t, name, global.Env().SystemConfig.NodeConfig.ID, expired) + if got := probeLockTakeover(pipelineSingleton, name, ttl); got != "" { + t.Fatalf("self-owned lock must not report takeover, got %q", got) + } +} + +func TestRecordPipelineEvent_TakeoverLandsInFlight(t *testing.T) { + installFlight() + recordPipelineEvent("evt-takeover-1", pipeline.OutcomeTakeover, "singleton lock taken over from instance node-A") + evs := flightQuery("evt-takeover-1", "", true, 10) + if len(evs) != 1 { + t.Fatalf("takeover event not retained: %+v", evs) + } + if evs[0].Outcome != pipeline.OutcomeTakeover { + t.Fatalf("outcome = %v, want %v", evs[0].Outcome, pipeline.OutcomeTakeover) + } +} diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go index 86bc9e15..dc312a39 100644 --- a/modules/pipeline/flight.go +++ b/modules/pipeline/flight.go @@ -88,3 +88,19 @@ func installFlight() { pipeline.RegisterObserver(newFlightRecorder()) globalFlight.Store(newFlightRecorder()) } + +// recordPipelineEvent files a control-plane lifecycle event (singleton +// lock takeover...) into the flight ring — the alert surface operators +// watch. No-op when flight is not installed; non-ok outcomes are always +// retained by the recorder. +func recordPipelineEvent(pipelineName string, outcome pipeline.Outcome, msg string) { + f := globalFlight.Load() + if f == nil { + return + } + f.(*flightRecorder).OnStep(&pipeline.StepEvent{ + Pipeline: pipelineName, + Outcome: outcome, + Err: msg, + }) +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 5df6ac80..b58790d6 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -68,6 +68,10 @@ type PipeModule struct { // startup version); each successful hot reload increments it and the // runtime report exposes it so controllers can see a reload landed generations sync.Map // taskID -> int + // cordoned marks the instance under maintenance: running tasks drain + // to completion, but no task starts (auto_start/_start/_resume, and + // tasks recreated by hot reload park STOPPED) until uncordoned + cordoned atomic.Bool } // recordFailure stores a pipeline compile/load failure for the runtime @@ -216,6 +220,20 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopPipelineTaskHandler) api.HandleUIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + // graceful stop (#121): finish the current batch, commit, park + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_drain", module.drainPipelineTaskHandler) + api.HandleUIMethod(api.POST, "/pipeline/task/:id/_drain", module.drainPipelineTaskHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + + // instance maintenance (#121): drain everything / cordon against new starts + api.HandleAPIMethod(api.POST, "/pipeline/_drain", module.drainInstanceHandler) + api.HandleUIMethod(api.POST, "/pipeline/_drain", module.drainInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.POST, "/pipeline/_cordon", module.cordonInstanceHandler) + api.HandleUIMethod(api.POST, "/pipeline/_cordon", module.cordonInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.POST, "/pipeline/_uncordon", module.uncordonInstanceHandler) + api.HandleUIMethod(api.POST, "/pipeline/_uncordon", module.uncordonInstanceHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + api.HandleAPIMethod(api.GET, "/pipeline/_cordon", module.getCordonHandler) + api.HandleUIMethod(api.GET, "/pipeline/_cordon", module.getCordonHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // processor discovery: names + config schemas for pipeline designers api.HandleAPIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler) api.HandleUIMethod(api.GET, "/pipeline/processors", module.getProcessorsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) @@ -710,6 +728,107 @@ func (module *PipeModule) reloadPipelineTaskAsync(pipelineID string, newCfg *pip go module.hotReloadPipeline(pipelineID, newCfg) } +// drainTask marks the task for a graceful stop (#121): the consumer +// finishes the current batch and commits its offset, then the pipeline +// loop parks STOPPED (resumable via _start). Returns false when the task +// does not exist locally. +func (module *PipeModule) drainTask(taskID string) bool { + v, ok := module.contexts.Load(taskID) + if !ok { + return false + } + ctx, ok := v.(*pipeline.Context) + if !ok { + return false + } + ctx.Drain() + ctx.Exit() // no keep_running restarts after the current pass + return true +} + +// taskRunningState reports the task's current state ("" when absent). +func (module *PipeModule) taskRunningState(taskID string) pipeline.RunningState { + v, ok := module.contexts.Load(taskID) + if !ok { + return "" + } + ctx, ok := v.(*pipeline.Context) + if !ok { + return "" + } + return ctx.GetRunningState() +} + +// waitTaskSettled polls until the task leaves the running states or the +// timeout elapses; returns the last observed state. +func (module *PipeModule) waitTaskSettled(taskID string, timeout time.Duration) pipeline.RunningState { + deadline := time.Now().Add(timeout) + for { + state := module.taskRunningState(taskID) + if state != pipeline.STARTED && state != pipeline.STARTING && state != pipeline.STOPPING { + return state + } + if time.Now().After(deadline) { + return state + } + time.Sleep(100 * time.Millisecond) + } +} + +// drainAllTasks gracefully stops every running task on this instance +// (maintenance path — pair with cordon so no new work starts while +// draining). All drains are marked first, then one bounded wait; the +// returned map carries each task's settled state. +func (module *PipeModule) drainAllTasks(timeout time.Duration) map[string]string { + module.contexts.Range(func(k, v any) bool { + if id, ok := k.(string); ok { + module.drainTask(id) + } + return true + }) + deadline := time.Now().Add(timeout) + out := map[string]string{} + for { + running := 0 + module.contexts.Range(func(k, v any) bool { + id, ok := k.(string) + if !ok { + return true + } + state := module.taskRunningState(id) + out[id] = string(state) + if state == pipeline.STARTED || state == pipeline.STARTING || state == pipeline.STOPPING { + running++ + } + return true + }) + if running == 0 || time.Now().After(deadline) { + return out + } + time.Sleep(100 * time.Millisecond) + } +} + +// probeLockTakeover reports the previous owner when the singleton lock +// for name sits EXPIRED under another instance — the Hold that follows +// takes it over. kv-less environments (unit tests, lightweight agents) +// report "" instead of panicking. +func probeLockTakeover(bucket, name string, ttl time.Duration) string { + defer func() { _ = recover() }() + ok, info, err := locker.GetAllocateInfo(bucket, name) + if err != nil || !ok || info == nil { + return "" + } + self := global.Env().SystemConfig.NodeConfig.ID + if info.ClientID == "" || info.ClientID == self { + return "" + } + if time.Since(info.Timestamp) > ttl { + return info.ClientID + } + return "" +} + // Helper function to run a pipeline, i.e., create a pipeline task. func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transient bool) error { if module.closed.Load() { @@ -833,17 +952,35 @@ func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transi switch state { case pipeline.STARTING: + // cordoned instance: the task registers but does not run — + // it parks STOPPED and resumes via _start after _uncordon + if module.cordoned.Load() { + ctx.Stopped() + ctx.Pause() + continue + } + //check if v.Singleton { if v.MaxRunningInMs <= 0 { v.MaxRunningInMs = 60000 } - ok, err := locker.Hold(pipelineSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, time.Duration(v.MaxRunningInMs)*time.Millisecond, true) + ttl := time.Duration(v.MaxRunningInMs) * time.Millisecond + // takeover detection (#122): the lock's lease expired + // under another instance — the Hold below acquires it, + // ending that instance's run; surface the event + takeoverFrom := probeLockTakeover(pipelineSingleton, v.ID, ttl) + ok, err := locker.Hold(pipelineSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, ttl, true) if !ok { log.Debugf("pipeline [%v] is already running somewhere, %v", cfg.Name, err) ctx.Finished() continue } + if takeoverFrom != "" { + msg := fmt.Sprintf("singleton lock taken over from instance %s (lease expired)", takeoverFrom) + log.Warnf("pipeline [%v]: %v", cfg.Name, msg) + recordPipelineEvent(cfg.Name, pipeline.OutcomeTakeover, msg) + } } // Pipeline needs to run diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index 9b5161fa..79f7cc93 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -214,9 +214,50 @@ func (module *PipeModule) flightPipelineTaskHandler(w http.ResponseWriter, req * module.WriteJSON(w, util.MapStr{"events": events}, http.StatusOK) } +// requestedGeneration parses the optional ?generation= control guard. +// Returns (0, false) when absent or malformed — legacy unguarded behavior. +func requestedGeneration(req *http.Request) (int, bool) { + g := req.URL.Query().Get("generation") + if g == "" { + return 0, false + } + n, err := strconv.Atoi(g) + if err != nil || n <= 0 { + return 0, false + } + return n, true +} + +// checkGenerationGuard enforces the fleet control contract (#120): an +// instruction stamped with a generation applies only when the local task +// runs exactly that config revision — an instance still on an older +// generation (hot reload not landed yet) must not act on instructions +// issued for the new one, and vice versa. Writes a 409 carrying both +// generations on mismatch; absent/malformed param keeps today's +// unconditional behavior. +func (module *PipeModule) checkGenerationGuard(w http.ResponseWriter, req *http.Request, id string) bool { + n, ok := requestedGeneration(req) + if !ok { + return true + } + cur := module.generationOf(id) + if cur != n { + module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{ + "error": "generation mismatch, instruction skipped", + "current_generation": cur, + "instruction_generation": n, + }) + return false + } + return true +} + // pausePipelineTaskHandler — POST /pipeline/task/:id/_pause func (module *PipeModule) pausePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") + if !module.checkGenerationGuard(w, req, id) { + return + } if !module.pauseTask(id) { module.WriteOpRecordNotFoundJSON(w, id) return @@ -227,6 +268,13 @@ func (module *PipeModule) pausePipelineTaskHandler(w http.ResponseWriter, req *h // resumePipelineTaskHandler — POST /pipeline/task/:id/_resume func (module *PipeModule) resumePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") + if !module.checkGenerationGuard(w, req, id) { + return + } + if module.cordoned.Load() { + module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{"error": "instance is cordoned"}) + return + } if !module.resumeTask(id) { module.WriteOpRecordNotFoundJSON(w, id) return @@ -322,6 +370,13 @@ func (module *PipeModule) injectPipelineTaskHandler(w http.ResponseWriter, req * func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") + if !module.checkGenerationGuard(w, req, id) { + return + } + if module.cordoned.Load() { + module.WriteAckJSON(w, false, http.StatusConflict, util.MapStr{"error": "instance is cordoned"}) + return + } exists := module.startTask(id) if exists { module.WriteAckOKJSON(w) @@ -335,6 +390,9 @@ func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *h // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_stop func (module *PipeModule) stopPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") + if !module.checkGenerationGuard(w, req, id) { + return + } exists := module.stopTask(id) if exists { module.WriteAckOKJSON(w) @@ -344,3 +402,65 @@ func (module *PipeModule) stopPipelineTaskHandler(w http.ResponseWriter, req *ht }) } } + +// drainPipelineTaskHandler — POST /pipeline/task/:id/_drain +// Graceful stop: the consumer finishes the current batch and commits its +// offset, then the task parks STOPPED (resumable via _start). Honors the +// ?generation= guard; ?timeout_seconds= bounds the settle wait (default +// 30, max 600). +func (module *PipeModule) drainPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + if !module.checkGenerationGuard(w, req, id) { + return + } + if !module.drainTask(id) { + module.WriteOpRecordNotFoundJSON(w, id) + return + } + timeout := 30 * time.Second + if v := req.URL.Query().Get("timeout_seconds"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 600 { + timeout = time.Duration(n) * time.Second + } + } + state := module.waitTaskSettled(id, timeout) + settled := state != pipeline.STARTED && state != pipeline.STARTING && state != pipeline.STOPPING + module.WriteJSON(w, util.MapStr{"drained": settled, "id": id, "state": string(state)}, http.StatusOK) +} + +// drainInstanceHandler — POST /pipeline/_drain +// Graceful-stop every running task on this instance (maintenance path; +// pair with _cordon so no new work starts while draining). Optional +// ?timeout_seconds= bounds the shared settle wait (default 60). +func (module *PipeModule) drainInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + timeout := 60 * time.Second + if v := req.URL.Query().Get("timeout_seconds"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 3600 { + timeout = time.Duration(n) * time.Second + } + } + states := module.drainAllTasks(timeout) + module.WriteJSON(w, util.MapStr{"drained": states}, http.StatusOK) +} + +// cordonInstanceHandler — POST /pipeline/_cordon +// Marks the instance cordoned: running tasks drain to completion, but no +// task starts (auto_start, _start, _resume; hot-reload recreations park +// STOPPED) until _uncordon. +func (module *PipeModule) cordonInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + module.cordoned.Store(true) + log.Info("pipeline instance cordoned: task starts are rejected until _uncordon") + module.WriteJSON(w, util.MapStr{"cordoned": true}, http.StatusOK) +} + +// uncordonInstanceHandler — POST /pipeline/_uncordon +func (module *PipeModule) uncordonInstanceHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + module.cordoned.Store(false) + log.Info("pipeline instance uncordoned") + module.WriteJSON(w, util.MapStr{"cordoned": false}, http.StatusOK) +} + +// getCordonHandler — GET /pipeline/_cordon +func (module *PipeModule) getCordonHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + module.WriteJSON(w, util.MapStr{"cordoned": module.cordoned.Load()}, http.StatusOK) +} diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 9be8f4a0..af55c224 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -274,7 +274,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { var lastDispatch = time.Now() for { - if c.IsCanceled() { + if c.IsCanceled() || c.IsDraining() { return } @@ -292,7 +292,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { //log.Errorf("filter queue by:%v, num of queues:%v", processor.config.Selector.ToString(), len(cfgs)) for _, v := range cfgs { - if c.IsCanceled() { + if c.IsCanceled() || c.IsDraining() { return } @@ -623,11 +623,14 @@ READ_DOCS: //TODO //consumerInstance.ResetOffset(queue.ConvertOffset(offset)) - EOF := false - for { - if global.ShuttingDown() || ctx.IsCanceled() || ctx.IsFailed() || parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled()) { - goto CLEAN_BUFFER - } + EOF := false + for { + // drain (graceful stop): break to CLEAN_BUFFER so the current + // offset commits, then return instead of fetching more — the + // batch boundary is the safe exit point for maintenance stops + if global.ShuttingDown() || ctx.IsCanceled() || ctx.IsFailed() || ctx.IsDraining() || parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled() || parentContext.IsDraining()) { + goto CLEAN_BUFFER + } if len(processor.config.WaitingAfter) > 0 { for _, v := range processor.config.WaitingAfter { @@ -805,7 +808,7 @@ CLEAN_BUFFER: } log.Tracef("slice_worker, goto READ_DOCS, return on queue:[%v], slice_id:%v", qConfig.Name, sliceID) - if !ctx.IsCanceled() && !(parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled())) { + if !ctx.IsCanceled() && !ctx.IsDraining() && !(parentContext != nil && (parentContext.IsFailed() || parentContext.IsCanceled() || parentContext.IsDraining())) { goto READ_DOCS } } From 265b867172f4f4ccb6b3dac118f508daa893f4d4 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 02:16:14 +0800 Subject: [PATCH 55/65] feat(pipeline): stage stats and step events carry instance_id (#127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-instance deployments need per-instance attribution before cluster views can sum rows: StepEvent gains InstanceID stamped at the emitStep choke point (control-plane events via recordPipelineEvent stamp it too), StageStats rows self-describe their emitting node, and the persisted PipelineStageStat rows map instance_id as a keyword so cross-instance queries group on it — generalizing the field logpilot's InstanceEPS already carried. Additive JSON; old consumers ignore the new field. --- core/pipeline/observer.go | 31 ++++++++++++------- modules/pipeline/flight.go | 8 +++-- modules/pipeline/instance_id_test.go | 46 ++++++++++++++++++++++++++++ modules/pipeline/metrics.go | 16 +++++++--- modules/pipeline/metrics_persist.go | 6 +++- 5 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 modules/pipeline/instance_id_test.go diff --git a/core/pipeline/observer.go b/core/pipeline/observer.go index c4188fc2..59988fd4 100644 --- a/core/pipeline/observer.go +++ b/core/pipeline/observer.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "infini.sh/framework/core/global" "infini.sh/framework/core/util" ) @@ -17,16 +18,21 @@ import ( // metrics aggregate it (counters/latency histograms), and the flight // recorder keeps a sampled ring of it (errors always kept). type StepEvent struct { - TraceID string // record-level trace id ("" when absent) - Pipeline string // pipeline/task name - Step int // 0-based index within the executing chain - Processor string // processor Name() - Scope ScopeInfo // batch | record + namespace ("" = unset) - Before util.MapStr // optional pre-state snapshot (nil unless captured) - After util.MapStr // optional post-state snapshot (nil unless captured) - Duration time.Duration - Outcome Outcome - Err string + TraceID string // record-level trace id ("" when absent) + // InstanceID identifies the executing node. Multi-instance + // deployments attribute events per instance (cluster views aggregate + // them); emitStep stamps it, so it is "" only in unit tests that + // build events by hand. + InstanceID string + Pipeline string // pipeline/task name + Step int // 0-based index within the executing chain + Processor string // processor Name() + Scope ScopeInfo // batch | record + namespace ("" = unset) + Before util.MapStr // optional pre-state snapshot (nil unless captured) + After util.MapStr // optional post-state snapshot (nil unless captured) + Duration time.Duration + Outcome Outcome + Err string } // Outcome classifies how a step ended. @@ -124,8 +130,11 @@ func hasObservers() bool { // emitStep fans a StepEvent out to the installed observers. Caller fills // the event; emitStep only stamps what it can derive (TraceID from the -// current record when in record scope). +// current record when in record scope, InstanceID from the local node). func emitStep(e *StepEvent, ctx *Context) { + if e.InstanceID == "" { + e.InstanceID = global.Env().SystemConfig.NodeConfig.ID + } if ctx != nil { if rec, ok := CurrentRecord(ctx); ok { e.TraceID = TraceID(rec) diff --git a/modules/pipeline/flight.go b/modules/pipeline/flight.go index dc312a39..8f6698b8 100644 --- a/modules/pipeline/flight.go +++ b/modules/pipeline/flight.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" + "infini.sh/framework/core/global" "infini.sh/framework/core/pipeline" ) @@ -99,8 +100,9 @@ func recordPipelineEvent(pipelineName string, outcome pipeline.Outcome, msg stri return } f.(*flightRecorder).OnStep(&pipeline.StepEvent{ - Pipeline: pipelineName, - Outcome: outcome, - Err: msg, + InstanceID: global.Env().SystemConfig.NodeConfig.ID, + Pipeline: pipelineName, + Outcome: outcome, + Err: msg, }) } diff --git a/modules/pipeline/instance_id_test.go b/modules/pipeline/instance_id_test.go new file mode 100644 index 00000000..356745ac --- /dev/null +++ b/modules/pipeline/instance_id_test.go @@ -0,0 +1,46 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "testing" + + "infini.sh/framework/core/global" + "infini.sh/framework/core/pipeline" +) + +// TestStageStatsCarryInstanceID — every stage row self-describes its +// emitting node (#127), so rows shipped to cluster views or persisted +// cross-instance can be grouped by origin. +func TestStageStatsCarryInstanceID(t *testing.T) { + installStats() + rec := globalStats.Load().(*statsRecorder) + rec.OnStep(&pipeline.StepEvent{Pipeline: "iid-1", Processor: "echo", Outcome: pipeline.OutcomeOK}) + + rows := stageStats("iid-1") + if len(rows) == 0 { + t.Fatal("no stage rows after feeding the recorder") + } + self := global.Env().SystemConfig.NodeConfig.ID + for _, r := range rows { + if r.InstanceID != self { + t.Fatalf("stage %q row instance_id = %q, want %q", r.Stage, r.InstanceID, self) + } + } +} + +// TestRecordPipelineEventStampsInstance — control-plane events (lock +// takeover) land in the flight ring attributed to this instance. +func TestRecordPipelineEventStampsInstance(t *testing.T) { + installFlight() + recordPipelineEvent("iid-evt-1", pipeline.OutcomeTakeover, "took from node-X") + evs := flightQuery("iid-evt-1", "", true, 10) + if len(evs) != 1 { + t.Fatalf("event not retained: %+v", evs) + } + if evs[0].InstanceID != global.Env().SystemConfig.NodeConfig.ID { + t.Fatalf("event instance_id = %q", evs[0].InstanceID) + } +} diff --git a/modules/pipeline/metrics.go b/modules/pipeline/metrics.go index df5a1a7c..15b916ac 100644 --- a/modules/pipeline/metrics.go +++ b/modules/pipeline/metrics.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "infini.sh/framework/core/global" "infini.sh/framework/core/lane" "infini.sh/framework/core/pipeline" ) @@ -17,6 +18,9 @@ import ( // processor (in-process stages) plus one row per lane carrier — the // runtime projection of the StepEvent stream. type StageStats struct { + // InstanceID identifies the emitting node: rows shipped to cluster + // views (or persisted cross-instance) self-describe their origin. + InstanceID string `json:"instance_id"` Stage string `json:"stage"` // processor name or lane name Carrier string `json:"carrier"` // "processor" | "lane:" Count int64 `json:"count"` // invocations @@ -67,13 +71,14 @@ func (s *statsRecorder) OnStep(e *pipeline.StepEvent) { // stageStats snapshots one pipeline's processor stages plus every lane // carrier (depth from the live registry). func stageStats(pipelineName string) []StageStats { + self := global.Env().SystemConfig.NodeConfig.ID out := []StageStats{} if rec := globalStats.Load(); rec != nil { s := rec.(*statsRecorder) s.mu.Lock() if m := s.per[pipelineName]; m != nil { for name, st := range m { - entry := StageStats{Stage: name, Carrier: "processor", Count: st.count, Errors: st.errors, LastOutcom: string(st.last)} + entry := StageStats{InstanceID: self, Stage: name, Carrier: "processor", Count: st.count, Errors: st.errors, LastOutcom: string(st.last)} if st.count > 0 { entry.AvgLatency = st.duration / st.count } @@ -84,10 +89,11 @@ func stageStats(pipelineName string) []StageStats { } for _, l := range lane.List() { out = append(out, StageStats{ - Stage: l.Name, - Carrier: "lane:" + l.Name, - Count: l.Emitted, - Errors: l.Drops + l.Spills, // degraded deliveries surface as stage errors + InstanceID: self, + Stage: l.Name, + Carrier: "lane:" + l.Name, + Count: l.Emitted, + Errors: l.Drops + l.Spills, // degraded deliveries surface as stage errors }) } return out diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index 0107187f..9a37e3b5 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -19,6 +19,10 @@ import ( type PipelineStageStat struct { orm.ORMObjectBase `json:",inline" elastic_mapping:"-"` + // InstanceID identifies the emitting node — cross-instance queries + // group on it; the #130 acceptance (cluster aggregate = sum of the + // per-instance rows) keys off this column. + InstanceID string `json:"instance_id" elastic_mapping:"instance_id:{type:keyword}"` Pipeline string `json:"pipeline" elastic_mapping:"pipeline:{type:keyword}"` Stage string `json:"stage" elastic_mapping:"stage:{type:keyword}"` Carrier string `json:"carrier" elastic_mapping:"carrier:{type:keyword}"` @@ -81,7 +85,7 @@ func (module *PipeModule) persistStageSnapshot() { continue } row := PipelineStageStat{ - Pipeline: name, Stage: st.Stage, Carrier: st.Carrier, + InstanceID: st.InstanceID, Pipeline: name, Stage: st.Stage, Carrier: st.Carrier, Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency, Ts: now, } From 68cb052acf9a870475131699fff783c2b864e302 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 02:18:11 +0800 Subject: [PATCH 56/65] =?UTF-8?q?feat(pipeline):=20GET=20/pipeline/cluster?= =?UTF-8?q?/stages=20=E2=80=94=20cluster-wide=20stage=20totals=20(#125/#13?= =?UTF-8?q?0=20groundwork)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregates the persisted PipelineStageStat snapshots: each instance's LATEST tick per stage is kept (older ticks superseded, never summed), counters sum across instances, latency becomes the invocation-weighted mean, and PerInstance detail ships alongside the totals so the aggregate stays auditable — cluster total must equal the sum of the parts, which is exactly the #130 acceptance. Optional ?pipeline= filter; input order is irrelevant (rows sorted newest-first internally). --- modules/pipeline/cluster_stages_test.go | 75 ++++++++++++++++ modules/pipeline/metrics_persist.go | 108 ++++++++++++++++++++++++ modules/pipeline/module.go | 4 + 3 files changed, 187 insertions(+) create mode 100644 modules/pipeline/cluster_stages_test.go diff --git a/modules/pipeline/cluster_stages_test.go b/modules/pipeline/cluster_stages_test.go new file mode 100644 index 00000000..c61139c9 --- /dev/null +++ b/modules/pipeline/cluster_stages_test.go @@ -0,0 +1,75 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "testing" + "time" +) + +// statRow builds a persisted snapshot row shorthand. +func statRow(inst, pipeline, stage string, count, errors int64, avg int64, ts time.Time) PipelineStageStat { + return PipelineStageStat{ + InstanceID: inst, Pipeline: pipeline, Stage: stage, Carrier: "processor", + Count: count, Errors: errors, AvgLatency: avg, Ts: ts, + } +} + +// TestAggregateStageRows_ClusterSumMatchesParts covers the #130 +// acceptance math: the cluster aggregate equals the sum of each +// instance's LATEST tick (older ticks superseded), and latency is the +// invocation-weighted mean across instances. +func TestAggregateStageRows_ClusterSumMatchesParts(t *testing.T) { + t0 := time.Now() + t1 := t0.Add(10 * time.Second) // instance A's second tick supersedes the first + rows := []PipelineStageStat{ + // instance A: tick1 (stale), tick2 (latest) — only tick2 may count + statRow("inst-a", "p1", "echo", 100, 1, 1_000_000, t0), + statRow("inst-a", "p1", "echo", 300, 2, 2_000_000, t1), + // instance B: single tick + statRow("inst-b", "p1", "echo", 100, 0, 6_000_000, t0), + // a different stage stays its own row + statRow("inst-b", "p1", "grok", 50, 5, 3_000_000, t0), + } + // deliberately shuffle: the aggregator must not depend on input order + rows = []PipelineStageStat{rows[1], rows[3], rows[0], rows[2]} + + out := aggregateStageRows(rows) + if len(out) != 2 { + t.Fatalf("want 2 stage rows, got %d: %+v", len(out), out) + } + + var echo *ClusterStageStat + for i := range out { + if out[i].Stage == "echo" { + echo = &out[i] + } + } + if echo == nil { + t.Fatal("echo stage row missing") + } + // aggregate = latest tick of A + tick of B (stale A tick1 ignored) + if echo.Count != 400 || echo.Errors != 2 { + t.Fatalf("echo aggregate = %d/%d, want 400/2", echo.Count, echo.Errors) + } + // weighted latency: (2ms*300 + 6ms*100) / 400 = 3ms + if echo.AvgLatency != 3_000_000 { + t.Fatalf("weighted avg latency = %d, want 3000000", echo.AvgLatency) + } + // per-instance detail present and consistent with the aggregate + if len(echo.PerInstance) != 2 { + t.Fatalf("per-instance detail missing: %+v", echo.PerInstance) + } + if a := echo.PerInstance["inst-a"]; a.Count != 300 { + t.Fatalf("inst-a detail = %+v, want the latest tick (300)", a) + } + var sum int64 + for _, st := range echo.PerInstance { + sum += st.Count + } + if sum != echo.Count { + t.Fatalf("aggregate %d != sum of parts %d", echo.Count, sum) + } +} diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index 9a37e3b5..cc067840 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -5,9 +5,13 @@ package pipeline import ( + "net/http" + "sort" "time" log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" "infini.sh/framework/core/env" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" @@ -96,3 +100,107 @@ func (module *PipeModule) persistStageSnapshot() { } } } + +// ClusterStageStat is one stage aggregated across every instance that +// reported it: counters sum over each instance's LATEST snapshot tick, +// latency is the invocation-weighted mean, and PerInstance carries the +// detail rows so the cluster math stays auditable (aggregate must equal +// the sum of the parts). +type ClusterStageStat struct { + Pipeline string `json:"pipeline"` + Stage string `json:"stage"` + Carrier string `json:"carrier"` + Count int64 `json:"count"` + Errors int64 `json:"errors"` + AvgLatency int64 `json:"avg_latency_ns"` + PerInstance map[string]StageStats `json:"per_instance"` +} + +// aggregateStageRows folds persisted snapshot rows (cumulative counters +// per tick) into per-stage cluster totals. Rows are sorted newest-first +// internally; the first row of each (instance, stage) pair is that +// instance's latest tick — older ticks are superseded, not summed. +func aggregateStageRows(rows []PipelineStageStat) []ClusterStageStat { + sort.SliceStable(rows, func(i, j int) bool { return rows[i].Ts.After(rows[j].Ts) }) + + type stageKey struct{ pipeline, stage, carrier string } + latest := map[string]map[stageKey]StageStats{} // instance -> stage -> row + for _, r := range rows { + inst := r.InstanceID + if inst == "" { + inst = "?" // pre-#127 rows: attribute to a synthetic bucket + } + key := stageKey{r.Pipeline, r.Stage, r.Carrier} + m := latest[inst] + if m == nil { + m = map[stageKey]StageStats{} + latest[inst] = m + } + if _, seen := m[key]; seen { + continue + } + m[key] = StageStats{ + InstanceID: r.InstanceID, Stage: r.Stage, Carrier: r.Carrier, + Count: r.Count, Errors: r.Errors, AvgLatency: r.AvgLatency, + } + } + + totals := map[stageKey]*ClusterStageStat{} + for _, m := range latest { + for key, st := range m { + agg := totals[key] + if agg == nil { + agg = &ClusterStageStat{ + Pipeline: key.pipeline, Stage: key.stage, Carrier: key.carrier, + PerInstance: map[string]StageStats{}, + } + totals[key] = agg + } + agg.Count += st.Count + agg.Errors += st.Errors + agg.PerInstance[st.InstanceID] = st + } + } + out := make([]ClusterStageStat, 0, len(totals)) + for _, agg := range totals { + var totalInvocations int64 + for _, st := range agg.PerInstance { + totalInvocations += st.Count + } + if totalInvocations > 0 { + var weighted int64 + for _, st := range agg.PerInstance { + weighted += st.AvgLatency * st.Count + } + agg.AvgLatency = weighted / totalInvocations + } + out = append(out, *agg) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Pipeline != out[j].Pipeline { + return out[i].Pipeline < out[j].Pipeline + } + return out[i].Stage < out[j].Stage + }) + return out +} + +// clusterStagesHandler — GET /pipeline/cluster/stages[?pipeline=] +// Cluster-wide stage totals from the persisted snapshots: every +// instance's latest tick summed per stage, with the per-instance detail +// alongside (the aggregate must equal the sum of the parts). +func (h *PipeModule) clusterStagesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + orm.WithModel(octx, &PipelineStageStat{}) + q := orm.NewQuery().Size(4000) + if p := req.URL.Query().Get("pipeline"); p != "" { + q = q.Filter(orm.TermQuery("pipeline", p)) + } + res, err := orm.SearchV2(octx, q) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + rows, _, _ := elastic.DecodeHits[PipelineStageStat](res) + h.WriteJSON(w, util.MapStr{"stages": aggregateStageRows(rows)}, http.StatusOK) +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index b58790d6..1ebd94e4 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -265,6 +265,10 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler) api.HandleUIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // cluster-wide stage totals from the persisted snapshots (#127/#130) + api.HandleAPIMethod(api.GET, "/pipeline/cluster/stages", module.clusterStagesHandler) + api.HandleUIMethod(api.GET, "/pipeline/cluster/stages", module.clusterStagesHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // pipeline version history + rollback api.HandleAPIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler) api.HandleUIMethod(api.GET, "/pipelines/:id/versions", module.listPipelineVersionsHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) From dc2e72b0b611e33f5287ce4e64b40061e9a38494 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 05:27:48 +0800 Subject: [PATCH 57/65] =?UTF-8?q?feat(pipeline):=20fleet=20control=20plane?= =?UTF-8?q?=20=E2=80=94=20staged=20rollout,=20cluster=20broadcast,=20insta?= =?UTF-8?q?nce=20proxy=20(#125)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _rollout: the RolloutSpec model finally does something. The reconciler gates managed-config writes through rolloutPlan — canary wave first, the next batch only opens when every wave instance REPORTS running the staged content, completion when the waves cover the fleet. Readiness is honest: RuntimeSnapshot now emits the running task's content digest (same canonical six-field shape the deployment renderer produces), it rides _sync as config_digest, and the gate compares pushed vs reported — a STARTED-but-stale canary holds the rollout instead of green-lighting it. POST /pipeline/deployments/:id/_rollout {advance|rollback} overrides the machine: advance forces the next batch past readiness, rollback rewrites every target to the recorded pre-rollout content at once (a brake — the reconciler re-stages desired from canary; the response says so and points at pipeline-version rollback for permanence). Cluster broadcast: POST /pipeline/deployments/:id/_pause|_resume|_stop fans the command out to the deployment's targets THROUGH the reverse channel (managed agents are behind NAT; the agent-dialed websocket is the only path). Commands address the task by pipeline NAME and carry the caller's ?generation=N verbatim — instances on another config generation answer 409 (#120), so the aggregate shows exactly who refused and why. Instance ops: POST /pipeline/instances/:id/_drain|_cordon|_uncordon forward #121's instance-local primitives with query passthrough (timeout_seconds), plus GET /pipeline/instances/:id/proxy?path= as a read-only /pipeline/-scoped debug passthrough (allowlisted, traversal refused, instance status codes pass through verbatim). DeploymentStatus now mirrors rollout progress for the overview UI. Handler tests stub the reverse channel at a package seam; rollout tests cover the pure gate (canary/batch/hold/force/prune/complete), digest parity between render and instance-side reconstruction, and a full staged converge-hold-advance-complete walk over the sqlite harness. --- modules/configs/client/client.go | 2 + modules/configs/common/domain.go | 5 + modules/pipeline/deployment.go | 233 +++++++++++- modules/pipeline/deployment_targeting_test.go | 21 +- modules/pipeline/fleet_control.go | 177 +++++++++ modules/pipeline/fleet_control_test.go | 182 ++++++++++ modules/pipeline/module.go | 35 ++ modules/pipeline/rollout.go | 304 ++++++++++++++++ modules/pipeline/rollout_test.go | 339 ++++++++++++++++++ 9 files changed, 1278 insertions(+), 20 deletions(-) create mode 100644 modules/pipeline/fleet_control.go create mode 100644 modules/pipeline/fleet_control_test.go create mode 100644 modules/pipeline/rollout.go create mode 100644 modules/pipeline/rollout_test.go diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 327ef45b..42bfc0da 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -602,9 +602,11 @@ func runtimePipelineSnapshot() *common.RuntimeReport { stateStr, _ := m["state"].(string) source, _ := m["config_source"].(string) reason, _ := m["reason"].(string) + digest, _ := m["config_digest"].(string) report.Pipelines[name] = common.PipelineRuntimeState{ State: stateStr, ConfigSource: source, + ConfigDigest: digest, Reason: reason, } } diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 10ddafd5..5c859d8b 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -102,6 +102,11 @@ type PipelineRuntimeState struct { State string `json:"state"` // STARTED|STOPPED|FAILED|... ConfigSource string `json:"config_source,omitempty"` // managed config name ConfigVersion int64 `json:"config_version,omitempty"` // managed config version loaded + // ConfigDigest is the running task's content digest (the six fields a + // deployment renders). The control plane compares it against the digest + // it pushed to verify rollout readiness — state alone cannot prove the + // instance runs the NEW content (#125). + ConfigDigest string `json:"config_digest,omitempty"` // Reason explains a FAILED state (e.g. the config references a // processor missing from this instance's catalog) — surfaced in the // deployment status so operators see why without instance shell access diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 7187a6e5..583ebe13 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -40,6 +40,11 @@ type PipelineDeployment struct { // Version increments on every spec change (reconciler compares with // the last-pushed version to detect drift). Version int `json:"version" elastic_mapping:"version:{type:integer}"` + + // State is the reconciler-owned progress of a staged rollout (#125): + // which content digest is being staged, who already carries it and + // what a rollback would fall back to. Managed by rollout.go. + State RolloutState `json:"rollout_state,omitempty" elastic_mapping:"rollout_state:{type:object}"` } // PlacementSpec declares WHERE and HOW the pipeline runs. @@ -75,6 +80,31 @@ type DeploymentStatus struct { Instances []InstanceReport `json:"instances"` DesiredCount int `json:"desired_count"` ReadyCount int `json:"ready_count"` + // Rollout mirrors the staged-delivery progress when one is (or was) + // active — the rollout progress block of the cluster overview. + Rollout *RolloutState `json:"rollout,omitempty"` +} + +// runtimePipelineReport is one pipeline entry of an instance's +// RuntimePipelines JSON (what _sync upserts on every heartbeat). +type runtimePipelineReport struct { + State string `json:"state"` + Reason string `json:"reason"` + ConfigDigest string `json:"config_digest"` +} + +// runtimePipelineStates parses the instance's reported pipeline states; +// absent/invalid JSON yields an empty map (unreported, blocks readiness). +func runtimePipelineStates(raw string) map[string]runtimePipelineReport { + out := map[string]runtimePipelineReport{} + if raw == "" { + return out + } + var parsed map[string]runtimePipelineReport + if json.Unmarshal([]byte(raw), &parsed) != nil { + return out + } + return parsed } type InstanceReport struct { @@ -275,7 +305,73 @@ func (module *PipeModule) reconcilePipeline(octx *orm.Context, pipelineID string } } + // staged rollout gate (#125): a rollout-bearing deployment stages the + // content — only the current wave (canary first, then batches once + // the wave reports running it, see rollout.go) may receive it. + var gateAllowed map[string]bool + if owner := rolloutGateOwner(deps, desired); owner != nil { + var sampleSpec renderSpec + for _, spec := range desired { + sampleSpec = spec + break + } + content := string(renderDeployedPipeline(&cfg, sampleSpec)) + digest := deployedContentDigest(deployedPipelineMap(&cfg, sampleSpec)) + // the fallback a rollback would restore: what rows that do not + // carry the new content still hold ("" when none exist yet) + prevContent := "" + for _, row := range existingRows { + if row.Content != content { + prevContent = row.Content + break + } + } + byID := map[string]model.Instance{} + for i := range instances { + byID[instances[i].ID] = instances[i] + } + singleton := owner.Placement.Strategy == "singleton" + ready := func(id string) bool { + inst, ok := byID[id] + if !ok { + return false + } + st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name] + if !reported || st.State == "FAILED" { + return false + } + if st.ConfigDigest != digest { + return false // still running the old content + } + if singleton { + return true // non-FAILED holders count: the lock picks the runner + } + return st.State == "STARTED" + } + targets := make([]string, 0, len(desired)) + for id := range desired { + targets = append(targets, id) + } + module.rolloutMu.Lock() + allowed, next, changed := rolloutPlan(owner.Placement.Rollout, owner.State, targets, digest, prevContent, ready, false) + if changed { + owner.State = next + dctx := orm.NewContext().DirectAccess() + orm.WithModel(dctx, &PipelineDeployment{}) + if serr := orm.Save(dctx, owner); serr != nil { + log.Warnf("reconciler: persist rollout state of %s failed: %v (gate held this pass)", owner.Name, serr) + } else { + log.Debugf("reconciler: rollout of %s now at %d/%d targets (active=%v)", owner.Name, len(next.Rolled), len(targets), next.Active) + } + } + module.rolloutMu.Unlock() + gateAllowed = allowed + } + for instID, spec := range desired { + if gateAllowed != nil && !gateAllowed[instID] { + continue // staged rollout: this instance's wave has not opened yet + } content := string(renderDeployedPipeline(&cfg, spec)) row, ok := existing[instID] // converged: the pushed content already matches — skip the @@ -335,6 +431,16 @@ func deploymentNames(deps []*PipelineDeployment) string { // the enabled→disabled transition while keeping the file for instant // re-enable. func renderDeployedPipeline(cfg *pipeline.PipelineConfigV2, spec renderSpec) []byte { + return util.MustToJSONBytes(map[string]interface{}{ + "pipeline": []interface{}{deployedPipelineMap(cfg, spec)}, + }) +} + +// deployedPipelineMap is the inner object a deployment renders — the six +// fields that define the deployed behavior. The same shape is digested on +// BOTH sides (control plane here, instance in RuntimeSnapshot), so a +// digest comparison proves which content an instance actually runs. +func deployedPipelineMap(cfg *pipeline.PipelineConfigV2, spec renderSpec) map[string]interface{} { pipelineEnabled := cfg.Enabled pipelineAutoStart := cfg.AutoStart if !spec.enabled { @@ -344,16 +450,119 @@ func renderDeployedPipeline(cfg *pipeline.PipelineConfigV2, spec renderSpec) []b pipelineEnabled = &f pipelineAutoStart = false } - return util.MustToJSONBytes(map[string]interface{}{ - "pipeline": []interface{}{map[string]interface{}{ - "name": cfg.Name, - "enabled": pipelineEnabled, - "auto_start": pipelineAutoStart, - "keep_running": cfg.KeepRunning, - "singleton": spec.singleton, - "processor": cfg.Processors, - }}, - }) + return map[string]interface{}{ + "name": cfg.Name, + "enabled": pipelineEnabled, + "auto_start": pipelineAutoStart, + "keep_running": cfg.KeepRunning, + "singleton": spec.singleton, + "processor": cfg.Processors, + } +} + +// deployedContentDigest digests a rendered pipeline map — the identity the +// staged rollout compares instance reports against. +func deployedContentDigest(m map[string]interface{}) string { + return util.MD5digest(string(util.MustToJSONBytes(m))) +} + +// deployedContentDigestOf digests FULL rendered config bytes (the +// {"pipeline":[{...}]} wrapper a managed-config row carries) by +// re-canonicalizing the inner object — comparable with +// deployedContentDigest. +func deployedContentDigestOf(rendered []byte) string { + var wrapper struct { + Pipeline []map[string]interface{} `json:"pipeline"` + } + if err := util.FromJSONBytes(rendered, &wrapper); err != nil || len(wrapper.Pipeline) == 0 { + return "" + } + return deployedContentDigest(wrapper.Pipeline[0]) +} + +// loadRegisteredInstances lists the instance universe selectors and +// explicit targeting resolve against. +func (h *PipeModule) loadRegisteredInstances(octx *orm.Context) []model.Instance { + orm.WithModel(octx, &model.Instance{}) + res, err := orm.SearchV2(octx, orm.NewQuery().Size(500)) + if err != nil { + return nil + } + instances, _, _ := elastic.DecodeHits[model.Instance](res) + return instances +} + +// deploymentTargetIDs resolves a deployment's scope to concrete instance +// IDs (broadcast = every registered instance). +func deploymentTargetIDs(dep *PipelineDeployment, instances []model.Instance) []string { + targets, broadcast := resolveTargets(dep, instances) + if broadcast { + ids := make([]string, 0, len(instances)) + for i := range instances { + ids = append(ids, instances[i].ID) + } + return ids + } + ids := make([]string, 0, len(targets)) + for id := range targets { + ids = append(ids, id) + } + return ids +} + +// rollbackDeploymentContent rewrites EVERY target's managed-config row to +// the deployment's recorded fallback content at once — a rollback is +// urgent, no staging (#125 _rollout action=rollback). +func (h *PipeModule) rollbackDeploymentContent(octx *orm.Context, dep *PipelineDeployment) error { + instances := h.loadRegisteredInstances(octx) + targets, broadcast := resolveTargets(dep, instances) + if broadcast { + targets = map[string]bool{} + for i := range instances { + targets[instances[i].ID] = true + } + } + if len(targets) == 0 { + return fmt.Errorf("deployment %s has no targets in scope right now", dep.Name) + } + + configName := fmt.Sprintf("pipeline-deploy-%s.yml", dep.PipelineID) + orm.WithModel(octx, &configs_server.ManagedConfig{}) + rowsRes, rerr := orm.SearchV2(octx, orm.NewQuery(). + Filter(orm.TermQuery("name", configName)).Size(500)) + if rerr != nil { + return fmt.Errorf("query managed config %s: %w", configName, rerr) + } + rows, _, _ := elastic.DecodeHits[configs_server.ManagedConfig](rowsRes) + existing := map[string]configs_server.ManagedConfig{} + maxVersion := int64(0) + for _, row := range rows { + existing[row.InstanceID] = row + if row.Version > maxVersion { + maxVersion = row.Version + } + } + + for instID := range targets { + row := existing[instID] + row.Name = configName + row.InstanceID = instID + row.Content = dep.State.RollbackContent + row.Version = maxVersion + 1 + row.Readonly = true + if row.ID == "" { + if instID == configs_server.AllInstancesID { + row.ID = fmt.Sprintf("deploy-%s", dep.PipelineID) + } else { + row.ID = fmt.Sprintf("deploy-%s-%s", dep.PipelineID, instID) + } + } + if err := orm.Save(octx, &row); err != nil { + return fmt.Errorf("rollback row %s→%s: %w", configName, instID, err) + } + maxVersion = row.Version + } + return nil } // getDeploymentStatus builds the REPORTED state from instance runtime @@ -439,6 +648,10 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat status.ReadyCount++ } } + if dep.State.Active || len(dep.State.Rolled) > 0 { + st := dep.State + status.Rollout = &st + } return status } diff --git a/modules/pipeline/deployment_targeting_test.go b/modules/pipeline/deployment_targeting_test.go index f5c4c4fe..5c23a6b7 100644 --- a/modules/pipeline/deployment_targeting_test.go +++ b/modules/pipeline/deployment_targeting_test.go @@ -26,14 +26,15 @@ import ( // is process-global, so every env gets unique instance/label/pipeline // IDs and tests stay isolated. type deployTestEnv struct { - t *testing.T - pid string - instA string - instB string - instC string - prod string // label value shared by instA+instC - stage string // label value of instB - edge string // extra label of instC + t *testing.T + pid string + pipeName string // the pipeline's config NAME (what instances run it under) + instA string + instB string + instC string + prod string // label value shared by instA+instC + stage string // label value of instB + edge string // extra label of instC } var ( @@ -64,7 +65,7 @@ func newDeployTestEnv(t *testing.T) *deployTestEnv { }) env := &deployTestEnv{ - t: t, pid: "pipe-" + sfx, + t: t, pid: "pipe-" + sfx, pipeName: "deploy-test-pipeline-" + sfx, instA: "gw-a-" + sfx, instB: "gw-b-" + sfx, instC: "gw-c-" + sfx, prod: "prod-" + sfx, stage: "staging-" + sfx, edge: "edge-" + sfx, } @@ -76,7 +77,7 @@ func newDeployTestEnv(t *testing.T) *deployTestEnv { orm.WithModel(octx, &pipeline.PipelineConfigV2{}) cfg := pipeline.PipelineConfigV2{} cfg.ID = env.pid - cfg.Name = "deploy-test-pipeline-" + sfx + cfg.Name = env.pipeName cfg.Processors = []map[string]interface{}{ {"validate_test_echo": map[string]interface{}{"message": "hi"}}, } diff --git a/modules/pipeline/fleet_control.go b/modules/pipeline/fleet_control.go new file mode 100644 index 00000000..6db5cf7c --- /dev/null +++ b/modules/pipeline/fleet_control.go @@ -0,0 +1,177 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" + configs_server "infini.sh/framework/modules/configs/server" +) + +// Fleet control (#125): transient commands never ride the config channel +// (SPEC §8.5) — the control plane forwards them to each target instance +// THROUGH the agent-dialed reverse channel. Managed instances sit behind +// NAT/firewalls and are not directly reachable; the reverse websocket is +// the only way in. + +// seam for tests: the reverse-channel client is a process-global that +// needs a live manager websocket — handler tests swap these out. +var ( + reverseProxyRequest = configs_server.ReverseProxyRequest + reverseIsConnected = configs_server.ReverseIsConnected +) + +// broadcastDeploymentActionHandler — POST /pipeline/deployments/:id/_pause|_resume|_stop +// Sends the control command to every instance in the deployment's scope, +// addressed by the pipeline's task name (instances key tasks by config +// name). The optional ?generation=N rides along verbatim: instances still +// on another config generation refuse the command with 409 (#120) — the +// split-brain guard the broadcast transport was missing. +func (h *PipeModule) broadcastDeploymentActionHandler(action string) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + + dep := PipelineDeployment{} + dep.ID = id + exists, err := orm.GetV2(octx, &dep) + if err != nil || !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + // resolve the pipeline NAME the instances run it under + cfg := pipeline.PipelineConfigV2{} + cfg.ID = dep.PipelineID + if found, gerr := orm.GetV2(octx, &cfg); gerr != nil || !found { + h.WriteError(w, fmt.Sprintf("pipeline %s not found", dep.PipelineID), http.StatusBadRequest) + return + } + + instances := h.loadRegisteredInstances(octx) + targets := deploymentTargetIDs(&dep, instances) + if len(targets) == 0 { + h.WriteJSON(w, util.MapStr{"total": 0, "delivered": 0, "results": map[string]interface{}{}}, http.StatusOK) + return + } + + gen := req.URL.Query().Get("generation") + results := map[string]util.MapStr{} + delivered := 0 + for _, instID := range targets { + results[instID] = h.controlOne(instID, cfg.Name, action, gen) + if ok, _ := results[instID]["ok"].(bool); ok { + delivered++ + } + } + h.WriteJSON(w, util.MapStr{ + "total": len(targets), + "delivered": delivered, + "results": results, + }, http.StatusOK) + } +} + +// controlOne issues one instance control command over the reverse channel +// and shapes the per-instance outcome for the broadcast aggregate. +func (h *PipeModule) controlOne(instID, taskName, action, generation string) util.MapStr { + if reverseIsConnected != nil && !reverseIsConnected(instID) { + return util.MapStr{"ok": false, "error": "reverse channel not connected (instance offline or pre-reverse-channel build)"} + } + path := fmt.Sprintf("/pipeline/task/%s/_%s", url.PathEscape(taskName), action) + if generation != "" { + path += "?generation=" + url.QueryEscape(generation) + } + res, err := reverseProxyRequest(instID, &util.Request{Method: http.MethodPost, Path: path}) + if err != nil { + return util.MapStr{"ok": false, "error": err.Error()} + } + out := util.MapStr{"ok": res.StatusCode >= 200 && res.StatusCode < 300, "status": res.StatusCode} + if len(res.Body) > 0 { + out["body"] = string(res.Body) + } + return out +} + +// instanceActionHandler — POST /pipeline/instances/:id/_drain|_cordon|_uncordon +// Forwards the instance-level maintenance operations (#121) to one managed +// instance through the reverse channel. Query strings pass through (drain +// accepts ?timeout_seconds=). +func (h *PipeModule) instanceActionHandler(action string) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + if !h.instanceExists(req, id) { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + path := "/pipeline/_" + action + if qs := req.URL.RawQuery; qs != "" { + path += "?" + qs + } + res, err := h.forwardToInstance(id, &util.Request{Method: http.MethodPost, Path: path}) + h.proxyReply(w, res, err) + } +} + +// instanceProxyHandler — GET /pipeline/instances/:id/proxy?path=/pipeline/... +// Read-only passthrough to an instance's own pipeline API (task detail, +// stages, flight — single-instance debugging without shell access). +// Methods stay GET: anything mutating has a dedicated action route above. +func (h *PipeModule) instanceProxyHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + target := req.URL.Query().Get("path") + // allowlist: pipeline-scope reads only — the proxy must not become an + // open relay onto the instance's management plane + if !strings.HasPrefix(target, "/pipeline/") || strings.Contains(target, "..") { + h.Error400(w, "path must target a /pipeline/ endpoint") + return + } + if !h.instanceExists(req, id) { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + res, err := h.forwardToInstance(id, &util.Request{Method: http.MethodGet, Path: target}) + h.proxyReply(w, res, err) +} + +// forwardToInstance runs one logical request over the reverse channel. +func (h *PipeModule) forwardToInstance(instID string, r *util.Request) (*util.Result, error) { + if reverseIsConnected != nil && !reverseIsConnected(instID) { + return nil, fmt.Errorf("instance %s is not connected through the reverse channel", instID) + } + return reverseProxyRequest(instID, r) +} + +// proxyReply maps a forwarded result onto the HTTP response, passing the +// instance's status code and body through verbatim (a 409 from a +// generation-guarded instance must stay a 409 for the caller). +func (h *PipeModule) proxyReply(w http.ResponseWriter, res *util.Result, err error) { + if err != nil { + h.WriteError(w, err.Error(), http.StatusBadGateway) + return + } + status := res.StatusCode + if status == 0 { + status = http.StatusBadGateway + } + w.Header().Set("Content-Type", util.ContentTypeJson) + w.WriteHeader(status) + _, _ = w.Write(res.Body) +} + +func (h *PipeModule) instanceExists(req *http.Request, id string) bool { + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + inst := model.Instance{} + inst.ID = id + found, err := orm.GetV2(octx, &inst) + return err == nil && found +} diff --git a/modules/pipeline/fleet_control_test.go b/modules/pipeline/fleet_control_test.go new file mode 100644 index 00000000..eb9ddba5 --- /dev/null +++ b/modules/pipeline/fleet_control_test.go @@ -0,0 +1,182 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" +) + +// The reverse channel is stubbed at the seam: tests capture what would be +// sent down the websocket and shape the response per instance. +type fakeReverse struct { + mu sync.Mutex + calls []fakeCall + statusOf map[string]int // instance → response status (default 200) +} + +type fakeCall struct { + Instance string + Method string + Path string +} + +func (f *fakeReverse) install(t *testing.T) { + origProxy, origConn := reverseProxyRequest, reverseIsConnected + reverseProxyRequest = func(peerID string, req *util.Request) (*util.Result, error) { + f.mu.Lock() + f.calls = append(f.calls, fakeCall{Instance: peerID, Method: req.Method, Path: req.Path}) + f.mu.Unlock() + status := http.StatusOK + if s, ok := f.statusOf[peerID]; ok { + status = s + } + return &util.Result{StatusCode: status, Body: []byte(`{"ok":true}`)}, nil + } + reverseIsConnected = func(string) bool { return true } + t.Cleanup(func() { reverseProxyRequest, reverseIsConnected = origProxy, origConn }) +} + +func (f *fakeReverse) recorded() []fakeCall { + f.mu.Lock() + defer f.mu.Unlock() + return append([]fakeCall(nil), f.calls...) +} + +func callHandler(t *testing.T, h httprouter.Handle, method, target string, params httprouter.Params) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, target, nil) + w := httptest.NewRecorder() + h(w, req, params) + return w +} + +func TestBroadcastDeploymentAction(t *testing.T) { + env := newDeployTestEnv(t) + dep := &PipelineDeployment{ + PipelineID: env.pid, + Enabled: true, + Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA, env.instB}}, + } + dep.ID = "d1" // shared upsert id (deployment_targeting_test.go convention) + dep.Name = "broadcast-test" + env.saveDeployment(dep) + + fr := &fakeReverse{statusOf: map[string]int{env.instB: http.StatusConflict}} // B is on another generation + fr.install(t) + + h := (&PipeModule{}).broadcastDeploymentActionHandler("pause") + w := callHandler(t, h, http.MethodPost, "/pipeline/deployments/"+dep.ID+"/_pause?generation=7", + httprouter.Params{{Key: "id", Value: dep.ID}}) + if w.Code != http.StatusOK { + t.Fatalf("aggregate must be 200 even with per-instance failures: %d %s", w.Code, w.Body.String()) + } + out := util.MapStr{} + if err := util.FromJSONBytes(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if out["total"].(float64) != 2 || out["delivered"].(float64) != 1 { + t.Fatalf("total=2 delivered=1 expected: %v", out) + } + + calls := fr.recorded() + if len(calls) != 2 { + t.Fatalf("both targets must receive the command: %+v", calls) + } + for _, c := range calls { + if c.Method != http.MethodPost { + t.Fatalf("control commands are POSTs: %+v", c) + } + if c.Path != "/pipeline/task/"+env.pipeName+"/_pause?generation=7" { + t.Fatalf("command must target the task by NAME with the generation guard: %+v", c) + } + } +} + +func TestBroadcastUnreachableInstance(t *testing.T) { + env := newDeployTestEnv(t) + dep := &PipelineDeployment{ + PipelineID: env.pid, + Enabled: true, + Placement: PlacementSpec{Strategy: "all", Instances: []string{env.instA}}, + } + dep.ID = "d1" // shared upsert id + dep.Name = "broadcast-offline" + env.saveDeployment(dep) + + origProxy, origConn := reverseProxyRequest, reverseIsConnected + reverseIsConnected = func(string) bool { return false } // nobody dialed in + t.Cleanup(func() { reverseProxyRequest, reverseIsConnected = origProxy, origConn }) + + h := (&PipeModule{}).broadcastDeploymentActionHandler("stop") + w := callHandler(t, h, http.MethodPost, "/pipeline/deployments/"+dep.ID+"/_stop", + httprouter.Params{{Key: "id", Value: dep.ID}}) + out := util.MapStr{} + _ = util.FromJSONBytes(w.Body.Bytes(), &out) + if out["delivered"].(float64) != 0 { + t.Fatalf("offline instance must not count as delivered: %v", out) + } +} + +func TestInstanceActionForwarding(t *testing.T) { + env := newDeployTestEnv(t) + fr := &fakeReverse{} + fr.install(t) + + h := (&PipeModule{}).instanceActionHandler("drain") + w := callHandler(t, h, http.MethodPost, "/pipeline/instances/"+env.instA+"/_drain?timeout_seconds=5", + httprouter.Params{{Key: "id", Value: env.instA}}) + if w.Code != http.StatusOK { + t.Fatalf("drain forward: %d %s", w.Code, w.Body.String()) + } + calls := fr.recorded() + if len(calls) != 1 || calls[0].Path != "/pipeline/_drain?timeout_seconds=5" { + t.Fatalf("query string must ride along: %+v", calls) + } + + // unknown instance → 404 before any reverse traffic + before := len(fr.recorded()) + w = callHandler(t, h, http.MethodPost, "/pipeline/instances/ghost/_drain", + httprouter.Params{{Key: "id", Value: "ghost"}}) + if w.Code != http.StatusNotFound || len(fr.recorded()) != before { + t.Fatalf("ghost instance must 404 without proxying: %d", w.Code) + } +} + +func TestInstanceProxyAllowlist(t *testing.T) { + env := newDeployTestEnv(t) + fr := &fakeReverse{} + fr.install(t) + h := (&PipeModule{}).instanceProxyHandler + + // off-scope path refused + w := callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/admin/", + httprouter.Params{{Key: "id", Value: env.instA}}) + if w.Code != http.StatusBadRequest { + t.Fatalf("non-pipeline path must be refused: %d", w.Code) + } + // traversal refused + w = callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/pipeline/../../x", + httprouter.Params{{Key: "id", Value: env.instA}}) + if w.Code != http.StatusBadRequest { + t.Fatalf("traversal must be refused: %d", w.Code) + } + // in-scope read passes through, status code preserved + fr.statusOf = map[string]int{env.instA: http.StatusConflict} + w = callHandler(t, h, http.MethodGet, "/pipeline/instances/"+env.instA+"/proxy?path=/pipeline/task/x/flight", + httprouter.Params{{Key: "id", Value: env.instA}}) + if w.Code != http.StatusConflict { + t.Fatalf("instance status code must pass through verbatim: %d", w.Code) + } + calls := fr.recorded() + if len(calls) != 1 || calls[0].Method != http.MethodGet || calls[0].Path != "/pipeline/task/x/flight" { + t.Fatalf("proxy must forward the read as-is: %+v", calls) + } +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 1ebd94e4..fcb3d8c0 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -72,6 +72,10 @@ type PipeModule struct { // to completion, but no task starts (auto_start/_start/_resume, and // tasks recreated by hot reload park STOPPED) until uncordoned cordoned atomic.Bool + // rolloutMu serializes staged-rollout state transitions between the + // reconciler tick and the _rollout endpoint (both read-modify-write + // the deployment's RolloutState) + rolloutMu sync.Mutex } // recordFailure stores a pipeline compile/load failure for the runtime @@ -117,6 +121,13 @@ func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { entry := map[string]interface{}{ "state": state, } + // the running config's content digest — same canonical shape the + // deployment renderer produces, so the control plane can verify a + // staged rollout wave actually runs the pushed content (#125) + entry["config_digest"] = deployedContentDigest(deployedPipelineMap(&ctx.Config, renderSpec{ + enabled: isPipelineEnabled(ctx.Config.Enabled), + singleton: ctx.Config.Singleton, + })) if taskID, ok := k.(string); ok { entry["generation"] = module.generationOf(taskID) } @@ -262,6 +273,30 @@ func (module *PipeModule) Setup() { api.HandleAPIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler) api.HandleUIMethod(api.GET, "/pipeline/deployments/:id/status", module.getDeploymentStatusHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + // staged rollout control (#125): manual advance / content rollback + api.HandleAPIMethod(api.POST, "/pipeline/deployments/:id/_rollout", module.rolloutActionHandler) + api.HandleUIMethod(api.POST, "/pipeline/deployments/:id/_rollout", module.rolloutActionHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + + // cluster broadcast of transient commands (#125): forwarded per target + // over the reverse channel; ?generation=N carries the #120 split-brain + // guard — instances on another config generation answer 409 + for _, action := range []string{"pause", "resume", "stop"} { + path := "/pipeline/deployments/:id/_" + action + api.HandleAPIMethod(api.POST, path, module.broadcastDeploymentActionHandler(action)) + api.HandleUIMethod(api.POST, path, module.broadcastDeploymentActionHandler(action), api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + } + + // per-instance maintenance over the reverse channel (#125): the + // instance-side primitives are #121's _drain/_cordon/_uncordon + for _, action := range []string{"drain", "cordon", "uncordon"} { + path := "/pipeline/instances/:id/_" + action + api.HandleAPIMethod(api.POST, path, module.instanceActionHandler(action)) + api.HandleUIMethod(api.POST, path, module.instanceActionHandler(action), api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineUpdate)) + } + // read-only passthrough onto an instance's own pipeline API + api.HandleAPIMethod(api.GET, "/pipeline/instances/:id/proxy", module.instanceProxyHandler) + api.HandleUIMethod(api.GET, "/pipeline/instances/:id/proxy", module.instanceProxyHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) + api.HandleAPIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler) api.HandleUIMethod(api.GET, "/pipeline/cluster/overview", module.getClusterOverviewHandler, api.RequireLogin(), api.RequirePermission(security.PermissionSystemPipelineRead)) diff --git a/modules/pipeline/rollout.go b/modules/pipeline/rollout.go new file mode 100644 index 00000000..e9e69128 --- /dev/null +++ b/modules/pipeline/rollout.go @@ -0,0 +1,304 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "fmt" + "net/http" + "sort" + + log "github.com/cihub/seelog" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// Staged rollout (#125, SPEC §8.2): a deployment with a rollout spec does +// not push new content to every target at once — the reconciler writes the +// canary wave first, waits until those instances REPORT running the staged +// content (digest match, see DeployedContentDigest), then advances in +// batches. The _rollout endpoint overrides the machine: force the next +// batch now, or roll every target back to the last converged content. + +// RolloutState is the persisted progress of a staged delivery. Lives on +// the deployment; the reconciler advances it, the endpoints override it. +type RolloutState struct { + // Active marks a rollout in progress (content staged, waves pending). + Active bool `json:"active,omitempty" elastic_mapping:"-"` + // Digest is the content digest being staged (what the waves converge to). + Digest string `json:"digest,omitempty" elastic_mapping:"digest:{type:keyword}"` + // Rolled lists the target instance IDs already on the staged content. + Rolled []string `json:"rolled,omitempty" elastic_mapping:"rolled:{type:keyword}"` + // RollbackContent is the last converged rendered config (the fallback + // the rollback action rewrites every target to). Digest-only would be + // unactionable — the bytes are what gets written back. + RollbackContent string `json:"rollback_content,omitempty" elastic_mapping:"-"` +} + +// rolloutSpecActive reports whether the spec asks for staged delivery at +// all (canary or batch > 0). +func rolloutSpecActive(spec RolloutSpec) bool { + return spec.Canary > 0 || spec.Batch > 0 +} + +// rolloutGateOwner picks the deployment whose rollout spec stages this +// pipeline's content: exactly one rollout bearer AND one uniform render +// spec across the merged targets. Two bearers, or a merge that produces +// diverging per-instance renders, have no single digest to stage — they +// converge ungated (logged once per pass by the caller's Debugf absence: +// the fleet behaves exactly as before the rollout feature existed). +func rolloutGateOwner(deps []*PipelineDeployment, desired map[string]renderSpec) *PipelineDeployment { + var owner *PipelineDeployment + for _, dep := range deps { + if rolloutSpecActive(dep.Placement.Rollout) { + if owner != nil { + return nil + } + owner = dep + } + } + if owner == nil { + return nil + } + var first renderSpec + for _, spec := range desired { + first = spec + break + } + for _, spec := range desired { + if spec != first { + return nil + } + } + return owner +} + +// rolloutWaveSizes returns (first, step): the canary wave size and the +// follow-up batch size. Only canary set → step = canary (keep the same +// chunk); only batch set → the first wave IS a batch. +func rolloutWaveSizes(spec RolloutSpec) (int, int) { + first, step := spec.Canary, spec.Batch + if first <= 0 { + first = spec.Batch + } + if step <= 0 { + step = first + } + return first, step +} + +// rolloutPlan is the pure gate the reconciler consults: given the rollout +// spec, the persisted state, the (deterministically ordered) target IDs, +// the digest of the content being distributed and a readiness callback, +// it returns which targets may receive the content NOW plus the next +// persisted state. +// +// - spec inactive → gate open, state untouched +// - content changed → new rollout: canary wave only, the +// previous converged content becomes the +// rollback target +// - same content, active → advance one batch when EVERY rolled +// target is ready; hold otherwise +// - waves cover all targets → rollout completes (Active=false) +// +// ready is only consulted for instances in the current wave. force skips +// the readiness check (the manual advance action). +func rolloutPlan(spec RolloutSpec, st RolloutState, targets []string, digest, prevContent string, ready func(id string) bool, force bool) (allowed map[string]bool, next RolloutState, changed bool) { + allowed = map[string]bool{} + next = st + + if !rolloutSpecActive(spec) { + for _, id := range targets { + allowed[id] = true + } + return allowed, next, false + } + + sorted := append([]string(nil), targets...) + sort.Strings(sorted) + + // drop rolled IDs that fell out of scope (deployment re-targeted) + rolled := make([]string, 0, len(st.Rolled)) + inTargets := map[string]bool{} + for _, id := range sorted { + inTargets[id] = true + } + for _, id := range st.Rolled { + if inTargets[id] { + rolled = append(rolled, id) + } + } + pruned := len(rolled) != len(st.Rolled) + + if !st.Active || st.Digest != digest { + // new staged rollout (or first ever): canary wave + first, _ := rolloutWaveSizes(spec) + if first > len(sorted) { + first = len(sorted) + } + rolled = append([]string(nil), sorted[:first]...) + next = RolloutState{ + Active: true, + Digest: digest, + Rolled: rolled, + RollbackContent: st.RollbackContent, + } + // the previous converged content is only a meaningful fallback + // when it differs from what we are staging now + if prevContent != "" { + next.RollbackContent = prevContent + } + changed = true + } else if force || rolloutWaveReady(rolled, ready) { + _, step := rolloutWaveSizes(spec) + nextRolled := append([]string(nil), rolled...) + for _, id := range sorted { + if len(nextRolled)-len(rolled) >= step { + break + } + if !containsString(nextRolled, id) { + nextRolled = append(nextRolled, id) + } + } + if len(nextRolled) != len(rolled) { + rolled = nextRolled + changed = true + } + } + + // completion: every target carries the staged content + if len(rolled) >= len(sorted) { + next = RolloutState{ + Active: false, + Digest: digest, + Rolled: rolled, + RollbackContent: next.RollbackContent, + } + if st.Active { + changed = true + } + } else { + next.Active = true + next.Digest = digest + next.Rolled = rolled + } + if pruned && !changed { + changed = true + } + + for _, id := range rolled { + allowed[id] = true + } + return allowed, next, changed +} + +// rolloutWaveReady reports whether every instance of the current wave +// confirms running the staged content. An instance that never reported +// blocks the wave — conservative on purpose: a canary that went dark +// must stop the rollout, not let it sweep the fleet (the manual advance +// endpoint is the operator's override). +func rolloutWaveReady(rolled []string, ready func(id string) bool) bool { + for _, id := range rolled { + if !ready(id) { + return false + } + } + return true +} + +func containsString(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// rolloutActionHandler — POST /pipeline/deployments/:id/_rollout +// Body: {"action":"advance"|"rollback"}. +// advance → force the next batch immediately (readiness bypass) +// rollback → rewrite EVERY target to the last converged content at once +func (h *PipeModule) rolloutActionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + var body struct { + Action string `json:"action"` + } + if err := h.DecodeJSON(req, &body); err != nil { + h.Error400(w, err.Error()) + return + } + + h.rolloutMu.Lock() + octx := orm.NewContextWithParent(req.Context()).DirectAccess() + dep := PipelineDeployment{} + dep.ID = id + exists, err := orm.GetV2(octx, &dep) + if err != nil || !exists { + h.rolloutMu.Unlock() + h.WriteOpRecordNotFoundJSON(w, id) + return + } + + switch body.Action { + case "advance": + if !dep.State.Active { + h.rolloutMu.Unlock() + h.WriteError(w, "no rollout in progress (state inactive)", http.StatusConflict) + return + } + // force-advance one batch: readiness always true + _, planned, _ := rolloutPlan(dep.Placement.Rollout, dep.State, + deploymentTargetIDs(&dep, h.loadRegisteredInstances(octx)), + dep.State.Digest, "", func(string) bool { return true }, true) + dep.State = planned + if err := orm.Save(octx, &dep); err != nil { + h.rolloutMu.Unlock() + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.rolloutMu.Unlock() + h.reconcileOnce() + h.WriteJSON(w, util.MapStr{"result": "advanced", "rollout": dep.State}, http.StatusOK) + + case "rollback": + if dep.State.RollbackContent == "" { + h.rolloutMu.Unlock() + h.WriteError(w, "no previous converged content recorded for this deployment", http.StatusConflict) + return + } + if err := h.rollbackDeploymentContent(octx, &dep); err != nil { + h.rolloutMu.Unlock() + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + // The rollback is a BRAKE, not a desired-state change: the pipeline + // object still carries the new content, so the next reconciler pass + // starts a fresh CANARY-staged rollout toward it — visible, gradual, + // stoppable. Making a rollback permanent means rolling the pipeline + // version back too (POST /pipelines/:id/versions/:v/rollback). + dep.State = RolloutState{ + Active: false, + Digest: deployedContentDigestOf([]byte(dep.State.RollbackContent)), + RollbackContent: "", // one level of fallback; deeper = pipeline version history + } + if err := orm.Save(octx, &dep); err != nil { + h.rolloutMu.Unlock() + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.rolloutMu.Unlock() + log.Infof("deployment %s rolled back: every target rewritten to the last converged content", dep.Name) + h.WriteJSON(w, util.MapStr{ + "result": "rolled-back", + "rollout": dep.State, + "note": "desired state still carries the new content — the next reconcile re-stages it from canary; roll the pipeline version back to make this permanent", + }, http.StatusOK) + + default: + h.rolloutMu.Unlock() + h.Error400(w, fmt.Sprintf("unknown action %q (advance|rollback)", body.Action)) + } +} diff --git a/modules/pipeline/rollout_test.go b/modules/pipeline/rollout_test.go new file mode 100644 index 00000000..df1cba1e --- /dev/null +++ b/modules/pipeline/rollout_test.go @@ -0,0 +1,339 @@ +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infinilabs.com */ + +package pipeline + +import ( + "fmt" + "testing" + + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/configs/server" +) + +// ── rolloutPlan (the pure gate) ────────────────────────────────────────── + +func TestRolloutPlanCanaryThenBatches(t *testing.T) { + spec := RolloutSpec{Canary: 1, Batch: 2} + targets := []string{"i1", "i2", "i3", "i4", "i5"} + digest := "d-new" + + // 1. first pass: canary only, rollback content captured + allowed, st, changed := rolloutPlan(spec, RolloutState{}, targets, digest, "old-content", nil, false) + if !changed || !st.Active || len(st.Rolled) != 1 || !allowed["i1"] || allowed["i2"] { + t.Fatalf("canary wave wrong: allowed=%v state=%+v changed=%v", allowed, st, changed) + } + if st.RollbackContent != "old-content" { + t.Fatalf("rollback target must capture prev content, got %q", st.RollbackContent) + } + + // 2. canary not ready → hold exactly where we are + ready := func(id string) bool { return false } + allowed, st2, changed := rolloutPlan(spec, st, targets, digest, "", ready, false) + if changed || !st2.Active || len(st2.Rolled) != 1 || len(allowed) != 1 { + t.Fatalf("unready wave must hold: allowed=%v state=%+v changed=%v", allowed, st2, changed) + } + + // 3. canary ready → one batch of 2 more + ready = func(id string) bool { return true } + _, st3, changed := rolloutPlan(spec, st, targets, digest, "", ready, false) + if !changed || len(st3.Rolled) != 3 || !st3.Active { + t.Fatalf("ready wave must advance by batch: %+v changed=%v", st3, changed) + } + + // 4. last batch covers the rest → rollout completes + _, st4, _ := rolloutPlan(spec, st3, targets, digest, "", ready, false) + if st4.Active || len(st4.Rolled) != 5 { + t.Fatalf("final wave must complete the rollout: %+v", st4) + } + if st4.RollbackContent != "old-content" { + t.Fatalf("completed rollout must keep the rollback target, got %q", st4.RollbackContent) + } +} + +func TestRolloutPlanForceAdvancesWithoutReadiness(t *testing.T) { + spec := RolloutSpec{Canary: 1, Batch: 1} + targets := []string{"a", "b"} + _, st, _ := rolloutPlan(spec, RolloutState{}, targets, "d", "", nil, false) + if len(st.Rolled) != 1 { + t.Fatalf("canary first: %+v", st) + } + // readiness says no, force says go — manual advance wins + _, forced, _ := rolloutPlan(spec, st, targets, "d", "", func(string) bool { return false }, true) + if len(forced.Rolled) != 2 || forced.Active { + t.Fatalf("forced advance must finish the rollout: %+v", forced) + } +} + +func TestRolloutPlanUnreportedCanaryBlocks(t *testing.T) { + spec := RolloutSpec{Canary: 1, Batch: 1} + targets := []string{"a", "b"} + _, st, _ := rolloutPlan(spec, RolloutState{}, targets, "d", "", nil, false) + // readiness callback returning false = canary never confirmed + _, held, changed := rolloutPlan(spec, st, targets, "d", "", func(string) bool { return false }, false) + if changed || len(held.Rolled) != 1 { + t.Fatalf("a dark canary must hold the rollout: %+v changed=%v", held, changed) + } +} + +func TestRolloutPlanPrunesOutOfScope(t *testing.T) { + spec := RolloutSpec{Canary: 2, Batch: 2} + st := RolloutState{Active: true, Digest: "d", Rolled: []string{"gone", "keep"}} + allowed, next, _ := rolloutPlan(spec, st, []string{"keep", "other"}, "d", "", func(string) bool { return true }, false) + if allowed["gone"] { + t.Fatal("out-of-scope instance must not stay in the wave") + } + if len(next.Rolled) != 2 || !containsString(next.Rolled, "keep") || !containsString(next.Rolled, "other") { + t.Fatalf("pruned wave must re-batch from the survivors: %+v", next) + } +} + +func TestRolloutPlanInactiveSpecOpenGate(t *testing.T) { + allowed, _, changed := rolloutPlan(RolloutSpec{}, RolloutState{}, []string{"a", "b"}, "d", "", nil, false) + if len(allowed) != 2 { + t.Fatalf("no rollout spec = ungated: %v", allowed) + } + if changed { + t.Fatal("ungated pass must not report a state change") + } +} + +// ── digest parity: control-plane render vs instance-side reconstruction ── + +func TestDeployedContentDigestParity(t *testing.T) { + cfg := &pipeline.PipelineConfigV2{} + cfg.Name = "parity-check" + cfg.Processors = []map[string]interface{}{ + {"test_rollout_echo": map[string]interface{}{"message": "hi"}}, + {"test_rollout_echo": map[string]interface{}{"message": "bye"}}, + } + + spec := renderSpec{enabled: true, singleton: false} + rendered := renderDeployedPipeline(cfg, spec) + controlSide := deployedContentDigestOf(rendered) + + // instance side: the config an agent actually loads from the rendered + // file, re-rendered through the same canonical shape + var wrapper struct { + Pipeline []struct { + Name string `json:"name"` + Enabled *bool `json:"enabled"` + AutoStart bool `json:"auto_start"` + KeepRunning bool `json:"keep_running"` + Singleton bool `json:"singleton"` + Processors []map[string]interface{} `json:"processor"` + } `json:"pipeline"` + } + if err := util.FromJSONBytes(rendered, &wrapper); err != nil || len(wrapper.Pipeline) != 1 { + t.Fatalf("rendered config must parse: %v", err) + } + loaded := pipeline.PipelineConfigV2{ + Name: wrapper.Pipeline[0].Name, + Enabled: wrapper.Pipeline[0].Enabled, + AutoStart: wrapper.Pipeline[0].AutoStart, + KeepRunning: wrapper.Pipeline[0].KeepRunning, + Singleton: wrapper.Pipeline[0].Singleton, + Processors: wrapper.Pipeline[0].Processors, + } + instanceSide := deployedContentDigest(deployedPipelineMap(&loaded, renderSpec{ + enabled: isPipelineEnabled(loaded.Enabled), + singleton: loaded.Singleton, + })) + + if controlSide != instanceSide || controlSide == "" { + t.Fatalf("digest mismatch: control=%s instance=%s", controlSide, instanceSide) + } +} + +// ── staged reconcile through the real ORM harness ──────────────────────── + +// reportInstance overwrites an instance's runtime report (what _sync +// would upsert). +func reportInstance(t *testing.T, id string, pipelines string) { + t.Helper() + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &model.Instance{}) + inst := model.Instance{} + inst.ID = id + found, err := orm.GetV2(octx, &inst) + if err != nil || !found { + t.Fatalf("instance %s missing: %v", id, err) + } + inst.RuntimePipelines = pipelines + if err := orm.Save(octx, &inst); err != nil { + t.Fatal(err) + } +} + +func loadDeployment(t *testing.T, id string) PipelineDeployment { + t.Helper() + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineDeployment{}) + dep := PipelineDeployment{} + dep.ID = id + found, err := orm.GetV2(octx, &dep) + if err != nil || !found { + t.Fatalf("deployment %s missing: %v", id, err) + } + return dep +} + +func rowInstanceIDs(rows map[string]server.ManagedConfig) []string { + ids := make([]string, 0, len(rows)) + for id := range rows { + ids = append(ids, id) + } + return ids +} + +func TestStagedReconcileCanaryBatchComplete(t *testing.T) { + env := newDeployTestEnv(t) + dep := &PipelineDeployment{ + PipelineID: env.pid, + Enabled: true, + Placement: PlacementSpec{ + Strategy: "all", + Instances: []string{env.instA, env.instB, env.instC}, + }, + } + dep.ID = "d1" // shared upsert id: exactly ONE deployment row exists per test run + dep.Name = "rollout-test" + env.saveDeployment(dep) + + // phase 0: converge WITHOUT a rollout spec — every target on content v1 + env.reconcile() + rows := env.rows() + if len(rows) != 3 { + t.Fatalf("ungated reconcile must converge all 3 targets, got %d", len(rows)) + } + var v1Content string + for _, row := range rows { + v1Content = row.Content + break + } + + // change the pipeline (content v2) AND arm staged delivery + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &pipeline.PipelineConfigV2{}) + cfg := pipeline.PipelineConfigV2{} + cfg.ID = env.pid + if found, err := orm.GetV2(octx, &cfg); err != nil || !found { + t.Fatalf("pipeline missing: %v", err) + } + cfg.Processors = append(cfg.Processors, map[string]interface{}{ + "validate_test_echo": map[string]interface{}{"message": "v2-marker"}, + }) + if err := orm.Save(octx, &cfg); err != nil { + t.Fatal(err) + } + armed := loadDeployment(t, dep.ID) + armed.Placement.Rollout = RolloutSpec{Canary: 1, Batch: 1} + env.saveDeployment(&armed) + + // pass 1: canary only — exactly one row on the new content + env.reconcile() + rows = env.rows() + if len(rows) != 3 { + t.Fatalf("staging never drops converged rows, got %d", len(rows)) + } + var canaryID string + onV2 := 0 + for id, row := range rows { + if row.Content != v1Content { + canaryID = id + onV2++ + } + } + if onV2 != 1 { + t.Fatalf("canary pass must update exactly 1 row to v2, got %d", onV2) + } + digest := deployedContentDigestOf([]byte(rows[canaryID].Content)) + + // pass 2: canary has not reported → nothing else moves + env.reconcile() + onV2 = 0 + for _, row := range env.rows() { + if row.Content != v1Content { + onV2++ + } + } + if onV2 != 1 { + t.Fatalf("unreported canary must hold the rollout, onV2=%d", onV2) + } + + // canary reports running the pushed content → next batch opens + reportInstance(t, canaryID, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":%q}}`, env.pipeName, digest)) + env.reconcile() + onV2 = 0 + for _, row := range env.rows() { + if row.Content != v1Content { + onV2++ + } + } + if onV2 != 2 { + t.Fatalf("ready canary must open the next batch, onV2=%d", onV2) + } + + // every v2 target reports → completion covers the fleet + for id, row := range env.rows() { + if row.Content != v1Content { + reportInstance(t, id, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":%q}}`, env.pipeName, digest)) + } + } + env.reconcile() + onV2 = 0 + for _, row := range env.rows() { + if row.Content != v1Content { + onV2++ + } + } + if onV2 != 3 { + t.Fatalf("completed rollout must cover all targets, onV2=%d", onV2) + } + + // terminal state persisted: inactive, v1 recorded as the rollback target + st := loadDeployment(t, dep.ID) + if st.State.Active { + t.Fatalf("rollout must be terminal: %+v", st.State) + } + if st.State.RollbackContent != v1Content { + t.Fatalf("rollback target must be the pre-rollout content (v1), got %d bytes", len(st.State.RollbackContent)) + } +} + +func TestStagedReconcileOldDigestDoesNotAdvance(t *testing.T) { + env := newDeployTestEnv(t) + dep := &PipelineDeployment{ + PipelineID: env.pid, + Enabled: true, + Placement: PlacementSpec{ + Strategy: "all", + Instances: []string{env.instA, env.instB}, + Rollout: RolloutSpec{Canary: 1, Batch: 1}, + }, + } + dep.ID = "d1" // shared upsert id (see rollout test above) + dep.Name = "rollout-stale-digest" + env.saveDeployment(dep) + + env.reconcile() + rows := env.rows() + if len(rows) != 1 { + t.Fatalf("canary pass must write exactly 1 row, got %d", len(rows)) + } + var canaryID string + for id := range rows { + canaryID = id + } + // canary reports STARTED but with the OLD content's digest — the very + // lie the digest check exists to catch + reportInstance(t, canaryID, fmt.Sprintf(`{%q:{"state":"STARTED","config_digest":"stale"}}`, env.pipeName)) + env.reconcile() + if got := len(env.rows()); got != 1 { + t.Fatalf("a STARTED-but-stale canary must hold the rollout, rows=%d", got) + } +} From 2e4b5b57a9fa39c4a49e19f5c0f4df02d727efb1 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 05:35:26 +0800 Subject: [PATCH 58/65] feat: instance cordon flag reports over configs _sync (#128) and RequestReconcile poke - RuntimeReport.Cordoned: the pipeline module exposes the local cordon flag via configs_client.SetInstanceFlagsProvider and merges it into every runtime snapshot; the configs server stamps it onto Instance (placement keeps flowing configs so running tasks drain in place) - InstanceReport.Cordoned surfaces the flag in deployment status, so the fleet UI can show cordoned instances on the deployment matrix - RequestReconcile() pokes the deployment reconciler for an immediate pass so publishers converge right after upserting deployments instead of waiting out the 15s tick --- core/model/instance.go | 6 ++++++ modules/configs/client/client.go | 14 ++++++++++++++ modules/configs/common/domain.go | 6 ++++++ modules/configs/server/server.go | 29 +++++++++++++++++++++-------- modules/pipeline/deployment.go | 31 ++++++++++++++++++++++++++----- modules/pipeline/module.go | 5 +++++ 6 files changed, 78 insertions(+), 13 deletions(-) diff --git a/core/model/instance.go b/core/model/instance.go index f0aa74ab..56b3da17 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -91,6 +91,12 @@ type Instance struct { // a JSON string (map[string]PipelineRuntimeState). RuntimePipelines string `json:"runtime_pipelines,omitempty" elastic_mapping:"runtime_pipelines:{type:text}"` + // Cordoned mirrors the instance-local maintenance flag (#121): running + // tasks drain but nothing starts. Reported with every _sync (#128) so + // the control plane can surface it; placement deliberately keeps + // flowing configs to cordoned instances (recall would strand tasks). + Cordoned bool `json:"cordoned,omitempty" elastic_mapping:"cordoned:{type:boolean}"` + //SearchText string `json:"search_text,omitempty" elastic_mapping:"search_text:{type:text,index_prefixes:{},index_phrases:true, analyzer:suggest_text_search }"` } diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 42bfc0da..25d94e19 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -59,6 +59,15 @@ func SetPipelineRuntimeSnapshotProvider(fn func() map[string]interface{}) { getPipelineRuntimeSnapshot = fn } +// SetInstanceFlagsProvider wires instance-level maintenance flags into the +// sync client's runtime report (called from PipeModule.Setup; the pipeline +// module owns the cordon flag). +func SetInstanceFlagsProvider(fn func() map[string]interface{}) { + getInstanceFlags = fn +} + +var getInstanceFlags func() map[string]interface{} + const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" const legacyManagedRegisterCompatMaxVersion = "1.30.4" @@ -597,6 +606,11 @@ func runtimePipelineSnapshot() *common.RuntimeReport { return nil } report := &common.RuntimeReport{Pipelines: map[string]common.PipelineRuntimeState{}} + if flags := getInstanceFlags(); flags != nil { + if v, ok := flags["cordoned"].(bool); ok { + report.Cordoned = v + } + } for name, state := range snapshot { m, _ := state.(map[string]interface{}) stateStr, _ := m["state"].(string) diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 5c859d8b..f5022799 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -96,6 +96,12 @@ type RuntimeReport struct { // Pipelines maps task name → state + the managed-config version the // instance last loaded (0 = unknown/not from managed config). Pipelines map[string]PipelineRuntimeState `json:"pipelines,omitempty"` + // Cordoned mirrors the instance-local maintenance flag (#121/#128): + // running tasks drain but nothing starts. Reported so the control + // plane can see WHO is cordoned without a proxy round-trip — the + // placement side keeps configs flowing (a cordoned instance still + // syncs; recall-on-cordon would strand its running tasks). + Cordoned bool `json:"cordoned,omitempty"` } type PipelineRuntimeState struct { diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go index b18a0c6f..a7c0e4f0 100644 --- a/modules/configs/server/server.go +++ b/modules/configs/server/server.go @@ -555,15 +555,28 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt log.Warnf("configs server: heartbeat upsert failed for %s: %v", obj.Client.ID, err) } - // Runtime report: persist the instance's pipeline state for the - // deployment reconciler (desired-vs-reported comparison). - if obj.Runtime != nil && len(obj.Runtime.Pipelines) > 0 { - rtJSON, rerr := json.Marshal(obj.Runtime.Pipelines) - if rerr == nil { - obj.Client.RuntimePipelines = string(rtJSON) - _, _ = upsertInstance(&obj.Client) // second upsert with runtime + // Runtime report: persist the instance's pipeline state for the + // deployment reconciler (desired-vs-reported comparison). The + // cordon flag rides the same report (#128) — the upsert must run + // even when no pipelines are running, or the un-cordon transition + // would never reach the control plane. + if obj.Runtime != nil { + updated := false + if len(obj.Runtime.Pipelines) > 0 { + rtJSON, rerr := json.Marshal(obj.Runtime.Pipelines) + if rerr == nil { + obj.Client.RuntimePipelines = string(rtJSON) + updated = true + } + } + if obj.Runtime.Cordoned != obj.Client.Cordoned { + obj.Client.Cordoned = obj.Runtime.Cordoned + updated = true + } + if updated { + _, _ = upsertInstance(&obj.Client) // second upsert with runtime + } } - } assigned, aerr := loadAssignedConfigs(obj.Client.ID) if aerr != nil { diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 583ebe13..6ffbc50b 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -118,10 +118,29 @@ type InstanceReport struct { // Stale marks an instance that still runs the pipeline although the // deployment scope no longer includes it (recall in flight/failed) Stale bool `json:"stale,omitempty"` + // Cordoned mirrors the instance's maintenance flag (reported via + // _sync, #128): running tasks drain, nothing new starts there + Cordoned bool `json:"cordoned,omitempty"` } // ─── reconciler ────────────────────────────────────────────────────────── +// reconcilePoke requests an immediate reconciler pass outside the 15s tick. +// Buffered so pokers never block; a poke landing while a pass is already +// running just coalesces into one follow-up pass. +var reconcilePoke = make(chan struct{}, 1) + +// RequestReconcile asks the deployment reconciler to run a pass as soon as +// possible (async, best-effort). Publishers call it after upserting +// PipelineDeployment/PipelineConfigV2 so convergence starts immediately +// instead of on the next tick (#128). +func RequestReconcile() { + select { + case reconcilePoke <- struct{}{}: + default: + } +} + // reconcileDeployments runs on a ticker: for each enabled deployment, // compare desired state with the managed-config rows that exist, and // create/update/delete configs to converge. @@ -132,11 +151,12 @@ func (module *PipeModule) startDeploymentReconciler() { for { select { case <-ticker.C: - if module.closed.Load() { - return - } - module.reconcileOnce() + case <-reconcilePoke: + } + if module.closed.Load() { + return } + module.reconcileOnce() } }() log.Debugf("deployment reconciler started (interval 15s)") @@ -628,7 +648,8 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat // stale = the instance still runs the pipeline but the // scope no longer includes it (config recall in flight, // or a recall that failed — visible instead of silent) - Stale: !inScope, + Stale: !inScope, + Cordoned: inst.Cordoned, }) } } diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index fcb3d8c0..84a8bc02 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -185,6 +185,11 @@ func (module *PipeModule) Setup() { // provide the runtime snapshot for the configs _sync client // (deployment reconciler reads it for health status) configs_client.SetPipelineRuntimeSnapshotProvider(module.RuntimeSnapshot) + // instance-level maintenance flags ride the same _sync report (#128): + // the control plane mirrors the local cordon state onto the Instance + configs_client.SetInstanceFlagsProvider(func() map[string]interface{} { + return map[string]interface{}{"cordoned": module.cordoned.Load()} + }) // ORM schema registration (table/index auto-creation on startup) orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs") orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions") From 15b4249a00e14bdad7afa036f6f747aa22bb3e52 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 05:58:35 +0800 Subject: [PATCH 59/65] feat(pipeline): deployment CRUD handlers poke the reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create/update/delete of a deployment requests an immediate reconcile pass so convergence starts right away instead of on the next 15s tick — same behavior the logpilot publisher gets via RequestReconcile. --- modules/pipeline/deployment.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 6ffbc50b..c4cffa67 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -727,6 +727,7 @@ func (h *PipeModule) createDeploymentHandler(w http.ResponseWriter, req *http.Re h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + RequestReconcile() h.WriteJSON(w, dep, http.StatusCreated) } @@ -807,6 +808,7 @@ func (h *PipeModule) updateDeploymentHandler(w http.ResponseWriter, req *http.Re h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + RequestReconcile() h.WriteJSON(w, newDep, http.StatusOK) } @@ -853,6 +855,7 @@ func (h *PipeModule) deleteDeploymentHandler(w http.ResponseWriter, req *http.Re } } } + RequestReconcile() h.WriteJSON(w, util.MapStr{"deleted": true, "id": id}, http.StatusOK) } From f9ae1f1785e9b30b1a640b240bf2d00dabe4319e Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 06:02:55 +0800 Subject: [PATCH 60/65] fix(pipeline): keep_running tasks report STARTED between passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keep_running loop parks the context in FINISHED during the inter-pass retry sleep. The runtime snapshot mapped FINISHED straight to STOPPED, so a healthy task flapped the fleet view to down once per second and rollout readiness (requires STARTED) stalled forever — caught live in #130 acceptance with two gateways both running the pinned echo pipeline while the deployment reported 0/2 ready. FINISHED + keep_running + not exiting now reports STARTED (the loop goroutine is alive and will run again); FAILED passes still report FAILED so crash-loops hold rollout waves. STOPPING also maps to STOPPED instead of UNKNOWN. --- modules/pipeline/module.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 84a8bc02..316d2c68 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -110,13 +110,25 @@ func (module *PipeModule) RuntimeSnapshot() map[string]interface{} { } } state := "UNKNOWN" - switch ctx.GetRunningState() { + switch st := ctx.GetRunningState(); st { case pipeline.STARTED, pipeline.STARTING: state = "STARTED" case pipeline.FAILED: state = "FAILED" - case pipeline.STOPPED, pipeline.FINISHED: + case pipeline.STOPPED, pipeline.STOPPING: state = "STOPPED" + case pipeline.FINISHED: + // keep_running tasks idle between passes (retry delay) with + // state FINISHED — the loop goroutine is alive and will run + // again. Reporting STOPPED here flaps a healthy fleet to + // "down" once per inter-pass sleep and permanently stalls + // rollout waves (readiness requires STARTED). A FAILED pass + // still reports FAILED: crash-loops must hold rollouts. + if ctx.Config.KeepRunning && !ctx.IsExit() { + state = "STARTED" + } else { + state = "STOPPED" + } } entry := map[string]interface{}{ "state": state, From 14872ef813d9f31c7d34de1653ac8261d62b5c55 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 06:12:00 +0800 Subject: [PATCH 61/65] fix(pipeline): deployment status reports real last-seen, rollout readiness requires a live heartbeat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LastSeen stamped time.Now() at read time, so a killed instance kept displaying STARTED with a brand-new timestamp forever — seen live in #130 when the lock-holding gateway was killed: the dead row looked fresher than the takeover. Now it shows the instance's last _sync heartbeat. Rollout readiness additionally requires a heartbeat within 3 default sync intervals: a frozen STARTED+digest report from a dead instance must not keep counting as a ready canary. --- modules/pipeline/deployment.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index c4cffa67..97814390 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -351,12 +351,19 @@ func (module *PipeModule) reconcilePipeline(octx *orm.Context, pipelineID string byID[instances[i].ID] = instances[i] } singleton := owner.Placement.Strategy == "singleton" - ready := func(id string) bool { - inst, ok := byID[id] - if !ok { - return false - } - st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name] + ready := func(id string) bool { + inst, ok := byID[id] + if !ok { + return false + } + // a frozen report must not count forever: an instance that + // died keeps its last STARTED+digest — require a heartbeat + // within 3 default sync intervals (30s each) or the wave + // holds until a live holder proves the content + if inst.Updated != nil && time.Since(*inst.Updated) > 90*time.Second { + return false + } + st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name] if !reported || st.State == "FAILED" { return false } @@ -616,6 +623,15 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat for _, inst := range instances { inScope := broadcast || targets[inst.ID] + // the honest "when did this instance last check in" — the ORM + // record's updated stamp refreshes on every _sync heartbeat. + // Read-time now() would show dead instances as freshly reporting + // forever (#130: a killed gateway kept displaying STARTED with a + // brand-new last_seen). + lastSeen := time.Now().UTC() + if inst.Updated != nil { + lastSeen = inst.Updated.UTC() + } if inst.RuntimePipelines == "" { // in-scope instances that have not reported yet are part of // the desired count — surface them as pending instead of @@ -625,7 +641,7 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat InstanceID: inst.ID, State: "PENDING", Version: dep.Version, - LastSeen: time.Now().UTC(), + LastSeen: lastSeen, }) } continue @@ -643,7 +659,7 @@ func (h *PipeModule) getDeploymentStatus(dep *PipelineDeployment) DeploymentStat InstanceID: inst.ID, State: st.State, Version: dep.Version, - LastSeen: time.Now().UTC(), + LastSeen: lastSeen, Reason: st.Reason, // stale = the instance still runs the pipeline but the // scope no longer includes it (config recall in flight, From 0f84de33d3884b38b25a6b30e40026f46443cc61 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 06:37:31 +0800 Subject: [PATCH 62/65] feat(easysearch): EnabledByDefault opt-in for apps that own the cluster registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway registers this module for one documented purpose — receiving LogPilot's sink-cluster pushes on /easysearch/ at stream publish time — but with no easysearch: section in its yml the module stayed dark and publishes failed with bare 404s while the sink lane logged 'cluster metadata not ready' forever (found live in #130 acceptance). EnabledByDefault flips the module on when the section is absent, so the registering app's intent survives an unconfigured yml. --- modules/easysearch/module.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/easysearch/module.go b/modules/easysearch/module.go index cdfe1962..e692c367 100644 --- a/modules/easysearch/module.go +++ b/modules/easysearch/module.go @@ -21,6 +21,13 @@ import ( // registration is a later refactor. type Module struct { moduleConfig moduleConfig + + // EnabledByDefault lets the registering app opt the module in when the + // yml carries no `easysearch:` section. The gateway sets this: its + // documented role includes receiving LogPilot's sink-cluster pushes on + // /easysearch/ at publish time, and a missing section silently broke + // that flow with bare 404s (found live in #130 acceptance). + EnabledByDefault bool } type moduleConfig struct { @@ -35,6 +42,9 @@ func (m *Module) Setup() { if exists && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { panic(err) } + if !exists && m.EnabledByDefault { + m.moduleConfig.Enabled = true + } if !m.moduleConfig.Enabled { return From 1e2ccfe4a3415ee391a04262a62911276537b28c Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 07:09:21 +0800 Subject: [PATCH 63/65] fix(pipeline): stage stats actually reach readers and survive security-enabled apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two live-verification bugs from #129 product binding: - installStats registered one statsRecorder as the StepEvent observer and stored a SECOND instance in globalStats — counters accumulated into an object nobody read; every stage view (runtime endpoint, persistence, cluster aggregation) stayed empty forever. One instance now serves both roles; TestStats_ObserverWiring drives a real Processors.Process chain and fails on the two-instance shape. - persistStageSnapshot wrote with a bare orm context; the security data-op hook requires a session user or the direct-access flags, so the first snapshot tick panicked ("invalid user") and killed security-enabled apps (coco) the moment pipeline_metrics persistence was enabled. The writer is a background system writer — it now uses DirectAccess(). --- modules/pipeline/metrics.go | 9 +++++++-- modules/pipeline/metrics_persist.go | 6 +++++- modules/pipeline/metrics_test.go | 30 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/modules/pipeline/metrics.go b/modules/pipeline/metrics.go index 15b916ac..49d5ac25 100644 --- a/modules/pipeline/metrics.go +++ b/modules/pipeline/metrics.go @@ -105,8 +105,13 @@ func installStats() { if globalStats.Load() != nil { return } - pipeline.RegisterObserver(newStatsRecorder()) - globalStats.Store(newStatsRecorder()) + // one recorder for BOTH roles: registered observer (fed by the + // execution loop) and the globalStats snapshot (read by stageStats). + // Two separate instances here would count into one and read from the + // other — stage views would silently stay empty. + rec := newStatsRecorder() + pipeline.RegisterObserver(rec) + globalStats.Store(rec) } var _ = time.Second diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index cc067840..0736e080 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -82,7 +82,11 @@ func (module *PipeModule) persistStageSnapshot() { return true }) now := time.Now().UTC() - octx := orm.NewContext() + // DirectAccess: this is a background system writer (no request user); + // the security orm hooks require either a session user or the direct + // read/write flags — a bare context panics on the first save under + // security-enabled apps (coco) and takes the process down + octx := orm.NewContext().DirectAccess() for _, name := range names { for _, st := range stageStats(name) { if st.Count == 0 && st.Errors == 0 { diff --git a/modules/pipeline/metrics_test.go b/modules/pipeline/metrics_test.go index a9e3d0c9..58ea7dbc 100644 --- a/modules/pipeline/metrics_test.go +++ b/modules/pipeline/metrics_test.go @@ -35,3 +35,33 @@ func TestStats_AggregatesPerProcessor(t *testing.T) { t.Fatalf("avg latency = %d, want %d", found.AvgLatency, want) } } + +// obsWiringProc pairs with TestStats_ObserverWiring: a no-op processor so +// the shared Processors.Process loop runs it and emits real StepEvents. +type obsWiringProc struct{} + +func (obsWiringProc) Name() string { return "wiring_probe" } +func (obsWiringProc) Process(_ *pipeline.Context) error { return nil } + +// TestStats_ObserverWiring guards the installStats identity contract: the +// recorder registered as observer must be the one stageStats reads. The +// historical bug registered one instance and stored a second — every +// stage view stayed empty while counters accumulated into a dead object. +func TestStats_ObserverWiring(t *testing.T) { + installStats() + + ctx := pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "stats-wiring"}) + ctx.Started() + chain := &pipeline.Processors{List: []pipeline.Processor{obsWiringProc{}}} + if err := chain.Process(ctx); err != nil { + t.Fatal(err) + } + + stages := stageStats("stats-wiring") + for _, st := range stages { + if st.Stage == "wiring_probe" && st.Carrier == "processor" && st.Count >= 1 { + return + } + } + t.Fatalf("observer-fed counters missing from stageStats: %+v", stages) +} From 891adbc00b31dc0f7b684317ba9362ed7541baa9 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 07:24:06 +0800 Subject: [PATCH 64/65] feat(pipeline): stage stats ride the configs _sync report; retention bounds the table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instances without a shared datastore (gateway-type, local sqlite) never reached the manager's cluster aggregation — /pipeline/cluster/stages only saw rows from ES-primary apps. The counters now travel the existing heartbeat: - RuntimeReport grows stage_stats: the instance's cumulative per-stage rows (wire shape common.StageStat), collected by the pipeline module (reportedStageRows — processor rows tagged by pipeline Config.Name, lane carriers once, not per pipeline) and attached by the sync client - the configs server sinks reported rows into its own store via PersistReportedStageStats (registered from PipeModule.Setup; inert on apps without the configs server) — same row shape and latest-tick fold as the local exporter - pipeline_metrics.persistence.retention_seconds (default 1h, negative disables): both write paths prune beyond the window — the fold only reads each instance's latest tick, unbounded history would eventually crowd the aggregation query - test ORM setup refactored into a side-effect-free ensureDeployORMHandler: orm.Register is a process-global last-wins registry, a second handler registration anywhere flips later tests onto a handler missing their schemas Live-verified on the e2e fleet: gateway's 边缘心跳 echo/node_stamp rows land in coco's store via _sync (two cumulative ticks) and the product's cluster tab aggregates them beside coco's own connector_dispatcher rows. --- modules/configs/client/client.go | 17 +++ modules/configs/common/domain.go | 19 +++ modules/configs/server/server.go | 19 +++ modules/pipeline/deployment_targeting_test.go | 21 ++- modules/pipeline/metrics.go | 19 +++ modules/pipeline/metrics_persist.go | 130 +++++++++++++++--- modules/pipeline/metrics_report_test.go | 105 ++++++++++++++ modules/pipeline/module.go | 7 + 8 files changed, 312 insertions(+), 25 deletions(-) create mode 100644 modules/pipeline/metrics_report_test.go diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 25d94e19..7ec8aacf 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -68,6 +68,16 @@ func SetInstanceFlagsProvider(fn func() map[string]interface{}) { var getInstanceFlags func() map[string]interface{} +// SetStageStatsProvider wires the pipeline module's per-stage counters +// into the sync report (called from PipeModule.Setup). The manager +// persists them so cluster aggregation covers instances that do not +// share its datastore. +func SetStageStatsProvider(fn func() []common.StageStat) { + getStageStats = fn +} + +var getStageStats func() []common.StageStat + const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" const legacyManagedRegisterCompatMaxVersion = "1.30.4" @@ -624,6 +634,13 @@ func runtimePipelineSnapshot() *common.RuntimeReport { Reason: reason, } } + // per-stage counters ride the same heartbeat — cumulative rows the + // manager snapshots; zero rows are noise, skip them + if getStageStats != nil { + if rows := getStageStats(); len(rows) > 0 { + report.StageStats = rows + } + } return report } diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index f5022799..bf49380b 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -102,6 +102,25 @@ type RuntimeReport struct { // placement side keeps configs flowing (a cordoned instance still // syncs; recall-on-cordon would strand its running tasks). Cordoned bool `json:"cordoned,omitempty"` + // StageStats carries the instance's cumulative per-stage counters on + // the same heartbeat — the same rows the local metrics exporter + // persists. Instances without a shared datastore (gateway-type, local + // sqlite) still appear in the manager's cluster aggregation. + StageStats []StageStat `json:"stage_stats,omitempty"` +} + +// StageStat is one per-stage counter row in the sync report — the wire +// shape of the pipeline module's StageStats (cumulative counters since +// process start; the manager persists them as pipeline-stage-stats +// snapshots and the cluster fold takes each instance's latest tick). +type StageStat struct { + Pipeline string `json:"pipeline"` + InstanceID string `json:"instance_id"` + Stage string `json:"stage"` // processor name or lane name + Carrier string `json:"carrier"` // "processor" | "lane:" + Count int64 `json:"count"` + Errors int64 `json:"errors"` + AvgLatency int64 `json:"avg_latency_ns"` } type PipelineRuntimeState struct { diff --git a/modules/configs/server/server.go b/modules/configs/server/server.go index a7c0e4f0..1edcdf5b 100644 --- a/modules/configs/server/server.go +++ b/modules/configs/server/server.go @@ -44,6 +44,17 @@ import ( "infini.sh/framework/modules/configs/common" ) +// persistStageStatsSink stores an instance's reported per-stage counters. +// Registered by the pipeline module (which owns the pipeline-stage-stats +// schema); nil on apps that don't load it. +var persistStageStatsSink func(octx *orm.Context, rows []common.StageStat) + +// SetStageStatsSink installs the reported-stage-stats persistence hook +// (called from the pipeline module's Setup). +func SetStageStatsSink(fn func(octx *orm.Context, rows []common.StageStat)) { + persistStageStatsSink = fn +} + // Config holds the server-side settings (configs.server.* in YAML). // Disabled by default; products opt in via Setup(). type Config struct { @@ -576,6 +587,14 @@ func (h *APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, _ htt if updated { _, _ = upsertInstance(&obj.Client) // second upsert with runtime } + + // Stage stats: the instance's cumulative counters snapshot into + // the manager's store — fleet-wide cluster aggregation without + // every instance sharing the datastore. The pipeline module + // registers the sink; nil on apps without it. + if persistStageStatsSink != nil && len(obj.Runtime.StageStats) > 0 { + persistStageStatsSink(orm.NewContext().DirectAccess(), obj.Runtime.StageStats) + } } assigned, aerr := loadAssignedConfigs(obj.Client.ID) diff --git a/modules/pipeline/deployment_targeting_test.go b/modules/pipeline/deployment_targeting_test.go index 5c23a6b7..5634aca4 100644 --- a/modules/pipeline/deployment_targeting_test.go +++ b/modules/pipeline/deployment_targeting_test.go @@ -42,12 +42,14 @@ var ( deployORMOnce sync.Once ) -func newDeployTestEnv(t *testing.T) *deployTestEnv { - deployEnvSeq++ - sfx := fmt.Sprintf("%d", deployEnvSeq) - // ONE handler for the whole test binary: the ORM registry is a - // process-global, and a per-test DB would be closed under the next - // test's feet. Each env isolates via unique IDs instead. +// ensureDeployORMHandler installs the ONE sqlite handler for the whole +// test binary (orm.Register is a process-global last-wins registry — a +// second registration anywhere would flip later tests onto a handler +// missing their schemas). Every schema any test in this package needs +// registers here; side-effect-free so stats tests can call it without +// minting instances. +func ensureDeployORMHandler(t *testing.T) { + t.Helper() deployORMOnce.Do(func() { handler := &sqlite.SQLiteORM{Config: sqlite.SQLiteConfig{Enabled: true, DBPath: filepath.Join(os.TempDir(), fmt.Sprintf("pipeline-deploy-test-%d.db", time.Now().UnixNano()))}} @@ -59,10 +61,17 @@ func newDeployTestEnv(t *testing.T) *deployTestEnv { orm.MustRegisterSchemaWithIndexName(PipelineDeployment{}, "pipeline-deployments-test") orm.MustRegisterSchemaWithIndexName(model.Instance{}, "instance-test") orm.MustRegisterSchemaWithIndexName(server.ManagedConfig{}, "managed-configs-test") + orm.MustRegisterSchemaWithIndexName(PipelineStageStat{}, "pipeline-stage-stats-test") if err := orm.InitSchema(); err != nil { t.Fatal(err) } }) +} + +func newDeployTestEnv(t *testing.T) *deployTestEnv { + deployEnvSeq++ + sfx := fmt.Sprintf("%d", deployEnvSeq) + ensureDeployORMHandler(t) env := &deployTestEnv{ t: t, pid: "pipe-" + sfx, pipeName: "deploy-test-pipeline-" + sfx, diff --git a/modules/pipeline/metrics.go b/modules/pipeline/metrics.go index 49d5ac25..fd95ff5f 100644 --- a/modules/pipeline/metrics.go +++ b/modules/pipeline/metrics.go @@ -71,6 +71,17 @@ func (s *statsRecorder) OnStep(e *pipeline.StepEvent) { // stageStats snapshots one pipeline's processor stages plus every lane // carrier (depth from the live registry). func stageStats(pipelineName string) []StageStats { + out := processorStageStats(pipelineName) + out = append(out, laneStageStats()...) + return out +} + +// processorStageStats snapshots one pipeline's processor rows only — +// the per-pipeline slice of stageStats. Persistence and the sync report +// call this per pipeline and add lane rows once globally (stageStats' +// lane section is per-CALL, so folding it into a per-pipeline loop would +// duplicate every lane under every pipeline). +func processorStageStats(pipelineName string) []StageStats { self := global.Env().SystemConfig.NodeConfig.ID out := []StageStats{} if rec := globalStats.Load(); rec != nil { @@ -87,6 +98,14 @@ func stageStats(pipelineName string) []StageStats { } s.mu.Unlock() } + return out +} + +// laneStageStats snapshots the cross-pipeline lane carrier rows — lanes +// live in a global registry, they belong to no single pipeline. +func laneStageStats() []StageStats { + self := global.Env().SystemConfig.NodeConfig.ID + out := make([]StageStats, 0, len(lane.List())) for _, l := range lane.List() { out = append(out, StageStats{ InstanceID: self, diff --git a/modules/pipeline/metrics_persist.go b/modules/pipeline/metrics_persist.go index 0736e080..8107c156 100644 --- a/modules/pipeline/metrics_persist.go +++ b/modules/pipeline/metrics_persist.go @@ -13,8 +13,11 @@ import ( httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" "infini.sh/framework/core/env" + "infini.sh/framework/core/global" "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" "infini.sh/framework/core/util" + "infini.sh/framework/modules/configs/common" ) // PipelineStageStat is the persisted snapshot row of one stage at one @@ -41,9 +44,19 @@ type metricsPersistConfig struct { Persistence struct { Enabled bool `config:"enabled"` Interval int `config:"interval_seconds"` + // RetentionSeconds bounds the snapshot table: rows older than the + // cutoff are pruned on every persist pass. The cluster fold only + // reads each instance's LATEST tick, so history beyond a window is + // dead weight (and would eventually crowd the aggregation query). + // Default 3600; 0 also means the default; negative disables. + Retention int `config:"retention_seconds"` } `config:"persistence"` } +// stageStatsRetention is the resolved retention window (shared by the +// local exporter and the sync-report sink — one table, one policy). +var stageStatsRetention = time.Hour + // startMetricsPersist runs the snapshot exporter when enabled // (default interval 10s). Snapshots are cumulative counters; consumers // derive rates by differencing consecutive ticks. @@ -57,6 +70,11 @@ func (module *PipeModule) startMetricsPersist() { if interval <= 0 { interval = 10 * time.Second } + if cfg.Persistence.Retention > 0 { + stageStatsRetention = time.Duration(cfg.Persistence.Retention) * time.Second + } else if cfg.Persistence.Retention < 0 { + stageStatsRetention = 0 // disabled + } go func() { ticker := time.NewTicker(interval) defer ticker.Stop() @@ -73,35 +91,109 @@ func (module *PipeModule) startMetricsPersist() { log.Debugf("pipeline metrics persistence enabled (interval %v)", interval) } -func (module *PipeModule) persistStageSnapshot() { - names := []string{} +// reportedStageRows collects every running pipeline's processor counters +// plus the lane carriers once, in the shared wire shape (common.StageStat): +// rows are non-zero-collectable, tagged with the pipeline they belong to +// (Config.Name, falling back to the context key for id-less configs — +// same resolution rule as RuntimeSnapshot). Lane rows are cross-pipeline +// and carry an empty Pipeline tag. +func (module *PipeModule) reportedStageRows() []common.StageStat { + self := global.Env().SystemConfig.NodeConfig.ID + toRow := func(pipelineName string, st StageStats) common.StageStat { + return common.StageStat{ + Pipeline: pipelineName, InstanceID: st.InstanceID, Stage: st.Stage, Carrier: st.Carrier, + Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency, + } + } + out := []common.StageStat{} + seen := map[string]bool{} module.contexts.Range(func(k, v interface{}) bool { - if name, ok := k.(string); ok { - names = append(names, name) + ctx, ok := v.(*pipeline.Context) + if !ok { + return true + } + name := ctx.Config.Name + if name == "" { + if s, ok := k.(string); ok { + name = s + } + } + if name == "" || seen[name] { + return true + } + seen[name] = true + for _, st := range processorStageStats(name) { + out = append(out, toRow(name, st)) } return true }) + for _, st := range laneStageStats() { + out = append(out, toRow("", st)) + } + // self-tag rows whose recorder ran before node config resolved + for i := range out { + if out[i].InstanceID == "" { + out[i].InstanceID = self + } + } + return out +} + +// saveStageStatRows writes one snapshot tick of counters — shared by the +// local exporter and the configs _sync sink (same table, same row shape). +func saveStageStatRows(octx *orm.Context, rows []common.StageStat, now time.Time, src string) { + for _, st := range rows { + if st.Count == 0 && st.Errors == 0 { + continue + } + row := PipelineStageStat{ + InstanceID: st.InstanceID, Pipeline: st.Pipeline, Stage: st.Stage, Carrier: st.Carrier, + Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency, + Ts: now, + } + row.ID = util.GetUUID() + if err := orm.Save(octx, &row); err != nil { + log.Warnf("%s: save %s/%s failed: %v", src, st.Pipeline, st.Stage, err) + } + } +} + +// PersistReportedStageStats is the configs-server sink: an instance's +// _sync-reported counters snapshot into the manager's store so cluster +// aggregation covers instances that do not share its datastore. Same +// row shape as the local exporter (cumulative; the fold takes each +// instance's latest tick) and the same retention policy. +func PersistReportedStageStats(octx *orm.Context, rows []common.StageStat) { + if len(rows) == 0 { + return + } now := time.Now().UTC() + saveStageStatRows(octx, rows, now, "stage stats report") + pruneStageStats(now) +} + +func (module *PipeModule) persistStageSnapshot() { // DirectAccess: this is a background system writer (no request user); // the security orm hooks require either a session user or the direct // read/write flags — a bare context panics on the first save under // security-enabled apps (coco) and takes the process down octx := orm.NewContext().DirectAccess() - for _, name := range names { - for _, st := range stageStats(name) { - if st.Count == 0 && st.Errors == 0 { - continue - } - row := PipelineStageStat{ - InstanceID: st.InstanceID, Pipeline: name, Stage: st.Stage, Carrier: st.Carrier, - Count: st.Count, Errors: st.Errors, AvgLatency: st.AvgLatency, - Ts: now, - } - row.ID = util.GetUUID() - if err := orm.Save(octx, &row); err != nil { - log.Warnf("metrics persist: save %s/%s failed: %v", name, st.Stage, err) - } - } + now := time.Now().UTC() + saveStageStatRows(octx, module.reportedStageRows(), now, "metrics persist") + pruneStageStats(now) +} + +// pruneStageStats drops snapshot rows older than the retention window. +// No-op when retention is disabled. +func pruneStageStats(now time.Time) { + if stageStatsRetention <= 0 { + return + } + octx := orm.NewContext().DirectAccess() + orm.WithModel(octx, &PipelineStageStat{}) + qb := orm.NewQuery().Filter(orm.Range("ts").Lt(now.Add(-stageStatsRetention))) + if _, err := orm.DeleteByQuery(octx, qb); err != nil { + log.Debugf("stage stats prune: %v", err) } } diff --git a/modules/pipeline/metrics_report_test.go b/modules/pipeline/metrics_report_test.go new file mode 100644 index 00000000..1d481c48 --- /dev/null +++ b/modules/pipeline/metrics_report_test.go @@ -0,0 +1,105 @@ +/* Copyright © INFINI Ltd. All rights reserved. */ + +package pipeline + +import ( + "testing" + "time" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/modules/configs/common" +) + +// the package's single sqlite handler (see ensureDeployORMHandler) already +// carries the pipeline-stage-stats-test schema — reuse it, no second +// orm.Register (last-wins would strand the deploy tests' schemas) + +func TestReportedStageRows_TagByPipelineAndDedupNames(t *testing.T) { + installStats() + rec := globalStats.Load().(*statsRecorder) + rec.OnStep(&pipeline.StepEvent{Pipeline: "pA", Processor: "a1", Outcome: pipeline.OutcomeOK}) + rec.OnStep(&pipeline.StepEvent{Pipeline: "pA", Processor: "a2", Outcome: pipeline.OutcomeOK}) + rec.OnStep(&pipeline.StepEvent{Pipeline: "pB", Processor: "b1", Outcome: pipeline.OutcomeOK}) + + module := &PipeModule{} + // two context keys resolving to the SAME Config.Name — the seen-map + // must collect pA once (id-less configs key by name; duplicate keys + // under one name must not double the rows) + module.contexts.Store("pA", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pA"})) + module.contexts.Store("pA-id-alias", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pA"})) + module.contexts.Store("pB", pipeline.AcquireContext(pipeline.PipelineConfigV2{Name: "pB"})) + // a context whose Config.Name is empty falls back to the key + module.contexts.Store("pC", pipeline.AcquireContext(pipeline.PipelineConfigV2{})) + + rows := module.reportedStageRows() + type key struct{ p, s string } + got := map[key]int64{} + for _, r := range rows { + if r.Carrier != "processor" { + t.Fatalf("unexpected carrier in report: %+v", r) + } + got[key{r.Pipeline, r.Stage}]++ + } + want := map[key]int64{ + {"pA", "a1"}: 1, {"pA", "a2"}: 1, + {"pB", "b1"}: 1, + {"pC", "pC"}: 0, // placeholder replaced below; pC has no recorded steps + } + delete(want, key{"pC", "pC"}) + if len(got) != len(want) { + t.Fatalf("rows = %+v, want exactly %+v", rows, want) + } + for k, n := range want { + if got[k] != n { + t.Fatalf("row %v appears %d times, want %d (rows: %+v)", k, got[k], n, rows) + } + } +} + +func TestPersistReportedStageStats_SavesAndPrunes(t *testing.T) { + ensureDeployORMHandler(t) + prevRetention := stageStatsRetention + defer func() { stageStatsRetention = prevRetention }() + + octx := orm.NewContext().DirectAccess() + PersistReportedStageStats(octx, []common.StageStat{ + {Pipeline: "sp", InstanceID: "inst-1", Stage: "echo", Carrier: "processor", Count: 10, Errors: 0, AvgLatency: 1000}, + {Pipeline: "sp", InstanceID: "inst-1", Stage: "idle", Carrier: "processor", Count: 0, Errors: 0}, // zero row skipped + }) + + // a stale row beyond the retention window — direct save, Ts backdated + old := PipelineStageStat{ + InstanceID: "inst-0", Pipeline: "sp", Stage: "echo", Carrier: "processor", + Count: 1, Ts: time.Now().UTC().Add(-2 * time.Hour), + } + old.ID = "stats-old-tick" + if err := orm.Save(octx, &old); err != nil { + t.Fatal(err) + } + + countRows := func() int { + qctx := orm.NewContext().DirectAccess() + orm.WithModel(qctx, &PipelineStageStat{}) + res, err := orm.SearchV2(qctx, orm.NewQuery().Size(100)) + if err != nil { + t.Fatal(err) + } + rows, _, _ := elastic.DecodeHits[PipelineStageStat](res) + return len(rows) + } + + if n := countRows(); n != 2 { + t.Fatalf("after sink: %d rows, want 2 (fresh + backdated)", n) + } + + // 1h retention prunes the 2h-old row on the next persist pass + stageStatsRetention = time.Hour + PersistReportedStageStats(octx, []common.StageStat{ + {Pipeline: "sp", InstanceID: "inst-1", Stage: "echo", Carrier: "processor", Count: 12, Errors: 0, AvgLatency: 1000}, + }) + if n := countRows(); n != 2 { + t.Fatalf("after prune: %d rows, want 2 (two fresh ticks, backdated gone)", n) + } +} diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go index 316d2c68..ecbd00b3 100755 --- a/modules/pipeline/module.go +++ b/modules/pipeline/module.go @@ -48,6 +48,7 @@ import ( "infini.sh/framework/core/rate" "infini.sh/framework/core/util" configs_client "infini.sh/framework/modules/configs/client" + configs_server "infini.sh/framework/modules/configs/server" ) type PipeModule struct { @@ -202,6 +203,12 @@ func (module *PipeModule) Setup() { configs_client.SetInstanceFlagsProvider(func() map[string]interface{} { return map[string]interface{}{"cordoned": module.cordoned.Load()} }) + // per-stage counters ride the same heartbeat: instances without a + // shared datastore still land in the manager's cluster aggregation. + // The sink half runs only on apps that embed the configs server + // (coco/logpilot managers); on agent-only processes it is inert. + configs_client.SetStageStatsProvider(module.reportedStageRows) + configs_server.SetStageStatsSink(PersistReportedStageStats) // ORM schema registration (table/index auto-creation on startup) orm.MustRegisterSchemaWithIndexName(pipeline.PipelineConfigV2{}, "pipeline-configs") orm.MustRegisterSchemaWithIndexName(PipelineVersion{}, "pipeline-versions") From a7300c9b1f183ac8cff14f7396c7227041f4352e Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 25 Oct 2026 12:52:12 +0800 Subject: [PATCH 65/65] feat(pipeline): group-based deployment targeting via the placement selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The placement selector matched instance labels only, but fleets are organized by console-assigned groups (server-owned data instances never report as labels), so deployments could not target a group. The reserved `group` selector key now matches membership instead: values are comma-separated group names, any of which puts the instance in scope; other keys keep plain label equality. Labels and groups are separate namespaces — a label named "group" does not satisfy the group key. Membership is re-evaluated every reconcile cycle, so instances added to a group pick the deployment up automatically. Docs updated. --- docs/content.en/docs/references/pipeline.md | 2 +- modules/pipeline/deployment.go | 55 ++++++++++++++----- modules/pipeline/deployment_targeting_test.go | 26 ++++++++- 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/docs/content.en/docs/references/pipeline.md b/docs/content.en/docs/references/pipeline.md index 3bf11a16..157f5608 100644 --- a/docs/content.en/docs/references/pipeline.md +++ b/docs/content.en/docs/references/pipeline.md @@ -630,7 +630,7 @@ Every update auto-snapshots the previous config as a `PipelineVersion` — there | Field | Semantics | |-------|-----------| -| `selector` | Label match (AND across entries) — an instance is in scope when its labels carry every key=value. | +| `selector` | Label match (AND across entries) — an instance is in scope when its labels carry every key=value. The reserved `group` key matches console-assigned group membership instead: values are comma-separated group names, any of which puts the instance in scope. | | `instances` | Explicit instance IDs (union with the selector match; unknown IDs are ignored until they register). | | *(neither set)* | Broadcast: a single `*` row every instance pulls. | diff --git a/modules/pipeline/deployment.go b/modules/pipeline/deployment.go index 97814390..5ae4bc96 100644 --- a/modules/pipeline/deployment.go +++ b/modules/pipeline/deployment.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "time" log "github.com/cihub/seelog" @@ -240,8 +241,18 @@ func resolveTargets(dep *PipelineDeployment, instances []model.Instance) (map[st return targets, false } +// instanceMatchesLabels resolves one selector entry per instance: plain +// keys match Labels; the reserved `group` key matches console-assigned +// group membership instead (any of the comma-separated groups), since +// groups are server-owned data that instances never report as labels. func instanceMatchesLabels(inst *model.Instance, selector map[string]string) bool { for k, v := range selector { + if k == "group" { + if !instanceInAnyGroup(inst, v) { + return false + } + continue + } if inst.Labels[k] != v { return false } @@ -249,6 +260,24 @@ func instanceMatchesLabels(inst *model.Instance, selector map[string]string) boo return true } +// instanceInAnyGroup reports whether the instance belongs to any of the +// comma-separated group names. Empty segments are ignored so "a,, b" +// behaves like "a,b"; an empty list matches nothing (never broadcast). +func instanceInAnyGroup(inst *model.Instance, groupsCSV string) bool { + for _, g := range strings.Split(groupsCSV, ",") { + g = strings.TrimSpace(g) + if g == "" { + continue + } + for _, member := range inst.Groups { + if member == g { + return true + } + } + } + return false +} + // reconcilePipeline converges ONE pipeline's deployments into the EXACT // set of managed-config rows: one row per target instance (or a single // broadcast "*" row). Rows for instances that fell out of every scope are @@ -351,19 +380,19 @@ func (module *PipeModule) reconcilePipeline(octx *orm.Context, pipelineID string byID[instances[i].ID] = instances[i] } singleton := owner.Placement.Strategy == "singleton" - ready := func(id string) bool { - inst, ok := byID[id] - if !ok { - return false - } - // a frozen report must not count forever: an instance that - // died keeps its last STARTED+digest — require a heartbeat - // within 3 default sync intervals (30s each) or the wave - // holds until a live holder proves the content - if inst.Updated != nil && time.Since(*inst.Updated) > 90*time.Second { - return false - } - st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name] + ready := func(id string) bool { + inst, ok := byID[id] + if !ok { + return false + } + // a frozen report must not count forever: an instance that + // died keeps its last STARTED+digest — require a heartbeat + // within 3 default sync intervals (30s each) or the wave + // holds until a live holder proves the content + if inst.Updated != nil && time.Since(*inst.Updated) > 90*time.Second { + return false + } + st, reported := runtimePipelineStates(inst.RuntimePipelines)[cfg.Name] if !reported || st.State == "FAILED" { return false } diff --git a/modules/pipeline/deployment_targeting_test.go b/modules/pipeline/deployment_targeting_test.go index 5634aca4..4f840d55 100644 --- a/modules/pipeline/deployment_targeting_test.go +++ b/modules/pipeline/deployment_targeting_test.go @@ -38,7 +38,7 @@ type deployTestEnv struct { } var ( - deployEnvSeq int + deployEnvSeq int deployORMOnce sync.Once ) @@ -158,7 +158,7 @@ func (e *deployTestEnv) module() *PipeModule { return &PipeModule{} } func TestResolveTargetsModes(t *testing.T) { instances := []model.Instance{ - {ORMObjectBase: orm.ORMObjectBase{ID: "a"}, Labels: map[string]string{"env": "prod"}}, + {ORMObjectBase: orm.ORMObjectBase{ID: "a"}, Labels: map[string]string{"env": "prod"}}, {ORMObjectBase: orm.ORMObjectBase{ID: "b"}, Labels: map[string]string{"env": "staging"}}, } // no selector + no instances → broadcast @@ -171,6 +171,28 @@ func TestResolveTargetsModes(t *testing.T) { if broadcast || len(targets) != 1 || !targets["a"] { t.Fatalf("expected {a}, got %v broadcast=%v", targets, broadcast) } + // group membership: the reserved `group` selector key matches + // console-assigned groups, comma-separated values act as OR + grouped := []model.Instance{ + {ORMObjectBase: orm.ORMObjectBase{ID: "g1"}, Groups: []string{"edge", "cn"}}, + {ORMObjectBase: orm.ORMObjectBase{ID: "g2"}, Labels: map[string]string{"group": "edge"}}, + {ORMObjectBase: orm.ORMObjectBase{ID: "g3"}}, + } + targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge"}}}, grouped) + if len(targets) != 1 || !targets["g1"] { + t.Fatalf("group selector should match membership only, got %v", targets) + } + targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge,cn , missing"}}}, grouped) + if len(targets) != 1 || !targets["g1"] { + t.Fatalf("multi-group OR should still match only g1, got %v", targets) + } + // labels and groups are separate namespaces: a label named group + // cannot satisfy the group selector, and vice versa + targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"group": "edge", "env": "prod"}}}, grouped) + if len(targets) != 0 { + t.Fatalf("group + unrelated label must AND to nothing here, got %v", targets) + } + // multi-label AND targets, _ = resolveTargets(&PipelineDeployment{Placement: PlacementSpec{Selector: map[string]string{"env": "prod", "tier": "edge"}}}, instances) if len(targets) != 0 {