diff --git a/README.md b/README.md index bc83dd93c67..3cbdeb7b4ce 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,9 @@ To track upcoming milestones, visit the [project roadmap](https://github.com/flu - 📊 **Monitoring**: Expose internal metrics over HTTP/Prometheus - 🧩 **Extensibility**: - Write plugins in **C**, filters in **Lua**, and outputs in **Go** + - Write telemetry policies in Lua for logs, metrics, and traces with + [Fluent Typed Scripting (FTS)](plugins/processor_fts/README.md) + (`master` branch, not yet in a release) - 🔌 **Supports Logs, Metrics, and Traces** with unified processing and delivery --- diff --git a/documentation/wasm-browser.md b/documentation/wasm-browser.md index f8c38fb8729..73df1c3d251 100644 --- a/documentation/wasm-browser.md +++ b/documentation/wasm-browser.md @@ -597,6 +597,11 @@ preserves JSON nulls. Edit the script directly in the YAML `code: |` block. Traditional `pipeline.filters` Lua configuration is supported too, as are the three-argument and five-argument (metadata-aware) callbacks. +The same interpreter runs the `fts` processor (Fluent Typed Scripting) for logs, +metrics and traces; `jit: true` is rejected. See the +[FTS guide](../plugins/processor_fts/README.md#native-and-browser-wasm-runtimes) +for the differences from native builds. + The browser uses **portable Lua 5.4.9, not LuaJIT**. `FLB_WASM_LUA=ON` is the browser default; native builds still use `FLB_LUAJIT` and the existing bundled LuaJIT. The build downloads Lua from lua.org with a pinned SHA-256 into the diff --git a/examples/fts/README.md b/examples/fts/README.md new file mode 100644 index 00000000000..9096203dc31 --- /dev/null +++ b/examples/fts/README.md @@ -0,0 +1,89 @@ +# Fluent Typed Scripting (FTS) examples + +Runnable policies for the [`fts` processor](../../plugins/processor_fts/README.md). +The OpenTelemetry examples share one layout; the quickstart has only a +configuration with an inline policy and its expected output. + +| File | Content | +| ---------------------- | ------- | +| `fluent-bit.yaml` | Pipeline: `opentelemetry` input, `fts` processor, `stdout` output in `otlp_json` format. | +| `policy.lua` | The FTS policy. | +| `input.json` | OTLP/HTTP JSON request sent to the input. | +| `expected-output.json` | What Fluent Bit prints, as a JSON array with one entry per printed line. | + +| Example | Signal | Shows | +| ------- | ------ | ----- | +| [`quickstart`](quickstart/) | Logs | Inline policy on a `dummy` input. No other tools needed. | +| [`logs-redaction`](logs-redaction/) | Logs | Key and free-form redaction, dropping records, severity and resource enrichment. | +| [`metrics-normalization`](metrics-normalization/) | Metrics | Unit conversion and aggregation that reduces series cardinality. | +| [`traces-filtering`](traces-filtering/) | Traces | Attribute migration, query-string removal, event redaction and selective span drop. | + +## Requirements + +A Fluent Bit binary built with FTS. FTS is in the `master` branch and is not +part of any published release yet; see +[Availability](../../plugins/processor_fts/README.md#availability). To build it: + +```shell +cmake -S . -B build +cmake --build build -j8 +./build/bin/fluent-bit --help | grep fts +``` + +The OTLP examples also need `curl` and `python3`. + +## Run the quickstart + +```shell +cd examples/fts/quickstart +../../../build/bin/fluent-bit -c fluent-bit.yaml +``` + +Stop it with Ctrl-C. Expected output: + +```json +{"user":"ana","password":"[REDACTED]","path":"/login","status":200,"environment":"dev"} +``` + +## Run an OTLP example + +The helper starts Fluent Bit, posts `input.json`, prints the output and +compares it with `expected-output.json`: + +```shell +examples/fts/run.sh examples/fts/logs-redaction +``` + +It uses `build/bin/fluent-bit` unless you pass another binary as the second +argument or set `FLUENT_BIT_BINARY`. The examples listen on `127.0.0.1:4318`, +so that port must be free. + +To run an example by hand, start Fluent Bit from the example directory, because +relative `script` paths are resolved from the working directory: + +```shell +cd examples/fts/logs-redaction +../../../build/bin/fluent-bit -c fluent-bit.yaml +``` + +In a second terminal, send the input to the endpoint of the signal (`logs`, +`metrics` or `traces`): + +```shell +curl -H 'Content-Type: application/json' --data-binary @input.json \ + http://127.0.0.1:4318/v1/logs +``` + +Fluent Bit prints each processed chunk as one OTLP JSON line. + +## Reading the output + +- The `otlp_json` format of the `stdout` output, like the `opentelemetry` + output, uses the `log` or `message` key of a map body as the whole body and + omits the other keys. The logs example uses a `msg` key so that the complete + redacted body is visible. +- Metric families can be printed in a different order than they were sent: + the internal metrics representation groups families by type. This happens + with or without FTS. +- These examples are regression tested by + `tests/integration/scenarios/processor_fts/tests/test_fts_examples.py`. diff --git a/examples/fts/logs-redaction/README.md b/examples/fts/logs-redaction/README.md new file mode 100644 index 00000000000..ee46d8bdae2 --- /dev/null +++ b/examples/fts/logs-redaction/README.md @@ -0,0 +1,59 @@ +# FTS logs: redaction and enrichment + +[`policy.lua`](policy.lua) processes the OpenTelemetry logs of a `checkout` +service: + +1. Drops records whose `http.route` attribute is `/healthz`. +2. Replaces the values of `password`, `authorization`, `token` and `card` keys + anywhere in the body with `[REDACTED]`. +3. Masks e-mail addresses and `token=` values inside the free-form `msg` text. +4. Replaces the `user.email` attribute with `user.email_domain`. +5. Sets the OpenTelemetry severity from the body `level` field. +6. Adds `deployment.environment.name` to the resource and `policy.name` to + the log attributes. + +## Run + +```shell +examples/fts/run.sh examples/fts/logs-redaction +``` + +Or by hand, from this directory: + +```shell +../../../build/bin/fluent-bit -c fluent-bit.yaml +# in another terminal +curl -H 'Content-Type: application/json' --data-binary @input.json \ + http://127.0.0.1:4318/v1/logs +``` + +## Expected output + +The complete output is in [`expected-output.json`](expected-output.json). +[`input.json`](input.json) has three records; the health check is dropped and +the other two become: + +| Field | Input | Output | +| ----- | ----- | ------ | +| `user.password` | `hunter2` | `[REDACTED]` | +| `headers.Authorization` | `Bearer abc123` | `[REDACTED]` (matching ignores case) | +| `card` | map with `number` and `expiry` | `[REDACTED]` (the whole value is replaced) | +| `db_password` | `not-a-match` | unchanged (keys must match exactly) | +| `msg` | `login succeeded for ana@example.com` | `login succeeded for ` | +| `msg` | `payment failed token=tok_live_42` | `payment failed token=[REDACTED]` | +| `attempt` | `intValue` 3 | `intValue` 3 (integer type preserved) | +| attribute `user.email` | `ana@example.com` | removed; `user.email_domain` is `example.com` | +| severity | none | `severityText` `info`/`error`, `severityNumber` 9/17 | +| resource | `service.name` | adds `deployment.environment.name: production` | + +## Notes + +- `fb.redact` matches whole keys, so `db_password` is kept. List every key name + you need, or walk the map yourself when you need prefix or pattern matches. +- Key redaction does not read strings. Secrets inside text, such as the `msg` + field here, need explicit string rules like `scrub()`, and those rules only + catch the formats they describe. +- `fb.redact` only visits the value it is given. This policy redacts the body; + attributes, resource attributes and metadata need their own calls. +- The processor keeps the default `error_mode: drop`, so a record whose policy + fails is removed instead of forwarded unredacted. diff --git a/examples/fts/logs-redaction/expected-output.json b/examples/fts/logs-redaction/expected-output.json new file mode 100644 index 00000000000..b86a947d968 --- /dev/null +++ b/examples/fts/logs-redaction/expected-output.json @@ -0,0 +1,176 @@ +[ + { + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "production" + } + } + ] + }, + "scopeLogs": [ + { + "scope": { + "name": "checkout.http", + "version": "1.4.0" + }, + "logRecords": [ + { + "timeUnixNano": "1767225600000000000", + "observedTimeUnixNano": "1767225600000000000", + "severityNumber": 9, + "severityText": "info", + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "user.email_domain", + "value": { + "stringValue": "example.com" + } + }, + { + "key": "policy.name", + "value": { + "stringValue": "logs-redaction" + } + } + ], + "body": { + "kvlistValue": { + "values": [ + { + "key": "msg", + "value": { + "stringValue": "login succeeded for " + } + }, + { + "key": "level", + "value": { + "stringValue": "info" + } + }, + { + "key": "user", + "value": { + "kvlistValue": { + "values": [ + { + "key": "name", + "value": { + "stringValue": "ana" + } + }, + { + "key": "password", + "value": { + "stringValue": "[REDACTED]" + } + } + ] + } + } + }, + { + "key": "headers", + "value": { + "kvlistValue": { + "values": [ + { + "key": "Authorization", + "value": { + "stringValue": "[REDACTED]" + } + }, + { + "key": "x-request-id", + "value": { + "stringValue": "req-1" + } + } + ] + } + } + } + ] + } + } + }, + { + "timeUnixNano": "1767225602000000000", + "observedTimeUnixNano": "1767225602000000000", + "severityNumber": 17, + "severityText": "error", + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/pay" + } + }, + { + "key": "policy.name", + "value": { + "stringValue": "logs-redaction" + } + } + ], + "body": { + "kvlistValue": { + "values": [ + { + "key": "msg", + "value": { + "stringValue": "payment failed token=[REDACTED]" + } + }, + { + "key": "level", + "value": { + "stringValue": "error" + } + }, + { + "key": "attempt", + "value": { + "intValue": "3" + } + }, + { + "key": "card", + "value": { + "stringValue": "[REDACTED]" + } + }, + { + "key": "db_password", + "value": { + "stringValue": "not-a-match" + } + } + ] + } + } + } + ] + } + ] + } + ] + } +] diff --git a/examples/fts/logs-redaction/fluent-bit.yaml b/examples/fts/logs-redaction/fluent-bit.yaml new file mode 100644 index 00000000000..cdac113caf7 --- /dev/null +++ b/examples/fts/logs-redaction/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + +pipeline: + inputs: + - name: opentelemetry + listen: 127.0.0.1 + port: 4318 + processors: + logs: + - name: fts + script: policy.lua + + outputs: + - name: stdout + match: '*' + format: otlp_json diff --git a/examples/fts/logs-redaction/input.json b/examples/fts/logs-redaction/input.json new file mode 100644 index 00000000000..35854d63879 --- /dev/null +++ b/examples/fts/logs-redaction/input.json @@ -0,0 +1,77 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "checkout"}} + ] + }, + "scopeLogs": [ + { + "scope": {"name": "checkout.http", "version": "1.4.0"}, + "logRecords": [ + { + "timeUnixNano": "1767225600000000000", + "observedTimeUnixNano": "1767225600000000000", + "attributes": [ + {"key": "http.route", "value": {"stringValue": "/login"}}, + {"key": "user.email", "value": {"stringValue": "ana@example.com"}} + ], + "body": { + "kvlistValue": { + "values": [ + {"key": "msg", "value": {"stringValue": "login succeeded for ana@example.com"}}, + {"key": "level", "value": {"stringValue": "info"}}, + {"key": "user", "value": {"kvlistValue": {"values": [ + {"key": "name", "value": {"stringValue": "ana"}}, + {"key": "password", "value": {"stringValue": "hunter2"}} + ]}}}, + {"key": "headers", "value": {"kvlistValue": {"values": [ + {"key": "Authorization", "value": {"stringValue": "Bearer abc123"}}, + {"key": "x-request-id", "value": {"stringValue": "req-1"}} + ]}}} + ] + } + } + }, + { + "timeUnixNano": "1767225601000000000", + "observedTimeUnixNano": "1767225601000000000", + "attributes": [ + {"key": "http.route", "value": {"stringValue": "/healthz"}} + ], + "body": { + "kvlistValue": { + "values": [ + {"key": "msg", "value": {"stringValue": "ok"}} + ] + } + } + }, + { + "timeUnixNano": "1767225602000000000", + "observedTimeUnixNano": "1767225602000000000", + "attributes": [ + {"key": "http.route", "value": {"stringValue": "/pay"}} + ], + "body": { + "kvlistValue": { + "values": [ + {"key": "msg", "value": {"stringValue": "payment failed token=tok_live_42"}}, + {"key": "level", "value": {"stringValue": "error"}}, + {"key": "attempt", "value": {"intValue": "3"}}, + {"key": "card", "value": {"kvlistValue": {"values": [ + {"key": "number", "value": {"stringValue": "4111111111111111"}}, + {"key": "expiry", "value": {"stringValue": "12/30"}} + ]}}}, + {"key": "db_password", "value": {"stringValue": "not-a-match"}} + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/examples/fts/logs-redaction/policy.lua b/examples/fts/logs-redaction/policy.lua new file mode 100644 index 00000000000..7e652cc42df --- /dev/null +++ b/examples/fts/logs-redaction/policy.lua @@ -0,0 +1,48 @@ +-- Redact secrets and enrich OpenTelemetry logs. +local fb = require("fluentbit") + +-- Values of these keys are replaced at any depth of the body. Matching is +-- exact and case-insensitive: "Authorization" matches, "db_password" does not. +local SECRET_KEYS = {"password", "authorization", "token", "card"} + +local SEVERITIES = {debug = true, info = true, warn = true, error = true, fatal = true} + +-- Key redaction never looks inside strings, so free-form text needs its own rules. +local function scrub(text) + text = text:gsub("[%w%._%+%-]+@[%w%.%-]+", "") + text = text:gsub("token=%S+", "token=[REDACTED]") + return text +end + +function process(log) + if not fb.is_otel(log) then + return + end + + -- Health checks carry no information worth shipping. + if log.attributes:get_string("http.route") == "/healthz" then + return fb.DROP + end + + fb.redact(log.body, SECRET_KEYS) + + local text = log.body:get_string("msg") + if text then + log.body:set("msg", scrub(text)) + end + + -- Keep only the domain of the user address. + local email = log.attributes:get_string("user.email") + if email then + log.attributes:remove("user.email") + log.attributes:set("user.email_domain", email:match("@(.+)$") or "unknown") + end + + local level = log.body:get_string("level") + if level and SEVERITIES[level] then + fb.set_severity(log, level) + end + + log.resource.attributes:set("deployment.environment.name", "production") + log.attributes:set("policy.name", "logs-redaction") +end diff --git a/examples/fts/metrics-normalization/README.md b/examples/fts/metrics-normalization/README.md new file mode 100644 index 00000000000..e7ee20f2780 --- /dev/null +++ b/examples/fts/metrics-normalization/README.md @@ -0,0 +1,52 @@ +# FTS metrics: unit normalization and cardinality reduction + +[`policy.lua`](policy.lua) processes the OpenTelemetry metrics of a `checkout` +service: + +1. Drops families whose name starts with `debug.`. +2. Converts `ms` to `s` and `KiBy` to `By` with `fb.scale_metric`. +3. Renames `http.server.duration` to `http.server.request.duration`. +4. Aggregates `http.server.requests` by `http.route` and + `http.response.status_code`, removing the per-client `client.id` + attribute and summing the series that become identical. + +## Run + +```shell +examples/fts/run.sh examples/fts/metrics-normalization +``` + +Or by hand, from this directory: + +```shell +../../../build/bin/fluent-bit -c fluent-bit.yaml +# in another terminal +curl -H 'Content-Type: application/json' --data-binary @input.json \ + http://127.0.0.1:4318/v1/metrics +``` + +## Expected output + +The complete output is in [`expected-output.json`](expected-output.json). +[`input.json`](input.json) has four families: + +| Family | Input | Output | +| ------ | ----- | ------ | +| `http.server.requests` (sum) | 5 series: `/login 200` from `client-a` (10) and `client-b` (5), `/login 401` (2), `/pay 200` from `client-a` (7) and `client-d` (3) | 3 series without `client.id`: `/login 200` = 15, `/login 401` = 2, `/pay 200` = 10 | +| `http.server.duration` (histogram, `ms`) | bounds `10, 50, 250`; sum `850`; min `4`; max `240` | renamed to `http.server.request.duration`, unit `s`; bounds `0.01, 0.05, 0.25`; sum `0.85`; min `0.004`; max `0.24`; bucket counts unchanged | +| `process.memory.usage` (gauge, `KiBy`) | `asInt` 2048 | unit `By`, `asInt` 2097152 (an integer factor keeps the integer type) | +| `debug.queue.depth` | gauge | dropped | + +## Notes + +- `fb.aggregate_metric` combines only datapoints whose remaining attributes, + start time and time are all equal. Series with different timestamps stay + separate. +- For sums, the only aggregation is `sum`. Gauges also accept `min`, `max`, + `mean` and `count`. Exponential histograms and summaries cannot be + aggregated. +- Removing an attribute without aggregating can make two series identical, + which fails the policy. That is why this example aggregates instead of + calling `remove` on `client.id`. +- `fb.scale_metric` with a non-integer factor turns integer values into + doubles. Exponential histograms cannot be scaled. diff --git a/examples/fts/metrics-normalization/expected-output.json b/examples/fts/metrics-normalization/expected-output.json new file mode 100644 index 00000000000..4fab0233d7b --- /dev/null +++ b/examples/fts/metrics-normalization/expected-output.json @@ -0,0 +1,156 @@ +[ + { + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeMetrics": [ + { + "scope": { + "name": "checkout.http", + "version": "1.4.0", + "droppedAttributesCount": 0 + }, + "metrics": [ + { + "name": "http.server.requests", + "description": "Completed requests", + "unit": "{request}", + "sum": { + "dataPoints": [ + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "timeUnixNano": "1767225600000000000", + "startTimeUnixNano": "1767225540000000000", + "flags": 0, + "asInt": "15" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "401" + } + } + ], + "timeUnixNano": "1767225600000000000", + "startTimeUnixNano": "1767225540000000000", + "flags": 0, + "asInt": "2" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/pay" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "timeUnixNano": "1767225600000000000", + "startTimeUnixNano": "1767225540000000000", + "flags": 0, + "asInt": "10" + } + ], + "aggregationTemporality": 2, + "isMonotonic": true + } + }, + { + "name": "process.memory.usage", + "description": "Resident memory", + "unit": "By", + "gauge": { + "dataPoints": [ + { + "timeUnixNano": "1767225600000000000", + "startTimeUnixNano": "0", + "flags": 0, + "asInt": "2097152" + } + ] + } + }, + { + "name": "http.server.request.duration", + "description": "Request duration", + "unit": "s", + "histogram": { + "dataPoints": [ + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + } + ], + "timeUnixNano": "1767225600000000000", + "startTimeUnixNano": "1767225540000000000", + "flags": 0, + "count": "17", + "sum": 0.85, + "min": 0.004, + "max": 0.24, + "bucketCounts": [ + "3", + "9", + "5", + "0" + ], + "explicitBounds": [ + 0.01, + 0.05, + 0.25 + ] + } + ], + "aggregationTemporality": 2 + } + } + ], + "schemaUrl": "" + } + ], + "schemaUrl": "" + } + ] + } +] diff --git a/examples/fts/metrics-normalization/fluent-bit.yaml b/examples/fts/metrics-normalization/fluent-bit.yaml new file mode 100644 index 00000000000..3184d6aa925 --- /dev/null +++ b/examples/fts/metrics-normalization/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + +pipeline: + inputs: + - name: opentelemetry + listen: 127.0.0.1 + port: 4318 + processors: + metrics: + - name: fts + script: policy.lua + + outputs: + - name: stdout + match: '*' + format: otlp_json diff --git a/examples/fts/metrics-normalization/input.json b/examples/fts/metrics-normalization/input.json new file mode 100644 index 00000000000..cf9095a2ffa --- /dev/null +++ b/examples/fts/metrics-normalization/input.json @@ -0,0 +1,226 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + } + ] + }, + "scopeMetrics": [ + { + "scope": { + "name": "checkout.http", + "version": "1.4.0" + }, + "metrics": [ + { + "name": "http.server.requests", + "unit": "{request}", + "description": "Completed requests", + "sum": { + "aggregationTemporality": 2, + "isMonotonic": true, + "dataPoints": [ + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "client.id", + "value": { + "stringValue": "client-a" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "asInt": "10" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "client.id", + "value": { + "stringValue": "client-b" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "asInt": "5" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "401" + } + }, + { + "key": "client.id", + "value": { + "stringValue": "client-c" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "asInt": "2" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/pay" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "client.id", + "value": { + "stringValue": "client-a" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "asInt": "7" + }, + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/pay" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "client.id", + "value": { + "stringValue": "client-d" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "asInt": "3" + } + ] + } + }, + { + "name": "http.server.duration", + "unit": "ms", + "description": "Request duration", + "histogram": { + "aggregationTemporality": 2, + "dataPoints": [ + { + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/login" + } + } + ], + "startTimeUnixNano": "1767225540000000000", + "timeUnixNano": "1767225600000000000", + "count": "17", + "sum": 850.0, + "min": 4.0, + "max": 240.0, + "explicitBounds": [ + 10.0, + 50.0, + 250.0 + ], + "bucketCounts": [ + "3", + "9", + "5", + "0" + ] + } + ] + } + }, + { + "name": "process.memory.usage", + "unit": "KiBy", + "description": "Resident memory", + "gauge": { + "dataPoints": [ + { + "attributes": [], + "timeUnixNano": "1767225600000000000", + "asInt": "2048" + } + ] + } + }, + { + "name": "debug.queue.depth", + "unit": "1", + "gauge": { + "dataPoints": [ + { + "attributes": [], + "timeUnixNano": "1767225600000000000", + "asInt": "4" + } + ] + } + } + ] + } + ] + } + ] +} diff --git a/examples/fts/metrics-normalization/policy.lua b/examples/fts/metrics-normalization/policy.lua new file mode 100644 index 00000000000..a7ebeaf529d --- /dev/null +++ b/examples/fts/metrics-normalization/policy.lua @@ -0,0 +1,29 @@ +-- Normalize metric units and reduce series cardinality. +local fb = require("fluentbit") + +-- Attributes that identify a request series. Anything else, such as a +-- per-client identifier, is removed and its series are summed together. +local SERIES_KEYS = {"http.route", "http.response.status_code"} + +function process(metric) + -- Internal debugging families never leave the host. + if metric.name:sub(1, 6) == "debug." then + return fb.DROP + end + + -- Convert to the base units recommended by OpenTelemetry. scale_metric + -- updates values, sums, histogram bounds and exemplars together. + if metric.unit == "ms" then + fb.scale_metric(metric, 0.001, "s") + elseif metric.unit == "KiBy" then + fb.scale_metric(metric, 1024, "By") + end + + if metric.name == "http.server.duration" then + metric.name = "http.server.request.duration" + end + + if metric.name == "http.server.requests" then + fb.aggregate_metric(metric, SERIES_KEYS, "sum") + end +end diff --git a/examples/fts/quickstart/README.md b/examples/fts/quickstart/README.md new file mode 100644 index 00000000000..7c8d2c4f546 --- /dev/null +++ b/examples/fts/quickstart/README.md @@ -0,0 +1,18 @@ +# FTS quickstart + +A `dummy` input generates one record and an inline FTS policy redacts its +`password` field and adds an `environment` field. No other tools are needed. + +```shell +../../../build/bin/fluent-bit -c fluent-bit.yaml +``` + +Stop it with Ctrl-C. Expected output, also in +[`expected-output.json`](expected-output.json): + +```json +{"user":"ana","password":"[REDACTED]","path":"/login","status":200,"environment":"dev"} +``` + +`status` is still the integer `200`: FTS only changes the values a policy +writes. diff --git a/examples/fts/quickstart/expected-output.json b/examples/fts/quickstart/expected-output.json new file mode 100644 index 00000000000..1288e5d1bb6 --- /dev/null +++ b/examples/fts/quickstart/expected-output.json @@ -0,0 +1,3 @@ +[ + {"user": "ana", "password": "[REDACTED]", "path": "/login", "status": 200, "environment": "dev"} +] diff --git a/examples/fts/quickstart/fluent-bit.yaml b/examples/fts/quickstart/fluent-bit.yaml new file mode 100644 index 00000000000..b8271e7858b --- /dev/null +++ b/examples/fts/quickstart/fluent-bit.yaml @@ -0,0 +1,25 @@ +pipeline: + inputs: + - name: dummy + dummy: '{"user": "ana", "password": "hunter2", "path": "/login", "status": 200}' + samples: 1 + processors: + logs: + - name: fts + code: | + local fb = require("fluentbit") + + function process(log) + if log.body:get_string("path") == "/healthz" then + return fb.DROP + end + + fb.redact(log.body, {"password"}) + log.body:set("environment", "dev") + end + + outputs: + - name: stdout + match: '*' + format: json_lines + json_date_key: false diff --git a/examples/fts/run.sh b/examples/fts/run.sh new file mode 100755 index 00000000000..e32f9fd5c97 --- /dev/null +++ b/examples/fts/run.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Run one FTS example: start Fluent Bit with the example configuration, send +# input.json to the OpenTelemetry input, print the processed output and +# compare it with expected-output.json. +# +# usage: examples/fts/run.sh [path-to-fluent-bit] + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 [path-to-fluent-bit]" >&2 + exit 2 +fi + +example=$(cd "$1" && pwd) +root=$(cd "$(dirname "$0")/../.." && pwd) +fluent_bit=${2:-${FLUENT_BIT_BINARY:-$root/build/bin/fluent-bit}} +port=4318 + +case "$(head -c 200 "$example/input.json")" in + *resourceLogs*) signal=logs ;; + *resourceMetrics*) signal=metrics ;; + *resourceSpans*) signal=traces ;; + *) echo "cannot detect the signal of $example/input.json" >&2; exit 2 ;; +esac + +work=$(mktemp -d) +trap 'kill "$pid" 2>/dev/null || true; rm -rf "$work"' EXIT + +# Relative script paths are resolved from the working directory. +(cd "$example" && exec "$fluent_bit" -c fluent-bit.yaml) >"$work/stdout" 2>"$work/stderr" & +pid=$! + +for _ in $(seq 1 50); do + if curl -s -o /dev/null "http://127.0.0.1:$port/"; then + break + fi + sleep 0.2 +done + +curl -sS --fail -H 'Content-Type: application/json' --data-binary "@$example/input.json" \ + "http://127.0.0.1:$port/v1/$signal" >/dev/null + +# The pipeline flushes every second; give it time to write the output. +for _ in $(seq 1 25); do + if grep -q '^{' "$work/stdout"; then + break + fi + sleep 0.2 +done +sleep 1 +kill -INT "$pid" +wait "$pid" || true + +grep '^{' "$work/stdout" | python3 -c ' +import json, sys +print(json.dumps([json.loads(line) for line in sys.stdin], indent=2)) +' >"$work/output.json" + +cat "$work/output.json" + +if diff -u "$example/expected-output.json" "$work/output.json" >&2; then + echo "OK: output matches expected-output.json" >&2 +else + echo "FAIL: output differs from expected-output.json" >&2 + cat "$work/stderr" >&2 + exit 1 +fi diff --git a/examples/fts/traces-filtering/README.md b/examples/fts/traces-filtering/README.md new file mode 100644 index 00000000000..dfb80c2a0a6 --- /dev/null +++ b/examples/fts/traces-filtering/README.md @@ -0,0 +1,49 @@ +# FTS traces: attribute transformation and selective span drop + +[`policy.lua`](policy.lua) processes the OpenTelemetry spans of a `checkout` +service: + +1. Drops spans whose `http.route` attribute is `/healthz`. +2. Drops `cache.lookup` spans whose `cache.hit` attribute is `true`. +3. Renames `http.method` to `http.request.method` and `http.status_code` to + `http.response.status_code`, keeping the original value types. +4. Replaces `http.target` with `url.path`, removing the query string. +5. Removes the `enduser.id` attribute. +6. Redacts `authorization` attributes inside span events. + +## Run + +```shell +examples/fts/run.sh examples/fts/traces-filtering +``` + +Or by hand, from this directory: + +```shell +../../../build/bin/fluent-bit -c fluent-bit.yaml +# in another terminal +curl -H 'Content-Type: application/json' --data-binary @input.json \ + http://127.0.0.1:4318/v1/traces +``` + +## Expected output + +The complete output is in [`expected-output.json`](expected-output.json). +[`input.json`](input.json) has four spans in two traces: + +| Span | Output | +| ---- | ------ | +| `GET /checkout` (root) | Kept. Attributes become `http.route`, `http.request.method: GET`, `http.response.status_code: 200` (still `intValue`) and `url.path: /checkout`. The `request.headers` event keeps `user-agent` and has `authorization: [REDACTED]`. | +| `cache.lookup` (child, cache hit) | Dropped. | +| `SELECT orders` (child) | Kept unchanged, with its `parentSpanId` pointing at the root span. | +| `GET /healthz` (root of a second trace) | Dropped. | + +## Notes + +- FTS processes one span at a time and does not repair trace topology. Dropping + a span that has children leaves them pointing at a parent that no longer + exists. This policy only drops leaf spans and single-span traces. +- FTS does not make sampling decisions across a trace. For tail sampling, use + the `sampling` processor. +- Span timestamps are exact 64-bit integers. They are larger than 2^53, so + `:number()` on them fails; compare or copy them as typed values. diff --git a/examples/fts/traces-filtering/expected-output.json b/examples/fts/traces-filtering/expected-output.json new file mode 100644 index 00000000000..ffe149d219c --- /dev/null +++ b/examples/fts/traces-filtering/expected-output.json @@ -0,0 +1,115 @@ +[ + { + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "checkout.http", + "version": "1.4.0" + }, + "spans": [ + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b174", + "name": "GET /checkout", + "kind": 2, + "startTimeUnixNano": "1767225600000000000", + "endTimeUnixNano": "1767225600250000000", + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/checkout" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "url.path", + "value": { + "stringValue": "/checkout" + } + } + ], + "events": [ + { + "name": "request.headers", + "timeUnixNano": "1767225600010000000", + "attributes": [ + { + "key": "authorization", + "value": { + "stringValue": "[REDACTED]" + } + }, + { + "key": "user-agent", + "value": { + "stringValue": "curl/8.5" + } + } + ] + } + ], + "status": { + "code": 0, + "message": "" + } + }, + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b176", + "name": "SELECT orders", + "parentSpanId": "eee19b7ec3c1b174", + "kind": 3, + "startTimeUnixNano": "1767225600030000000", + "endTimeUnixNano": "1767225600200000000", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": 0, + "message": "" + } + } + ], + "schemaUrl": "" + } + ], + "schemaUrl": "" + } + ] + } +] diff --git a/examples/fts/traces-filtering/fluent-bit.yaml b/examples/fts/traces-filtering/fluent-bit.yaml new file mode 100644 index 00000000000..d5a618d1ea6 --- /dev/null +++ b/examples/fts/traces-filtering/fluent-bit.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + +pipeline: + inputs: + - name: opentelemetry + listen: 127.0.0.1 + port: 4318 + processors: + traces: + - name: fts + script: policy.lua + + outputs: + - name: stdout + match: '*' + format: otlp_json diff --git a/examples/fts/traces-filtering/input.json b/examples/fts/traces-filtering/input.json new file mode 100644 index 00000000000..c3cc055e2e1 --- /dev/null +++ b/examples/fts/traces-filtering/input.json @@ -0,0 +1,160 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "checkout.http", + "version": "1.4.0" + }, + "spans": [ + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b174", + "name": "GET /checkout", + "kind": 2, + "startTimeUnixNano": "1767225600000000000", + "endTimeUnixNano": "1767225600250000000", + "attributes": [ + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/checkout" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/checkout?session=s3cr3t&step=2" + } + }, + { + "key": "enduser.id", + "value": { + "stringValue": "ana" + } + } + ], + "status": { + "code": 0 + }, + "events": [ + { + "timeUnixNano": "1767225600010000000", + "name": "request.headers", + "attributes": [ + { + "key": "authorization", + "value": { + "stringValue": "Bearer abc123" + } + }, + { + "key": "user-agent", + "value": { + "stringValue": "curl/8.5" + } + } + ] + } + ] + }, + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b175", + "name": "cache.lookup", + "kind": 1, + "startTimeUnixNano": "1767225600020000000", + "endTimeUnixNano": "1767225600020500000", + "attributes": [ + { + "key": "cache.hit", + "value": { + "boolValue": true + } + } + ], + "status": { + "code": 0 + }, + "parentSpanId": "eee19b7ec3c1b174" + }, + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b176", + "name": "SELECT orders", + "kind": 3, + "startTimeUnixNano": "1767225600030000000", + "endTimeUnixNano": "1767225600200000000", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": 0 + }, + "parentSpanId": "eee19b7ec3c1b174" + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "b7ad6b7169203331", + "name": "GET /healthz", + "kind": 2, + "startTimeUnixNano": "1767225601000000000", + "endTimeUnixNano": "1767225601001000000", + "attributes": [ + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/healthz" + } + } + ], + "status": { + "code": 0 + } + } + ] + } + ] + } + ] +} diff --git a/examples/fts/traces-filtering/policy.lua b/examples/fts/traces-filtering/policy.lua new file mode 100644 index 00000000000..369e6fceffe --- /dev/null +++ b/examples/fts/traces-filtering/policy.lua @@ -0,0 +1,41 @@ +-- Migrate span attributes and drop low-value spans. +local fb = require("fluentbit") + +-- Older HTTP semantic convention names and their current replacements. An +-- array keeps the rewrite order, and so the output, deterministic. +local RENAMES = { + {"http.method", "http.request.method"}, + {"http.status_code", "http.response.status_code"}, +} + +function process(span) + local attrs = span.attributes + + -- Health-check traces and cache hits add volume without insight. Both are + -- leaf spans here; dropping a parent would leave its children orphaned. + if attrs:get_string("http.route") == "/healthz" then + return fb.DROP + end + if span.name == "cache.lookup" and attrs:get_bool("cache.hit") then + return fb.DROP + end + + -- Moving the typed value keeps integers as integers. + for _, rename in ipairs(RENAMES) do + local value = attrs:get(rename[1]) + if value then + attrs:set(rename[2], value) + attrs:remove(rename[1]) + end + end + + -- Query strings can carry tokens: keep only the path. + local target = attrs:get_string("http.target") + if target then + attrs:remove("http.target") + attrs:set("url.path", (target:gsub("%?.*$", ""))) + end + + attrs:remove("enduser.id") + fb.redact(span.events, {"authorization"}) +end diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md index 28842a56351..4c424eafe41 100644 --- a/plugins/processor_fts/README.md +++ b/plugins/processor_fts/README.md @@ -1,16 +1,15 @@ # Fluent Typed Scripting (FTS) processor -FTS is a processor for writing telemetry policies in Lua. Policies get typed -access to the record and its context: integer widths, binary values, and -nanosecond timestamps are preserved, and OpenTelemetry resources, scopes, and -attributes are exposed as properties instead of raw metadata. +Fluent Typed Scripting (FTS) runs telemetry policies written in Lua, with typed +access to logs, metrics, traces, and OpenTelemetry context. Its configuration +name is `fts`. -FTS supports logs, metrics, and traces. Its configuration name is `fts`. - -The experimental browser WASM profile includes FTS when `FLB_WASM_LUA=ON` -(the default). It uses the portable Lua 5.4 interpreter; `jit: true` is rejected. -Typed values, restricted policy libraries and instruction/memory limits remain -available. Setting `FLB_WASM_LUA=OFF` disables both FTS and the Lua filter. +"Typed" describes how FTS handles telemetry data, not the Lua language. Lua +stays dynamically typed and policies are not type checked. What FTS keeps is +the type of every telemetry value: 64-bit signed and unsigned integers, doubles, +binary data, nulls, maps and arrays keep their exact type and value unless the +policy changes them. OpenTelemetry resources, scopes, and attributes are +properties of the item instead of raw metadata that the policy has to decode. | Signal | Callback | Called once per | | ------- | ----------------- | --------------- | @@ -18,41 +17,146 @@ available. Setting `FLB_WASM_LUA=OFF` disables both FTS and the Lua filter. | Metrics | `process(metric)` | metric family | | Traces | `process(span)` | span | +- [Quick start](#quick-start) +- [Examples](#examples) +- [Availability](#availability) +- [FTS or the Lua filter](#fts-or-the-lua-filter) +- [Configuration](#configuration) +- [Callback results and errors](#callback-results-and-errors) +- [Typed values](#typed-values) +- [Logs](#logs), [Metrics](#metrics), [Traces](#traces) +- [Common helpers](#common-helpers) and [redaction limits](#redaction-limits) +- [Limits and sandbox](#limits-and-sandbox) +- [Diagnostics](#diagnostics) + ## Quick start +Processors are configured in YAML. Save this as `fluent-bit.yaml`: + ```yaml pipeline: inputs: - - name: opentelemetry - port: 4318 + - name: dummy + dummy: '{"user": "ana", "password": "hunter2", "path": "/login", "status": 200}' + samples: 1 processors: logs: - name: fts - script: policy.lua + code: | + local fb = require("fluentbit") + + function process(log) + if log.body:get_string("path") == "/healthz" then + return fb.DROP + end + + fb.redact(log.body, {"password"}) + log.body:set("environment", "dev") + end + outputs: - name: stdout match: '*' - format: otlp_json + format: json_lines + json_date_key: false ``` -```lua -local fb = require("fluentbit") +Run it and stop it with Ctrl-C: -function process(log) - if log.body:get_string("path") == "/health" then - return fb.DROP - end - - fb.redact(log.body, {"authorization", "password", "access_token"}) +```shell +fluent-bit -c fluent-bit.yaml +``` - if fb.is_otel(log) then - log.resource.attributes:set("deployment.environment.name", "production") - log.attributes:set("sanitized", true) - end -end +```json +{"user":"ana","password":"[REDACTED]","path":"/login","status":200,"environment":"dev"} ``` -Use one processor entry and one script for each signal. +For longer policies, put the code in a file and set `script` instead of `code`. +Use one processor entry and one policy for each signal. + +## Examples + +[`examples/fts`](../../examples/fts/README.md) has complete examples with +configuration, policy, input, run command, and expected output: + +- [Logs](../../examples/fts/logs-redaction/README.md): redaction of sensitive + fields and free-form text, dropping records, severity and resource + enrichment. +- [Metrics](../../examples/fts/metrics-normalization/README.md): unit + normalization and aggregation that reduces series cardinality. +- [Traces](../../examples/fts/traces-filtering/README.md): attribute + transformation and selective span drop. + +## Availability + +FTS is in the `master` branch, which currently builds as Fluent Bit 5.2.0. It +is not part of any published release: the latest release, v5.1.5, and the +`5.1` branch do not include it. Release packages and container images include +FTS only from the first release that contains it. Until then, build from +source. + +Run `fluent-bit --help` and look for `fts` in the processor list to check +whether a binary includes it. + +### Build options + +| Option | Default | Requirement | +| ------------------- | ------- | ----------- | +| `FLB_PROCESSOR_FTS` | `ON` | Builds the processor. | +| `FLB_LUAJIT` | `ON` | Required for native builds. FTS is not built without it. | +| `FLB_YYJSON` | `ON` | Required. | + +The build turns `FLB_LUAJIT` off on riscv64, and on s390x unless +`FLB_PREFER_SYSTEM_LIB_LUAJIT` is enabled with a system LuaJIT. FTS is not +available on those builds. + +### Native and browser WASM runtimes + +Native builds run policies on the bundled LuaJIT 2.1. The JIT compiler is off +unless `jit: true` is set. + +The experimental browser WASM profile (`FLB_WASM_BROWSER=ON`) also includes +FTS, through the portable Lua 5.4 interpreter enabled by `FLB_WASM_LUA=ON`, the +browser default. Differences from native builds: + +- `jit: true` is rejected. +- Lua 5.4 language and library behavior applies instead of LuaJIT's Lua 5.1. +- Typed values, the restricted policy environment, and instruction and memory + limits work as in native builds. +- `FLB_WASM_LUA=OFF` removes both FTS and the Lua filter. + +The browser profile as a whole is experimental; see +[the browser WASM guide](../../documentation/wasm-browser.md). + +## FTS or the Lua filter + +Both run Lua. They differ in what they process, how they see values, and what +happens when a script fails. + +| | FTS processor (`fts`) | Lua filter (`lua`) | +| ----------------------- | --------------------- | ------------------ | +| Signals | Logs, metrics, traces | Logs | +| Where it runs | Input and output `processors` | `pipeline.filters`, or `processors.logs` | +| Values | Typed: int64, uint64, double, bytes, null, maps and arrays are kept exactly | Converted to Lua tables and numbers | +| OpenTelemetry context | `resource`, `scope`, `attributes` and log fields as properties | Group attributes and record metadata as plain tables (five-argument callback) | +| Failed script | The item's edits are rolled back; `error_mode` drops it (default) or forwards the original | The error is logged and the whole chunk passes unmodified | +| Resource limits | Instruction, memory and output limits | None | +| Lua environment | Restricted: no `io`, `os`, `require` of other modules, `print` | Full standard library, `require` of modules | +| Records per call | One item in, kept or dropped. Metrics can emit derived families | Can return several records for one input | +| Configuration format | YAML | YAML and classic | + +For example, with the record +`{"big": 18446744073709551615, "id": 9007199254740993, "n": null}`, a Lua +filter that only changes another field outputs `"big": -9223372036854775808` +and `"id": 9007199254740992`, and drops `"n"`. FTS forwards all three +unchanged. + +Choose FTS for metrics and traces, for policies that must not change values +they do not touch, such as IDs, counters and nulls, for OpenTelemetry context, +and where a failure must not forward unprocessed data. Choose the Lua filter +when a script needs the standard library or Lua modules, such as file access or +`os.time`, needs to split one record into several, or must run in a classic +configuration file. ## Configuration @@ -63,32 +167,53 @@ Use one processor entry and one script for each signal. | `call` | `process` | Name of the callback function. | | `error_mode` | `drop` | What happens to an item when its policy fails: `drop` removes it, `keep` forwards the original. | | `instruction_limit` | `1000000` | Maximum Lua instructions per callback and during script load. Minimum `1000`. `0` is allowed only with `jit: true`. | -| `jit` | `false` | Enable the LuaJIT compiler. Requires `instruction_limit: 0`. | +| `jit` | `false` | Enable the LuaJIT compiler. Requires `instruction_limit: 0`. Not available in browser WASM builds. | | `memory_limit` | `32M` | Maximum memory for the policy: the Lua heap plus the typed values it holds. Minimum `262144`. | | `output_limit` | `64M` | Maximum serialized size of one processed batch. Minimum `65536`. | Size suffixes are decimal: `64K` is 64000 bytes. -## Callback results +A relative `script` path is resolved from the working directory of the Fluent +Bit process, not from the directory of the configuration file. Use absolute +paths in services. + +The policy file is loaded once, when the processor starts. Fluent Bit does not +start if the policy fails to load, the callback function does not exist, or a +setting is invalid. -| Return value | Effect | +## Callback results and errors + +| Callback result | Effect | | -------------------- | ------ | | `nil` or `fb.KEEP` | Commit all changes made by the callback. | | `fb.DROP` | Remove the item. | | Lua error | Roll back every change made by the callback, then apply `error_mode`. | +| Any other value | Treated as an error: `callback must return nil, fb.KEEP or fb.DROP`. | Changes are transactional per item. Edits to the body, metadata, timestamp, -resource, and scope are committed or rolled back together. +resource, and scope are committed or rolled back together; a failed item is +never forwarded half-modified. Exceeding the instruction or memory limit, and +writing a value that is invalid for the signal, are errors like any other. + +`error_mode` decides what happens to an item whose policy failed: + +- `drop` (default) removes it. +- `keep` forwards the original, unmodified item. + +For privacy policies such as redaction, keep `error_mode: drop`. With `keep`, +an item whose policy fails is forwarded without redaction. + +Errors outside a single item apply `error_mode` to a whole batch: -For logs, if a processed batch exceeds `output_limit`, the whole batch is -treated as failed: `drop` discards it and `keep` forwards the original batch. -For metrics and traces the limit is checked per item: an item whose result does -not fit follows `error_mode` (`drop` removes it, `keep` forwards the original -item), and the whole batch fails only when the kept originals do not fit -either. Internal failures, such as a batch that cannot be decoded, apply -`error_mode` to the whole batch. +- For logs, if a processed batch exceeds `output_limit`, `drop` discards it and + `keep` forwards the original batch. +- For metrics and traces the limit is checked per item: an item whose result + does not fit follows `error_mode`, and the whole batch fails only when the + kept originals do not fit either. +- Internal failures, such as a batch that cannot be decoded, apply `error_mode` + to the whole batch. -A batch is the set of records handed to one callback run. Small chunks are +A batch is the set of records handed to one processor run. Small chunks are processed as one batch. Larger buffers, such as complete chunks reaching output processors, are processed in batches of about 64 KiB or 1024 records, so decoded records never hold more than one batch in memory. Groups, such as one @@ -100,9 +225,6 @@ stays open for the records that follow. Records that arrive before the end marker of a group opened earlier are processed without a context, keep their place in that group, and cannot set one. -For privacy policies such as redaction, keep the default `error_mode: drop`. -With `keep`, an item whose policy fails is forwarded unmodified. - ## Typed values Bodies, attributes, and other structured fields are typed maps and arrays. @@ -126,7 +248,12 @@ Bodies, attributes, and other structured fields are typed maps and arrays. | `v:number()` | numeric | Lua number. Fails when an integer is beyond ±(2^53 − 1). | | `v:copy()` | any | Independent, writable copy. | -Keys match exact bytes; a dot in a key has no special meaning. +Keys match exact bytes; a dot in a key has no special meaning, so +`get("http.route")` reads the key `http.route`, not a nested path. + +Passing a value obtained with `get` to `set` keeps its type. Nanosecond +timestamps are larger than 2^53, so `:number()` fails on them; copy or compare +them as typed values, or use `:value()` for the exact decimal string. ### Constructors @@ -140,9 +267,9 @@ Keys match exact bytes; a dot in a key has no special meaning. | `fb.null` | Null value | Lua strings, booleans, and numbers can be passed to `set` and `append` -directly. Lua numbers are stored as doubles; use `fb.int64` or `fb.uint64` for -integers. Lua tables and `nil` are not values: use `fb.map()`, `fb.array()`, -and `fb.null`. +directly. Lua numbers are always stored as doubles, in both runtimes: `set("n", +3)` writes `3.0`. Use `fb.int64` or `fb.uint64` for integers. Lua tables and +`nil` are not values: use `fb.map()`, `fb.array()`, and `fb.null`. ### Value lifetime @@ -153,7 +280,8 @@ changing them never affects later items or committed output, and keeping them retains their memory. To keep data between callbacks, store `v:copy()`. Copies are independent: changing them never changes a committed record. -Lua globals persist for the lifetime of the processor instance. +Lua globals persist for the lifetime of the processor instance. Each processor +instance has its own Lua state. ## Logs @@ -169,9 +297,9 @@ Lua globals persist for the lifetime of the processor instance. | `log.context.attributes`, `.metadata` | Group attributes and metadata of non-OpenTelemetry grouped logs. | OpenTelemetry properties exist only for logs that carry OpenTelemetry context, -for example logs received by the `opentelemetry` input. Use `fb.is_otel(log)` -to check. To give ordinary logs OpenTelemetry context, place the -`opentelemetry_envelope` processor before FTS. +for example logs received by the `opentelemetry` input; for other logs they are +`nil`. Use `fb.is_otel(log)` to check. To give ordinary logs OpenTelemetry +context, place the `opentelemetry_envelope` processor before FTS. Resource, scope, and context changes apply only to the current record. Record order is preserved. @@ -185,6 +313,8 @@ order is preserved. | `fb.set_trace_context(log, trace_hex, span_hex)` | Set the trace and span IDs from hexadecimal strings. IDs must be nonzero. | | `fb.log_field(log, name)` | Read an OpenTelemetry log field such as `severity_number`, `trace_id`, or `observed_timestamp`. | +The OpenTelemetry helpers fail on logs without OpenTelemetry context. + ### Example: parse an embedded JSON message ```lua @@ -232,8 +362,8 @@ function process(metric) fb.scale_metric(metric, 0.001, "s") end - for i = 1, metric.datapoints:len() do - metric.datapoints:get(i):get("attributes"):remove("request.id") + if metric.name == "http.server.requests" then + fb.aggregate_metric(metric, {"http.route", "http.response.status_code"}, "sum") end end ``` @@ -254,7 +384,8 @@ Datapoint maps use OpenTelemetry field names in snake case, for example per-bucket counts. Removing a datapoint attribute can make two series identical, which fails the -policy. Use `fb.aggregate_metric` to combine them deliberately. +policy with `duplicate series/window; aggregate explicitly`. Use +`fb.aggregate_metric` to combine them deliberately. ### Metric helpers @@ -262,8 +393,8 @@ policy. Use `fb.aggregate_metric` to combine them deliberately. | ------ | ----------- | | `fb.metric_value(point)` | Scalar value of a gauge or sum datapoint. | | `fb.set_metric_value(point, value)` | Set the scalar value of a datapoint. | -| `fb.scale_metric(metric, factor, unit)` | Multiply values by a positive factor and optionally set the unit. Applies to gauges, sums, histograms, and summaries, including bounds and exemplars. | -| `fb.aggregate_metric(metric, keys, op)` | Keep only the listed attributes and combine equal series. `op` is `sum`, or for gauges also `min`, `max`, `mean`, or `count`. | +| `fb.scale_metric(metric, factor, unit)` | Multiply values by a positive factor and optionally set the unit. Applies to gauges, sums, histograms, and summaries, including bounds and exemplars. An integer factor keeps integer values; another factor turns them into doubles. | +| `fb.aggregate_metric(metric, keys, op)` | Keep only the listed attributes and combine series that become equal. Points are combined only when their start time and time are also equal. `op` is `sum`, or for gauges also `min`, `max`, `mean`, or `count`. Supports gauges, sums, and explicit histograms. | | `fb.merge_histogram_buckets(metric, bounds)` | Merge histogram buckets into a subset of the existing bounds. | | `fb.copy_metric(metric, name, description, unit)` | Emit a copy of the family with a new name. | | `fb.extract_metric(metric, field, name, monotonic)` | Emit a sum derived from a histogram or summary: `count`, `sum`, or `avg`. | @@ -306,26 +437,46 @@ end | `span.dropped_attributes_count`, `.dropped_events_count`, `.dropped_links_count` | Dropped counts. | | `span.resource`, `span.scope` | Same properties as for logs. | -Dropping spans or changing IDs can break trace topology; FTS does not repair -parent references. +FTS sees one span at a time. Dropping spans or changing IDs can break trace +topology, and FTS does not repair parent references. It does not make +trace-level sampling decisions; use the `sampling` processor for that. ## Common helpers | Helper | Description | | ------ | ----------- | -| `fb.redact(value, keys, replacement)` | Replace the values of matching keys in nested maps and arrays. Key matching is case-insensitive. The default replacement is `[REDACTED]`. | -| `fb.keep_keys(map, keys)` | Remove every key not listed. | +| `fb.redact(value, keys, replacement)` | Replace the values of matching keys in nested maps and arrays. The default replacement is `[REDACTED]`. See [redaction limits](#redaction-limits). | +| `fb.keep_keys(map, keys)` | Remove every top-level key not listed. | | `fb.parse_json(text)` | Parse a JSON document into typed values. Integer widths are preserved; duplicate keys and nesting beyond 64 levels are rejected. Input is limited to 1 MiB. | | `fb.parse_time(text)` | Parse an RFC 3339 timestamp into `{sec, nsec}`. | -Key-based redaction does not inspect free-form strings; secrets embedded in -message text need an explicit string policy. +### Redaction limits + +`fb.redact` is key-based. It is useful for structured secrets, but it does not +find sensitive data by itself: + +- It matches whole keys, ignoring ASCII case. `password` matches `Password`, + but not `db_password`, `password_hash` or `pass`. There are no prefixes, + patterns or paths; list every key name. +- A matching key has its whole value replaced, including nested maps and + arrays. Duplicate occurrences of the same key in one map are removed. +- It never reads string values. Secrets inside free-form text, such as a token + in a log message, need an explicit string policy, and that policy only + catches the formats it describes. +- It only visits the value passed to it. Redacting `log.body` does not touch + `log.attributes`, `log.metadata` or resource attributes; for spans, + `span.attributes` and `span.events` are separate calls. +- With `error_mode: keep`, an item whose policy fails is forwarded unredacted. +- An FTS processor on one output only affects that output. To redact data for + every destination, attach the policy to the input. ## Limits and sandbox - Values can have at most 64 levels of nesting. - OpenTelemetry strings and keys must be valid UTF-8 without NUL characters; use `fb.bytes` for binary data. +- OpenTelemetry values have no unsigned 64-bit type. In items with + OpenTelemetry context, a `uint64` above 2^63 − 1 fails the policy. - Only `require("fluentbit")` is available. The filesystem, process, FFI, debug, coroutine, and module loading libraries are disabled, as are `print`, `xpcall`, `getfenv`, `setfenv`, and `module`. `collectgarbage` accepts only @@ -335,11 +486,23 @@ message text need an explicit string policy. on untrusted input. - With `jit: true` there is no CPU limit. Use it only for policies whose execution time you control. -- The sandbox is not a security boundary: policies are trusted configuration. -- Each processor instance has its own Lua state. + +The restricted environment and the limits protect the pipeline from policy +mistakes, such as an endless loop or unbounded memory growth. They are not a +security boundary: do not run policies from untrusted authors. Treat policies +as trusted configuration, with the same access control as the rest of the +Fluent Bit configuration. ## Diagnostics -Each batch logs at most five policy errors plus a summary count. The processor -error metric counts batches with handled policy failures, and the drop metrics -count removed records. +Policy errors are logged with the Lua message and `edits rolled back`, for +example: + +```text +[error] [processor:fts:fts.0] log policy failed: [string "fts_policy"]:1: boom; edits rolled back +``` + +Each batch logs at most five policy errors plus a summary count. With +`http_server` enabled, `fluentbit_processor_errors_total` with the label +`processor="fts"` counts batches with handled policy failures, and +`fluentbit_processor_items_drop_total` counts removed items. diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_examples.py b/tests/integration/scenarios/processor_fts/tests/test_fts_examples.py new file mode 100644 index 00000000000..6ea1b547e63 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_examples.py @@ -0,0 +1,68 @@ +"""Run the FTS examples under examples/fts and compare with their expected output.""" +import json +from pathlib import Path + +import pytest +import requests +import yaml + +from utils.test_service import FluentBitTestService + + +EXAMPLES = Path(__file__).resolve().parents[5] / "examples" / "fts" +SIGNALS = {"resourceLogs": "logs", "resourceMetrics": "metrics", "resourceSpans": "traces"} + + +def example_config(tmp_path, example): + """Load an example configuration and adapt only what the test harness needs.""" + config = yaml.safe_load((example / "fluent-bit.yaml").read_text()) + service = config.setdefault("service", {}) + service.update({"http_server": True, "http_listen": "127.0.0.1", + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}", "grace": 1}) + source = config["pipeline"]["inputs"][0] + if source["name"] == "opentelemetry": + source["port"] = "${FLUENT_BIT_TEST_LISTENER_PORT}" + for processors in source.get("processors", {}).values(): + for processor in processors: + if "script" in processor: + processor["script"] = str(example / processor["script"]) + path = tmp_path / f"{example.name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def printed_json(service): + text = Path(service.flb.log_file).read_text(encoding="utf-8") + return [json.loads(line) for line in text.splitlines() if line.startswith("{")] + + +def test_quickstart_example(tmp_path): + example = EXAMPLES / "quickstart" + expected = json.loads((example / "expected-output.json").read_text()) + service = FluentBitTestService(str(example_config(tmp_path, example))) + try: + service.start() + output = service.wait_for_condition(lambda: printed_json(service), timeout=30) + finally: + service.stop() + assert output == expected + + +@pytest.mark.parametrize("name", ["logs-redaction", "metrics-normalization", "traces-filtering"]) +def test_otlp_example(tmp_path, name): + example = EXAMPLES / name + payload = json.loads((example / "input.json").read_text()) + expected = json.loads((example / "expected-output.json").read_text()) + signal = SIGNALS[next(iter(payload))] + service = FluentBitTestService(str(example_config(tmp_path, example))) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/{signal}", + json=payload, timeout=15) + assert response.status_code in (200, 201), response.text + output = service.wait_for_condition(lambda: printed_json(service), timeout=30) + finally: + service.stop() + log = Path(service.flb.log_file).read_text(encoding="utf-8") + assert "policy failed" not in log + assert output == expected