diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a5933ea --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +jobs: + go: + name: go test + runs-on: ubuntu-latest + defaults: + run: + working-directory: go + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go/go.mod + - name: go vet + run: go vet ./... + - name: go test + run: go test ./... + + python: + name: python pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: install sdk + run: pip install -e python + - name: install framework + test deps + # httpx2:starlette.testclient(FastAPI 中间件单测用)的传输依赖,缺了会 RuntimeError + run: pip install 'fastapi>=0.100' 'flask>=2.0' 'django>=4.0' 'pytest>=8.0' 'httpx2>=2.0.0' + - name: pytest + working-directory: python + run: python -m pytest -q + + node: + name: node test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + - name: npm ci + working-directory: node + run: npm ci + - name: npm test + working-directory: node + run: npm test + + java: + name: java mvn test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - name: mvn test + working-directory: java + run: mvn -B test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..813097b --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Python +__pycache__/ +*.pyc +.venv/ +*.egg-info/ +.pytest_cache/ + +# Node +node_modules/ +dist/ + +# Go +*.test + +# Java +target/ + +# 编辑器/系统 +.DS_Store +*.swp diff --git a/README.md b/README.md index 880cb58..70efc3c 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,27 @@ > 需求:[opensourceways/backlog#1938](https://github.com/opensourceways/backlog/issues/1938) 微服务可观测建设 · 子任务:[#2061](https://github.com/opensourceways/backlog/issues/2061) -统一薄封装 SDK:日志 + 指标两个 package,内部用官方库,不自研 instrumentation。契约层单一事实来源。 +统一薄封装 SDK:**结构化 JSON 日志 + Prometheus 指标**两个能力,内部引用各语言官方库(不自研 instrumentation), +输出格式由 [spec/](spec/README.md) 契约层统一约束(单一事实来源)。 ## 目录结构 -- `spec/` — 契约层(日志 JSON schema、通用字段、指标命名 prefix/label 规范)——单一事实来源 -- `go/` — obs-sdk-go(log + metrics) -- `python/` — obs-sdk-python -- `java/` — obs-sdk-java -- `node/` — obs-sdk-node +- [spec/](spec/README.md) — 契约层(日志 JSON schema、通用字段、指标命名/label、community 双层注入、trace_id 预留)——单一事实来源 +- [go/](go/README.md) — obs-sdk-go(encoding/json 单行 JSON 日志 + client_golang 指标 + http/gin 中间件) +- [python/](python/README.md) — obs-sdk-python(logging JSON Formatter + prometheus-client + FastAPI/Flask/Django 适配) +- [java/](java/README.md) — obs-sdk-java(logback + logstash JSON encoder + micrometer/prometheus registry) +- [node/](node/README.md) — obs-sdk-node(JSON serializer + prom-client + express 中间件) 各语言子目录自管版本(go.mod / pyproject.toml / pom.xml / package.json),Git tag 用 `go-v1.0.0` 等前缀区分。 -> 骨架状态:仅建目录结构,实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 +## 通用能力(四种语言对齐) + +| 能力 | 说明 | +| --- | --- | +| community 双层注入 | `service/env/instance` 部署级 const;`community` 可变 label/字段:请求级可信判定点覆盖,未覆盖回退部署默认(`OBS_*` 环境变量) | +| 请求上下文 | Go `sdkctx`(context.Context)、Python `contextvars`、Node `AsyncLocalStorage`、Java `RequestContext`(ThreadLocal) | +| trace_id 预留 | 字段可写可透传,首期不落 span | +| 服务端指标 | Go/Python/Node 由 SDK 中间件埋 `obs_http_server_*`;Java 走 Actuator + Micrometer 官方 server instrumentation(不重复埋点) | + +## 验证 + +四语言 UT + CI:`.github/workflows/ci.yml`(Go/Python/Node 本机已通过;Java 无本地 JDK,由 CI 的 maven job 编译验证)。 diff --git a/go/README.md b/go/README.md index 27f911f..68bbfd6 100644 --- a/go/README.md +++ b/go/README.md @@ -1 +1,68 @@ -# go — obs-sdk-go:Go SDK。内部结构 log/(日志 package)+ metrics/(指标 package,client_golang)。待 #2061 实现。 +# obs-sdk-go + +opensourceways 微服务可观测薄封装 SDK 的 Go 实现,契约见 [spec/](../spec/README.md)。 + +- **日志**:`log` package —— 加锁单行 JSON 写 stdout(`encoding/json` marshal map),字段规范见 spec/log-format.md +- **指标**:`metrics` package —— `client_golang` 薄封装(counter/gauge/histogram),label 规范见 spec/metrics-format.md +- **请求上下文**:`sdkctx` —— `context.Context` 承载 `community/request_id/trace_id` +- **中间件**:`middleware`(net/http)+ `middleware/ginmw`(gin)—— 注入 request_id、可信判定点解析 community、记 `obs_http_server_*` 指标 +- **community 双层注入**:`service/env/instance` 部署级 const label;`community` 普通可变 label —— 请求上下文覆盖,未覆盖回退部署默认(`OBS_*` 环境变量) + +## 目录 + +| package | 说明 | +| --- | --- | +| [sdkctx](sdkctx/context.go) | `Request{Community,RequestID,TraceID}` + `WithCommunity/WithRequestID/WithTraceID` + `From/Community/RequestID/TraceID` | +| [log](log/) | `New(cfg Config) *Logger`,方法 `Info/Warn/Debug/Error(msg, kv...)`,`With(kv...)`、`WithRequest(ctx)` | +| [metrics](metrics/) | `New(cfg Config) *Metrics`;`NewCounterVec/NewGaugeVec/NewHistogramVec(WithBuckets)(name, help, businessLabels...)`;`Handler()`(promhttp) | +| [middleware](middleware/) | `New(opts Options).Then(http.Handler)`,net/http | +| [middleware/ginmw](middleware/ginmw/) | `Middleware(opts Options) gin.HandlerFunc` | + +## 用法 + +```go +import ( + obslog "github.com/opensourceways/obs-sdk/go/log" + obsmetrics "github.com/opensourceways/obs-sdk/go/metrics" + obshttpmw "github.com/opensourceways/obs-sdk/go/middleware" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +// 日志:字段空则回退 OBS_SERVICE / OBS_ENV / OBS_INSTANCE / OBS_COMMUNITY +l := obslog.New(obslog.Config{Service: "review", Env: "test", + Instance: "pod-1", Community: "openeuler"}) +l.Info("job done", "event", "release", "issue", "2061") + +// 指标:注册一次,community 值按请求覆盖或回退部署默认 +m := obsmetrics.New(obsmetrics.Config{Service: "review", Env: "test", + Instance: "pod-1", Community: "openeuler"}) +built := m.NewCounterVec("built_releases", "发布的构建数", "kind") +built.Inc("tag") + +// 中间件:可信判定点解析 community → 写入 context(未覆盖则回退默认) +h := obshttpmw.New(obshttpmw.Options{ + Metrics: m, + ResolveCommunity: func(r *http.Request) string { + if strings.HasPrefix(r.URL.Path, "/mindspore") { + return "mindspore" + } + return "" + }, +}).Then(myHandler) +http.ListenAndServe(":8080", h) + +// gin 变体:import "github.com/opensourceways/obs-sdk/go/middleware/ginmw" +// r.Use(ginmw.Middleware(ginmw.Options{Metrics: m})) + +// 业务请求内覆盖 community / request_id / trace_id(trace_id 预留位,不落 span) +ctx := sdkctx.WithCommunity(r.Context(), "mindspore") +l.WithRequest(ctx).Info("scoped") +built.IncWithContext(ctx, "tag") +``` + +## 验证 + +```bash +go test ./... +go vet ./... +``` diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..6074e2b --- /dev/null +++ b/go/go.mod @@ -0,0 +1,49 @@ +module github.com/opensourceways/obs-sdk/go + +go 1.22 + +require ( + github.com/gin-gonic/gin v1.10.0 + github.com/prometheus/client_golang v1.20.5 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000..29fffcc --- /dev/null +++ b/go/go.sum @@ -0,0 +1,113 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/go/internal/env/env.go b/go/internal/env/env.go new file mode 100644 index 0000000..e2ba46e --- /dev/null +++ b/go/internal/env/env.go @@ -0,0 +1,63 @@ +// Package env 提供通用字段环境变量解析辅助。 +// +// 四语言 SDK 统一读同一套 OBS_* 环境变量(见 spec/common-fields.md), +// 保证部署配置模板不因语言而异。解析优先级:显式参数 > 环境变量 > 内置默认。 +package env + +import ( + "os" +) + +const ( + envService = "OBS_SERVICE" + envEnv = "OBS_ENV" + envInstance = "OBS_INSTANCE" + envCommunity = "OBS_COMMUNITY" +) + +// Service 返回服务名:explicit 非空优先,否则读 OBS_SERVICE,再否则 "unknown"。 +func Service(explicit string) string { + if explicit != "" { + return explicit + } + if v := os.Getenv(envService); v != "" { + return v + } + return "unknown" +} + +// Env 返回部署环境:explicit 非空优先,否则读 OBS_ENV,再否则 "unknown"。 +func Env(explicit string) string { + if explicit != "" { + return explicit + } + if v := os.Getenv(envEnv); v != "" { + return v + } + return "unknown" +} + +// Instance 返回实例标识:explicit 非空优先,否则读 OBS_INSTANCE,再否则 hostname。 +func Instance(explicit string) string { + if explicit != "" { + return explicit + } + if v := os.Getenv(envInstance); v != "" { + return v + } + if h, err := os.Hostname(); err == nil && h != "" { + return h + } + return "unknown" +} + +// Community 返回社区:explicit 非空优先,否则读 OBS_COMMUNITY,再否则 "unknown"。 +func Community(explicit string) string { + if explicit != "" { + return explicit + } + if v := os.Getenv(envCommunity); v != "" { + return v + } + return "unknown" +} diff --git a/go/log/README.md b/go/log/README.md index 4ef3b44..8ea46ce 100644 --- a/go/log/README.md +++ b/go/log/README.md @@ -1,2 +1,14 @@ # log package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 + +单行 JSON 结构化日志(`encoding/json` marshal map → stdout,加锁保证单行完整)。 +字段按 [spec/log-format.md](../../spec/log-format.md) / [common-fields.md](../../spec/common-fields.md): +`service/env/instance/community` 常驻 + 请求级 `request_id/trace_id`/business kv。 + +```go +l := obslog.New(obslog.Config{Service: "review", Community: "openeuler"}) +l.Info("job done", "event", "release") // 单行 JSON → stdout +l.WithRequest(ctx).Info("scoped") // ctx 里 community/request_id 覆盖常驻字段 +``` + +`ParseLevel` 支持 `debug/info/warn/error`;空字段回退 `OBS_SERVICE/OBS_ENV/OBS_INSTANCE/OBS_COMMUNITY`。 +详细用法见 [../README.md](../README.md)。 diff --git a/go/log/log.go b/go/log/log.go new file mode 100644 index 0000000..1c2add3 --- /dev/null +++ b/go/log/log.go @@ -0,0 +1,236 @@ +// Package log 提供结构化 JSON 日志(obs-sdk-go 的 log 部分)。 +// +// 输出格式与字段遵循 spec/log-format.md 与 spec/common-fields.md: +// - 单行 JSON(不 pretty),经 stdout 采集进 LTS; +// - 常驻字段 service / env / instance / community 在进程启动时注入; +// - request_id / community 覆盖值 / trace_id(预留)在请求上下文存在时附加; +// - trace_id 预留注入位:二期 trace 经 sdkctx.WithTraceID 注入即可见,零返工。 +// +// 用法: +// +// l := log.New(log.Config{Service: "robot-universal-review"}) +// l.Info("handle webhook", "event", "pull_request", "action", "opened") +// +// 请求级覆盖: +// +// l := log.New(log.Config{Service: "srv"}) +// ctx := sdkctx.WithCommunity(r.Context(), "openeuler") +// l.WithRequest(ctx).Info("by community", "path", r.URL.Path) +package log + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "github.com/opensourceways/obs-sdk/go/internal/env" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +// Level 日志级别。 +type Level int + +const ( + LevelDebug Level = iota + LevelInfo + LevelWarn + LevelError +) + +// ParseLevel 解析级别字符串;不识别的默认 LevelInfo。 +func ParseLevel(s string) Level { + switch strings.ToLower(strings.TrimSpace(s)) { + case "debug": + return LevelDebug + case "warn", "warning": + return LevelWarn + case "error": + return LevelError + default: + return LevelInfo + } +} + +func (l Level) String() string { + switch l { + case LevelDebug: + return "debug" + case LevelWarn: + return "warn" + case LevelError: + return "error" + default: + return "info" + } +} + +// Config 日志初始化配置。空字段会回退到 OBS_* 环境变量 / 内置默认。 +type Config struct { + // Service 服务名。缺省取 OBS_SERVICE,再否则 "unknown"。 + Service string + // Env 部署环境。缺省取 OBS_ENV,再否则 "unknown"。 + Env string + // Instance 实例标识。缺省取 OBS_INSTANCE,再否则 hostname。 + Instance string + // Community 部署级默认社区。缺省取 OBS_COMMUNITY,再否则 "unknown"。 + Community string + // Level 最小输出级别。空字符串等价 LevelInfo。 + Level string + // Output 日志输出 Writer。默认 os.Stdout。 + Output io.Writer +} + +// Logger 结构化 JSON 日志器。方法并发安全。 +type Logger struct { + mu sync.Mutex + + service string + envName string + instance string + // community 为部署级默认值;请求覆盖值在处理时优先。 + community string + level Level + out io.Writer + + // req 保存 WithRequest 绑定的请求级字段;nil 表示未绑定。 + req *sdkctx.Request + + // baseFields 是 With(kv...) 附加的字段,构建器模式叠加。 + baseFields map[string]any +} + +// New 创建日志器。缺省配置取环境变量回退。 +func New(cfg Config) *Logger { + out := cfg.Output + if out == nil { + out = os.Stdout + } + return &Logger{ + service: env.Service(cfg.Service), + envName: env.Env(cfg.Env), + instance: env.Instance(cfg.Instance), + community: env.Community(cfg.Community), + level: ParseLevel(cfg.Level), + out: out, + baseFields: map[string]any{}, + } +} + +// Service 返回注入的服务名。 +func (l *Logger) Service() string { return l.service } + +// Community 返回部署级默认社区。 +func (l *Logger) Community() string { return l.community } + +// With 返回携带附加字段的子日志器(叠加不改原日志器)。 +func (l *Logger) With(kv ...any) *Logger { + cp := l.clone() + for k, v := range toMap(kv) { + cp.baseFields[k] = v + } + return cp +} + +// WithRequest 返回绑定请求上下文的子日志器:该请求日志自动携带 +// sdkctx 中 request_id / community(覆盖)/ trace_id(预留)。 +func (l *Logger) WithRequest(ctx context.Context) *Logger { + cp := l.clone() + if ctx != nil { + if v := sdkctx.From(ctx); v != (sdkctx.Request{}) { + cp.req = &v + } + } + return cp +} + +func (l *Logger) clone() *Logger { + cp := &Logger{ + service: l.service, + envName: l.envName, + instance: l.instance, + community: l.community, + level: l.level, + out: l.out, + req: l.req, + baseFields: make(map[string]any, len(l.baseFields)+4), + } + for k, v := range l.baseFields { + cp.baseFields[k] = v + } + return cp +} + +// --- 级别方法 --- + +func (l *Logger) Debug(msg string, kv ...any) { l.log(LevelDebug, msg, kv) } +func (l *Logger) Info(msg string, kv ...any) { l.log(LevelInfo, msg, kv) } +func (l *Logger) Warn(msg string, kv ...any) { l.log(LevelWarn, msg, kv) } +func (l *Logger) Error(msg string, kv ...any) { l.log(LevelError, msg, kv) } + +func (l *Logger) log(level Level, msg string, kv []any) { + if level < l.level { + return + } + + fields := make(map[string]any, len(l.baseFields)+len(kv)/2+8) + for k, v := range l.baseFields { + fields[k] = v + } + for k, v := range toMap(kv) { + fields[k] = v + } + + // 请求级字段注入(覆盖静态默认)。 + if l.req != nil { + if l.req.Community != "" { + fields["community"] = l.req.Community + } + if l.req.RequestID != "" { + fields["request_id"] = l.req.RequestID + } + if l.req.TraceID != "" { + fields["trace_id"] = l.req.TraceID + } + } + + // 常驻字段。community 未被请求覆盖时用部署级默认。 + if _, ok := fields["community"]; !ok { + fields["community"] = l.community + } + fields["service"] = l.service + fields["env"] = l.envName + fields["instance"] = l.instance + fields["level"] = level.String() + fields["msg"] = msg + fields["time"] = time.Now().UTC().Format(time.RFC3339Nano) + + line, err := json.Marshal(fields) + if err != nil { + line, _ = json.Marshal(map[string]any{ + "level": level.String(), "msg": msg, "service": l.service, + "error": fmt.Sprintf("marshal log fields: %v", err), + }) + } + + l.mu.Lock() + defer l.mu.Unlock() + _, _ = l.out.Write(append(line, '\n')) +} + +// toMap 把 kv... 对拍平为 map;奇数长度忽略末尾无键值。 +func toMap(kv []any) map[string]any { + m := map[string]any{} + for i := 0; i+1 < len(kv); i += 2 { + key, ok := kv[i].(string) + if !ok { + continue + } + m[key] = kv[i+1] + } + return m +} diff --git a/go/log/log_test.go b/go/log/log_test.go new file mode 100644 index 0000000..a464bb9 --- /dev/null +++ b/go/log/log_test.go @@ -0,0 +1,111 @@ +package log + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +// capture 构造写入 buffer 的 logger。 +func capture(cfg Config) (*Logger, *bytes.Buffer) { + buf := &bytes.Buffer{} + cfg.Output = buf + return New(cfg), buf +} + +func parseLine(t *testing.T, line []byte) map[string]any { + t.Helper() + var m map[string]any + assert.NoError(t, json.Unmarshal(line, &m)) + return m +} + +func TestStaticFieldsInjected(t *testing.T) { + l, buf := capture(Config{ + Service: "review", + Env: "test", + Instance: "pod-1", + Community: "openeuler", + }) + + l.Info("hello", "event", "pull_request") + + fields := parseLine(t, bytes.TrimSpace(buf.Bytes())) + assert.Equal(t, "info", fields["level"]) + assert.Equal(t, "hello", fields["msg"]) + assert.Equal(t, "review", fields["service"]) + assert.Equal(t, "test", fields["env"]) + assert.Equal(t, "pod-1", fields["instance"]) + assert.Equal(t, "openeuler", fields["community"]) + assert.Equal(t, "pull_request", fields["event"]) + // 请求上下文缺省时 request_id / trace_id 不输出(键可缺省)。 + _, hasRid := fields["request_id"] + _, hasTid := fields["trace_id"] + assert.False(t, hasRid) + assert.False(t, hasTid) +} + +func TestRequestCommunityOverride(t *testing.T) { + l, buf := capture(Config{ + Service: "review", + Community: "openeuler", + }) + + // 业务在可信判定点写入覆盖社区。 + ctx := sdkctx.WithCommunity(context.Background(), "mindspore") + ctx = sdkctx.WithRequestID(ctx, "req-123") + l.WithRequest(ctx).Info("scoped", "k", "v") + + fields := parseLine(t, bytes.TrimSpace(buf.Bytes())) + assert.Equal(t, "mindspore", fields["community"]) + assert.Equal(t, "req-123", fields["request_id"]) +} + +func TestCommunityStaysDefaultWhenNoOverride(t *testing.T) { + l, buf := capture(Config{Service: "review", Community: "ascend"}) + + l.Info("no override") + + fields := parseLine(t, bytes.TrimSpace(buf.Bytes())) + assert.Equal(t, "ascend", fields["community"]) +} + +func TestLevelFilter(t *testing.T) { + l, buf := capture(Config{Service: "review", Level: "warn"}) + + l.Info("dropped") + l.Warn("kept") + + // debug/info 被过滤,只留 warn 一行。 + lines := bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + assert.Len(t, lines, 1) + fields := parseLine(t, lines[0]) + assert.Equal(t, "warn", fields["level"]) + assert.Equal(t, "kept", fields["msg"]) +} + +func TestWithAddsFields(t *testing.T) { + l, buf := capture(Config{Service: "review"}) + child := l.With("component", "webhook") + child.Info("with fields", "extra", 1) + + fields := parseLine(t, bytes.TrimSpace(buf.Bytes())) + assert.Equal(t, "webhook", fields["component"]) + assert.Equal(t, float64(1), fields["extra"]) +} + +func TestTraceIDReservedInject(t *testing.T) { + l, buf := capture(Config{Service: "review", Community: "openeuler"}) + // 二期:trace 经 WithTraceID 注入,日志零返工带上 trace_id。 + ctx := sdkctx.WithTraceID(context.Background(), "trace-xyz") + l.WithRequest(ctx).Info("with trace") + + fields := parseLine(t, bytes.TrimSpace(buf.Bytes())) + assert.Equal(t, "trace-xyz", fields["trace_id"]) + assert.Equal(t, "openeuler", fields["community"]) +} diff --git a/go/metrics/README.md b/go/metrics/README.md index bd09d8c..023ee1d 100644 --- a/go/metrics/README.md +++ b/go/metrics/README.md @@ -1,2 +1,19 @@ # metrics package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 + +`client_golang` 薄封装。注册时 `service/env/instance` 为 const label,`community` 为普通可变 label +(请求上下文覆盖,未覆盖回退默认),见 [spec/metrics-format.md](../../spec/metrics-format.md)。 + +```go +m := obsmetrics.New(obsmetrics.Config{Service: "review", Env: "test", + Instance: "pod-1", Community: "openeuler"}) // 可选 Namespace 前缀 +built := m.NewCounterVec("built_releases", "发布的构建数", "kind") +built.Inc("tag") // community=部署默认 +built.IncWithContext(ctx, "tag") // ctx 里有覆盖则取覆盖值 + +m.NewGaugeVec("in_flight", "在飞请求数").Set(3) +m.NewHistogramVec("latency", "耗时", "api").Observe(0.2) + +http.Handle("/metrics", m.Handler()) // promhttp 暴露 +``` + +默认注册 go / process collectors(`Config.IncludeProcess=false` 可关)。详细用法见 [../README.md](../README.md)。 diff --git a/go/metrics/metrics.go b/go/metrics/metrics.go new file mode 100644 index 0000000..3771f8d --- /dev/null +++ b/go/metrics/metrics.go @@ -0,0 +1,143 @@ +// Package metrics 提供 Prometheus 指标装配(obs-sdk-go 的 metrics 部分)。 +// +// 底层使用官方库 github.com/prometheus/client_golang,SDK 不自研 +// instrumentation,只做装配与命名/label 对齐(见 spec/metrics-format.md): +// - 所有指标自动附加 const label:service / env / instance / community(默认值); +// - community 是唯一可动态的公共 label:中心化多社区服务注册 +// Community 动态 Vec,打点时可取请求覆盖值,未覆盖回退部署级默认; +// - 统一暴露 /metrics Handler(Prometheus text exposition)。 +// +// 两类形态(见 spec:同一套字段、两种取值来源): +// 1. 静态形态(单社区独立部署,community 作 const label): +// +// m := metrics.New(metrics.Config{Service: "review"}) +// http.Handle("/metrics", m.Handler()) +// c := m.CounterVec("http_requests_total", "http requests", "method") +// c.WithLabelValues("GET").Add(1) +// +// 2. 动态形态(中心化多社区,community 作可变 label,双层注入): +// +// dyn := m.CounterVecCommunity("http_requests_total", "http requests", "method") +// dyn.WithDefault("GET").Add(1) // 部署级默认 community +// dyn.WithContext(ctx, "GET").Add(1) // ctx 覆盖 community +package metrics + +import ( + "context" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/opensourceways/obs-sdk/go/internal/env" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +const ( + // CommunityLabel 是 community 公共 label 的键名(唯一可动态项)。 + CommunityLabel = "community" + + labelService = "service" + labelEnv = "env" + labelInstance = "instance" +) + +// Config 指标初始化配置。空字段回退到 OBS_* 环境变量 / 内置默认。 +type Config struct { + // Service 服务名。 + Service string + // Env 部署环境。 + Env string + // Instance 实例标识。 + Instance string + // Community 部署级默认社区(const label / 动态覆盖回退值)。 + Community string + // Namespace 可选指标命名空间,参与 FQName 拼接。 + Namespace string + // IncludeProcess 是否注册 go/process 运行时采集器(默认 true)。 + IncludeProcess *bool +} + +// Metrics 是注册中心与装配入口。持有一个独立 prometheus.Registry,避免与 +// 全局 DefaultRegisterer 冲突。 +type Metrics struct { + reg *prometheus.Registry + + service string + envName string + instance string + defaultCommunity string + namespace string +} + +// New 创建装配实例。 +func New(cfg Config) *Metrics { + reg := prometheus.NewRegistry() + + inc := cfg.IncludeProcess == nil || *cfg.IncludeProcess + if inc { + reg.MustRegister(collectors.NewGoCollector()) + reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + } + + return &Metrics{ + reg: reg, + service: env.Service(cfg.Service), + envName: env.Env(cfg.Env), + instance: env.Instance(cfg.Instance), + defaultCommunity: env.Community(cfg.Community), + namespace: cfg.Namespace, + } +} + +// Registry 返回底层 prometheus.Registry(高级用法 / 测试)。 +func (m *Metrics) Registry() *prometheus.Registry { return m.reg } + +// Handler 返回 /metrics HTTP handler(Prometheus text exposition)。 +func (m *Metrics) Handler() http.Handler { + return promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{}) +} + +// Service 返回注入的服务名。 +func (m *Metrics) Service() string { return m.service } + +// DefaultCommunity 返回部署级默认社区。 +func (m *Metrics) DefaultCommunity() string { return m.defaultCommunity } + +// constLabels 返回自动附加的 const label:service/env/instance + community 默认值。 +func (m *Metrics) constLabels() prometheus.Labels { + return prometheus.Labels{ + labelService: m.service, + labelEnv: m.envName, + labelInstance: m.instance, + CommunityLabel: m.defaultCommunity, + } +} + +// communityConstOnly 返回仅含 service/env/instance 的 const label +// (community 将由动态 Vec 作为可变 label 自行声明)。 +func (m *Metrics) constLabelsNoCommunity() prometheus.Labels { + return prometheus.Labels{ + labelService: m.service, + labelEnv: m.envName, + labelInstance: m.instance, + } +} + +// fqName 拼 FQName:namespace[.subsystem.]name,去空段。 +func (m *Metrics) fqName(name string) string { + return prometheus.BuildFQName(m.namespace, "", name) +} + +// communityFromContext 解析 community:ctx 覆盖优先,否则部署默认。 +// ctx 为 nil 视为无覆盖。 +func (m *Metrics) communityFromContext(ctx context.Context) string { + if ctx == nil { + return m.defaultCommunity + } + if c := sdkctx.Community(ctx); c != "" { + return c + } + return m.defaultCommunity +} diff --git a/go/metrics/metrics_test.go b/go/metrics/metrics_test.go new file mode 100644 index 0000000..b3cd1bf --- /dev/null +++ b/go/metrics/metrics_test.go @@ -0,0 +1,106 @@ +package metrics + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +func noProcess() *bool { b := false; return &b } + +func newTest(t *testing.T, cfg Config) *Metrics { + t.Helper() + cfg.IncludeProcess = noProcess() + return New(cfg) +} + +func TestConstLabelsPresent(t *testing.T) { + m := newTest(t, Config{Service: "review", Env: "test", Instance: "pod-1", Community: "openeuler"}) + c := m.NewCounterVec("events_total", "events") + c.Inc() + + fam, err := m.Registry().Gather() + require.NoError(t, err) + require.Len(t, fam, 1) + + series := fam[0].GetMetric() + require.Len(t, series, 1) + + seen := map[string]string{} + for _, lp := range series[0].GetLabel() { + seen[lp.GetName()] = lp.GetValue() + } + assert.Equal(t, "review", seen["service"]) + assert.Equal(t, "test", seen["env"]) + assert.Equal(t, "pod-1", seen["instance"]) + assert.Equal(t, "openeuler", seen["community"]) + assert.Equal(t, "events_total", fam[0].GetName()) +} + +func TestDefaultCommunityValue(t *testing.T) { + m := newTest(t, Config{Service: "review", Community: "openeuler"}) + c := m.NewCounterVec("events_total", "events") + c.Inc() + + // 打点用部署级默认 community。 + assert.Equal(t, float64(1), testutil.ToFloat64( + c.Vec().WithLabelValues("openeuler"))) +} + +func TestDynamicCommunityOverride(t *testing.T) { + m := newTest(t, Config{Service: "review", Community: "openeuler"}) + c := m.NewCounterVec("events_total", "events", "kind") + + c.Add(2, "pr") // 默认 openeuler + c.IncWithContext(ctx("mindspore"), "pr") // 覆盖 mindspore + + assert.Equal(t, float64(2), testutil.ToFloat64(c.Vec().WithLabelValues("openeuler", "pr"))) + assert.Equal(t, float64(1), testutil.ToFloat64(c.Vec().WithLabelValues("mindspore", "pr"))) + + // 覆盖只影响打点时刻,不污染后续默认值。 + c.Add(1, "issue") + assert.Equal(t, float64(1), testutil.ToFloat64(c.Vec().WithLabelValues("openeuler", "issue"))) +} + +func TestGauge(t *testing.T) { + m := newTest(t, Config{Service: "review", Community: "openeuler"}) + g := m.NewGaugeVec("in_flight", "in flight") + g.Set(3) + g.Inc() + assert.Equal(t, float64(4), testutil.ToFloat64(g.Vec().WithLabelValues("openeuler"))) +} + +func TestHistogramCount(t *testing.T) { + m := newTest(t, Config{Service: "review", Community: "openeuler"}) + h := m.NewHistogramVec("request_duration_seconds", "latency", "method") + h.Observe(0.1, "GET") + h.Observe(0.2, "GET") + + // histogram 不能 testutil.ToFloat64 直接读,改按 family 校验 _count。 + fam, err := m.Registry().Gather() + require.NoError(t, err) + require.Len(t, fam, 1) + mf := fam[0].GetMetric() + require.Len(t, mf, 1) + assert.Equal(t, uint64(2), mf[0].GetHistogram().GetSampleCount()) + assert.Equal(t, "request_duration_seconds", fam[0].GetName()) +} + +func TestFQNameNamespace(t *testing.T) { + m := newTest(t, Config{Service: "review", Namespace: "obs"}) + c := m.NewCounterVec("events_total", "events") + c.Inc() + + fam, err := m.Registry().Gather() + require.NoError(t, err) + require.Equal(t, "obs_events_total", fam[0].GetName()) +} + +func ctx(community string) context.Context { + return sdkctx.WithCommunity(context.Background(), community) +} diff --git a/go/metrics/vec.go b/go/metrics/vec.go new file mode 100644 index 0000000..4ad76d5 --- /dev/null +++ b/go/metrics/vec.go @@ -0,0 +1,156 @@ +package metrics + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" +) + +// 设计说明(对应 spec/metrics-format.md「community 双层注入在指标上的实现」): +// community 统一声明为可变 label(非 const),这样单社区服务注册后每次打点 +// 填部署默认值、中心化多社区服务可填请求覆盖值 —— 「注册一次,两用」。 +// service / env / instance 恒为 const label,随 Vec 自动附加。 + +// community + business 全量 label 名。 +func (m *Metrics) allLabelNames(business []string) []string { + out := make([]string, 0, len(business)+1) + out = append(out, CommunityLabel) + out = append(out, business...) + return out +} + +// resolve 构造一次性 label 值:community 放首位,后接业务 label 值。 +func (m *Metrics) resolve(ctx context.Context, businessVals []string) []string { + out := make([]string, 0, len(businessVals)+1) + out = append(out, m.communityFromContext(ctx)) + out = append(out, businessVals...) + return out +} + +// --- Counter --- + +// CounterVec 是业务计数器。注册时自动附加 service/env/instance const label 与 +// community 可变 label。同名重复注册会 panic(与 client_golang 语义一致)。 +type CounterVec struct { + m *Metrics + vec *prometheus.CounterVec +} + +// NewCounterVec 注册一个计数器。businessLabels 为业务维度(不含 community)。 +func (m *Metrics) NewCounterVec(name, help string, businessLabels ...string) *CounterVec { + vec := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: m.fqName(name), + Help: help, + ConstLabels: m.constLabelsNoCommunity(), + }, + m.allLabelNames(businessLabels), + ) + m.reg.MustRegister(vec) + return &CounterVec{m: m, vec: vec} +} + +// Inc 以部署级默认 community 自增。businessVals 顺序须与注册时 businessLabels 一致。 +func (v *CounterVec) Inc(businessVals ...string) { v.Add(1, businessVals...) } + +// Add 以部署级默认 community 增加 v。businessVals 同上。 +func (v *CounterVec) Add(delta float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(nil, businessVals)...).Add(delta) +} + +// IncWithContext 以 ctx 覆盖 community 自增(未覆盖回退默认)。 +func (v *CounterVec) IncWithContext(ctx context.Context, businessVals ...string) { + v.AddWithContext(ctx, 1, businessVals...) +} + +// AddWithContext 以 ctx 覆盖 community 增加 v。 +func (v *CounterVec) AddWithContext(ctx context.Context, delta float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(ctx, businessVals)...).Add(delta) +} + +// Vec 返回底层 CounterVec(高级用法;注意 community 为首个 label)。 +func (v *CounterVec) Vec() *prometheus.CounterVec { return v.vec } + +// --- Gauge --- + +// GaugeVec 与 CounterVec 同构,取 gauge 语义。 +type GaugeVec struct { + m *Metrics + vec *prometheus.GaugeVec +} + +// NewGaugeVec 注册一个 gauge。businessLabels 为业务维度(不含 community)。 +func (m *Metrics) NewGaugeVec(name, help string, businessLabels ...string) *GaugeVec { + vec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: m.fqName(name), + Help: help, + ConstLabels: m.constLabelsNoCommunity(), + }, + m.allLabelNames(businessLabels), + ) + m.reg.MustRegister(vec) + return &GaugeVec{m: m, vec: vec} +} + +// Set 以部署级默认 community 设值。 +func (v *GaugeVec) Set(val float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(nil, businessVals)...).Set(val) +} + +// SetWithContext 以 ctx 覆盖 community 设值。 +func (v *GaugeVec) SetWithContext(ctx context.Context, val float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(ctx, businessVals)...).Set(val) +} + +// Inc 以部署级默认 community 自增 1(gauge 语义)。 +func (v *GaugeVec) Inc(businessVals ...string) { v.Add(1, businessVals...) } + +// Add 以部署级默认 community 增减 v。 +func (v *GaugeVec) Add(delta float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(nil, businessVals)...).Add(delta) +} + +// Vec 返回底层 GaugeVec。 +func (v *GaugeVec) Vec() *prometheus.GaugeVec { return v.vec } + +// --- Histogram --- + +// HistogramVec 与 CounterVec 同构,取 histogram 语义(默认桶见 NewHistogramVec 注释)。 +type HistogramVec struct { + m *Metrics + vec *prometheus.HistogramVec +} + +// NewHistogramVec 注册一个 histogram(默认 Prometheus 桶)。 +func (m *Metrics) NewHistogramVec(name, help string, businessLabels ...string) *HistogramVec { + return m.NewHistogramVecWithBuckets(name, help, nil, businessLabels...) +} + +// NewHistogramVecWithBuckets 注册带自定义桶的 histogram。 +func (m *Metrics) NewHistogramVecWithBuckets(name, help string, buckets []float64, businessLabels ...string) *HistogramVec { + vec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: m.fqName(name), + Help: help, + Buckets: buckets, + ConstLabels: m.constLabelsNoCommunity(), + }, + m.allLabelNames(businessLabels), + ) + m.reg.MustRegister(vec) + return &HistogramVec{m: m, vec: vec} +} + +// Observe 以部署级默认 community 记录一个观测值。 +func (v *HistogramVec) Observe(val float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(nil, businessVals)...).Observe(val) +} + +// ObserveWithContext 以 ctx 覆盖 community 记录观测值。 +func (v *HistogramVec) ObserveWithContext(ctx context.Context, val float64, businessVals ...string) { + v.vec.WithLabelValues(v.m.resolve(ctx, businessVals)...).Observe(val) +} + +// Vec 返回底层 HistogramVec。 +func (v *HistogramVec) Vec() *prometheus.HistogramVec { return v.vec } diff --git a/go/middleware/ginmw/ginmw.go b/go/middleware/ginmw/ginmw.go new file mode 100644 index 0000000..6603dd8 --- /dev/null +++ b/go/middleware/ginmw/ginmw.go @@ -0,0 +1,92 @@ +// Package ginmw 提供 gin 框架的观测中间件(obs-sdk-go)。 +// +// 与 net/http 的 middleware 职责一致(注入 request_id / community、可选记录 +// obs_http_server_* 服务器指标),只是适配 gin.HandlerFunc。gin 用户按此接入: +// +// import "github.com/opensourceways/obs-sdk/go/middleware/ginmw" +// +// m := metrics.New(metrics.Config{Service: "review"}) +// r := gin.New() +// r.Use(ginmw.Middleware(ginmw.Options{Metrics: m})) +package ginmw + +import ( + "crypto/rand" + "encoding/hex" + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "github.com/opensourceways/obs-sdk/go/metrics" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +// HeaderRequestID 与 net/http 中间件一致。 +const HeaderRequestID = "X-Request-Id" + +// CommunityResolver 从 *gin.Context 解析 community 覆盖值;空串表示不覆盖。 +// 解析须来自可信判定点(路由/认证主体/白名单)。 +type CommunityResolver func(c *gin.Context) string + +// Options 中间件配置。 +type Options struct { + // Metrics 若非 nil,则记录服务器指标。 + Metrics *metrics.Metrics + // ResolveCommunity 可选:从请求解析 community。 + ResolveCommunity CommunityResolver + // DisableRequestID 置 true 跳过 request_id 注入。 + DisableRequestID bool +} + +// Middleware 返回 gin.HandlerFunc。 +func Middleware(opts Options) gin.HandlerFunc { + var ( + reqTotal *metrics.CounterVec + reqDuration *metrics.HistogramVec + ) + if m := opts.Metrics; m != nil { + reqTotal = m.NewCounterVec("http_server_requests_total", "HTTP requests handled", + "method", "path", "status_code") + reqDuration = m.NewHistogramVec("http_server_request_duration_seconds", + "HTTP request latency", "method", "path", "status_code") + } + + return func(c *gin.Context) { + ctx := c.Request.Context() + + if !opts.DisableRequestID { + rid := c.GetHeader(HeaderRequestID) + if rid == "" { + rid = newRequestID() + } + ctx = sdkctx.WithRequestID(ctx, rid) + } + if opts.ResolveCommunity != nil { + if cc := opts.ResolveCommunity(c); cc != "" { + ctx = sdkctx.WithCommunity(ctx, cc) + } + } + c.Request = c.Request.WithContext(ctx) + + start := time.Now() + c.Next() + + if reqTotal != nil { + path := c.Request.URL.Path // 低基数提示:接入服务应归一化为路由模板 + reqTotal.AddWithContext(c.Request.Context(), 1, + c.Request.Method, path, strconv.Itoa(c.Writer.Status())) + reqDuration.ObserveWithContext(c.Request.Context(), + time.Since(start).Seconds(), c.Request.Method, path, + strconv.Itoa(c.Writer.Status())) + } + } +} + +func newRequestID() string { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + return strconv.FormatInt(time.Now().UnixNano(), 36) + } + return hex.EncodeToString(b) +} diff --git a/go/middleware/ginmw/ginmw_test.go b/go/middleware/ginmw/ginmw_test.go new file mode 100644 index 0000000..f3ee849 --- /dev/null +++ b/go/middleware/ginmw/ginmw_test.go @@ -0,0 +1,45 @@ +package ginmw + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/opensourceways/obs-sdk/go/metrics" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +func noProcess() *bool { b := false; return &b } + +func TestGinMiddlewareRequestIDAndCommunity(t *testing.T) { + gin.SetMode(gin.TestMode) + + var ( + gotRid string + gotC string + ) + r := gin.New() + r.Use(Middleware(Options{ + Metrics: metrics.New(metrics.Config{ + Service: "review", Community: "openeuler", IncludeProcess: noProcess(), + }), + ResolveCommunity: func(c *gin.Context) string { return "openEuler" }, + })) + r.GET("/ping", func(c *gin.Context) { + gotRid = sdkctx.RequestID(c.Request.Context()) + gotC = sdkctx.Community(c.Request.Context()) + c.String(http.StatusOK, "pong") + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/ping", nil) + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + assert.NotEmpty(t, gotRid) + assert.Equal(t, "openEuler", gotC) +} diff --git a/go/middleware/middleware.go b/go/middleware/middleware.go new file mode 100644 index 0000000..40cc4c2 --- /dev/null +++ b/go/middleware/middleware.go @@ -0,0 +1,146 @@ +// Package middleware 提供 net/http 的观测中间件(obs-sdk-go)。 +// +// 中间件职责(薄装配,不自研 instrumentation): +// - 为请求注入 request_id(无则生成);有可信入站头(X-Request-Id)则沿用; +// - 可选从可信来源解析 community 写入 context(由解析函数提供,SDK 不裸透传); +// - 若配置绑定了 *metrics.Metrics,则记录 HTTP 服务器指标 +// obs_http_server_requests_total / obs_http_server_request_duration_seconds。 +// +// 用法(net/http): +// +// m := metrics.New(metrics.Config{Service: "review"}) +// mid := middleware.New(middleware.Options{Metrics: m}) +// http.ListenAndServe(":8080", mid.Then(http.DefaultServeMux)) +package middleware + +import ( + "context" + "crypto/rand" + "encoding/hex" + "net/http" + "strconv" + "time" + + "github.com/opensourceways/obs-sdk/go/metrics" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +// HeaderRequestID 是入站请求 ID 头名。中间件仅在请求已携带时沿用, +// 否则自动生成 —— 该头是否可信由部署方决定。 +const HeaderRequestID = "X-Request-Id" + +// CommunityResolver 从请求解析 community 覆盖值;返回空串表示「不覆盖,回退默认」。 +// 解析必须来自可信判定点(路由/认证主体/白名单),不得原样透传外部入参。 +type CommunityResolver func(r *http.Request) string + +// Options 中间件配置。 +type Options struct { + // Metrics 若非 nil,则记录 HTTP 服务器指标。 + Metrics *metrics.Metrics + // ResolveCommunity 可选:从请求解析 community。 + ResolveCommunity CommunityResolver + // DisableRequestID 置 true 跳过 request_id 注入。 + DisableRequestID bool +} + +// Middleware 是可复用的 net/http 中间件(Next 由 Then 注入)。 +type Middleware struct { + opts Options + + reqTotal *metrics.CounterVec + reqDuration *metrics.HistogramVec +} + +// New 构建中间件。opts.Metrics 为 nil 时仅做上下文注入,不记指标。 +func New(opts Options) *Middleware { + md := &Middleware{opts: opts} + if m := opts.Metrics; m != nil { + // 服务器公共指标用 obs_ 前缀(spec:不以 service 名开头,按 label 过滤)。 + md.reqTotal = m.NewCounterVec("http_server_requests_total", "HTTP requests handled", + "method", "path", "status_code") + md.reqDuration = m.NewHistogramVec("http_server_request_duration_seconds", + "HTTP request latency", "method", "path", "status_code") + } + return md +} + +// Then 包装下游 handler。 +func (md *Middleware) Then(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + if !md.opts.DisableRequestID { + rid := r.Header.Get(HeaderRequestID) + if rid == "" { + rid = newRequestID() + } + ctx = sdkctx.WithRequestID(ctx, rid) + } + if md.opts.ResolveCommunity != nil { + if c := md.opts.ResolveCommunity(r); c != "" { + ctx = sdkctx.WithCommunity(ctx, c) + } + } + r = r.WithContext(ctx) + + rec := &responseRecorder{ResponseWriter: w} + start := time.Now() + next.ServeHTTP(rec, r) + dur := time.Since(start).Seconds() + + if md.reqTotal != nil { + status := rec.status + if status == 0 { + status = http.StatusOK + } + // 低基数提示:path 可能含动态段,接入服务应将高基数路径归一化 + // 为路由模板再入 label(spec/metrics-format.md:路径可选 / 按模板入 label)。 + md.reqTotal.AddWithContext(ctx, 1, r.Method, r.URL.Path, strconv.Itoa(status)) + md.reqDuration.ObserveWithContext(ctx, dur, r.Method, r.URL.Path, strconv.Itoa(status)) + } + }) +} + +type responseRecorder struct { + http.ResponseWriter + status int + bytes int +} + +func (r *responseRecorder) WriteHeader(code int) { + r.status = code + r.ResponseWriter.WriteHeader(code) +} + +func (r *responseRecorder) Write(b []byte) (int, error) { + if r.status == 0 { + r.status = http.StatusOK + } + n, err := r.ResponseWriter.Write(b) + r.bytes += n + return n, err +} + +// Flush 透传 http.Flusher(SSE 等)。 +func (r *responseRecorder) Flush() { + if f, ok := r.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +func newRequestID() string { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + return strconv.FormatInt(time.Now().UnixNano(), 36) + } + return hex.EncodeToString(b) +} + +// ContextWithCommunity 便捷函数:返回携带 community 覆盖值的 context +// (供业务在无中间件场景手动注入)。 +func ContextWithCommunity(ctx context.Context, community string) context.Context { + if community != "" { + return sdkctx.WithCommunity(ctx, community) + } + return ctx +} diff --git a/go/middleware/middleware_test.go b/go/middleware/middleware_test.go new file mode 100644 index 0000000..9bc4c4d --- /dev/null +++ b/go/middleware/middleware_test.go @@ -0,0 +1,161 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/opensourceways/obs-sdk/go/metrics" + "github.com/opensourceways/obs-sdk/go/sdkctx" +) + +func noProcess() *bool { b := false; return &b } + +func TestRequestIDInjected(t *testing.T) { + var got string + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = sdkctx.RequestID(r.Context()) + w.WriteHeader(http.StatusNoContent) + }) + + mid := New(Options{}) + srv := httptest.NewServer(mid.Then(inner)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/ping") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + assert.NotEmpty(t, got) +} + +func TestRequestIDHonorsInbound(t *testing.T) { + var got string + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = sdkctx.RequestID(r.Context()) + w.WriteHeader(http.StatusOK) + }) + + mid := New(Options{}) + srv := httptest.NewServer(mid.Then(inner)) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/ping", nil) + req.Header.Set(HeaderRequestID, "inbound-1") + _, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, "inbound-1", got) +} + +func TestCommunityResolverWritesContext(t *testing.T) { + var got string + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = sdkctx.Community(r.Context()) + w.WriteHeader(http.StatusOK) + }) + + mid := New(Options{ + ResolveCommunity: func(r *http.Request) string { + // 仅演示:真实场景须来自可信判定点(路由/认证主体/白名单)。 + return "openeuler" + }, + }) + srv := httptest.NewServer(mid.Then(inner)) + defer srv.Close() + + _, err := http.Get(srv.URL + "/ping") + require.NoError(t, err) + assert.Equal(t, "openeuler", got) +} + +// findCounter 从 Gather 结果中按 family 名与 label 匹配找 counter 值。 +func findCounter(t *testing.T, m *metrics.Metrics, familyName string, wantLabels map[string]string) float64 { + t.Helper() + fams, err := m.Registry().Gather() + require.NoError(t, err) + for _, f := range fams { + if f.GetName() != familyName { + continue + } + for _, s := range f.GetMetric() { + labels := map[string]string{} + for _, lp := range s.GetLabel() { + labels[lp.GetName()] = lp.GetValue() + } + ok := true + for k, v := range wantLabels { + if labels[k] != v { + ok = false + break + } + } + if ok { + return s.GetCounter().GetValue() + } + } + } + t.Fatalf("family %s with labels %v not found", familyName, wantLabels) + return 0 +} + +func TestHTTPMetricsRecordedWithDefaultCommunity(t *testing.T) { + m := metrics.New(metrics.Config{ + Service: "review", + Env: "test", + Instance: "pod-1", + Community: "openeuler", + IncludeProcess: noProcess(), + }) + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + }) + + mid := New(Options{Metrics: m}) + srv := httptest.NewServer(mid.Then(inner)) + defer srv.Close() + + // POST /jobs → 201。 + _, err := http.Post(srv.URL+"/jobs", "text/plain", nil) + require.NoError(t, err) + + // 服务器指标按默认 community 记(无覆盖)。 + assert.Equal(t, float64(1), findCounter(t, m, + "http_server_requests_total", map[string]string{ + "service": "review", "env": "test", "instance": "pod-1", + "community": "openeuler", "method": "POST", + "path": "/jobs", "status_code": "201", + })) +} + +func TestHTTPMetricsRespectsCommunityOverride(t *testing.T) { + m := metrics.New(metrics.Config{ + Service: "review", + Community: "openeuler", + IncludeProcess: noProcess(), + }) + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // 通过 ResolveCommunity 在中间件内完成注入,覆盖发生在打点(收尾)之前。 + mid := New(Options{ + Metrics: m, + ResolveCommunity: func(r *http.Request) string { + return "mindspore" + }, + }) + srv := httptest.NewServer(mid.Then(inner)) + defer srv.Close() + + _, err := http.Get(srv.URL + "/jobs") + require.NoError(t, err) + + assert.Equal(t, float64(1), findCounter(t, m, + "http_server_requests_total", map[string]string{ + "community": "mindspore", "method": "GET", + "path": "/jobs", "status_code": "200", + })) +} diff --git a/go/sdkctx/context.go b/go/sdkctx/context.go new file mode 100644 index 0000000..aaa19ff --- /dev/null +++ b/go/sdkctx/context.go @@ -0,0 +1,70 @@ +// Package sdkctx 定义请求级通用字段(community / request_id / trace_id)在 +// context.Context 中的读写。log 与 metrics 两个 SDK package 共用本包,保证 +// 双层注入(部署级默认 + 请求级覆盖)取自同一来源。 +// +// 设计约束(见 spec/common-fields.md):请求级 community 必须由业务在可信判定点 +// (路由前缀 / 认证主体 / 白名单)显式写入,SDK 不负责从外部请求原样透传。 +package sdkctx + +import "context" + +// key 是 context 值的私有 key 类型,避免与其他包冲突。 +type key struct{} + +// Request 请求级通用字段。 +type Request struct { + // Community 覆盖值;空串表示「未覆盖,回退部署级默认」。 + Community string + // RequestID 单请求关联 ID;空串表示未设置。 + RequestID string + // TraceID 预留位(二期 trace 接入);本期恒为空。 + TraceID string +} + +// WithCommunity 返回一个携带 community 覆盖值的新 context。 +// 业务须在可信判定点调用(见包注释)。 +func WithCommunity(ctx context.Context, community string) context.Context { + return withField(ctx, func(r *Request) { r.Community = community }) +} + +// WithRequestID 返回一个携带 request_id 的新 context。 +func WithRequestID(ctx context.Context, requestID string) context.Context { + return withField(ctx, func(r *Request) { r.RequestID = requestID }) +} + +// WithTraceID 返回一个携带 trace_id 的新 context(二期 trace 预留注入点)。 +func WithTraceID(ctx context.Context, traceID string) context.Context { + return withField(ctx, func(r *Request) { r.TraceID = traceID }) +} + +// From 取出 context 中携带的请求级字段;未设置时返回零值 Request。 +func From(ctx context.Context) Request { + if ctx == nil { + return Request{} + } + if v, ok := ctx.Value(key{}).(*Request); ok && v != nil { + return *v + } + return Request{} +} + +// Community 返回当前 community 覆盖值;未覆盖返回空串。 +func Community(ctx context.Context) string { + return From(ctx).Community +} + +// RequestID 返回当前 request_id;未设置返回空串。 +func RequestID(ctx context.Context) string { + return From(ctx).RequestID +} + +// TraceID 返回当前 trace_id;未设置返回空串。 +func TraceID(ctx context.Context) string { + return From(ctx).TraceID +} + +func withField(ctx context.Context, mutate func(*Request)) context.Context { + cur := From(ctx) + mutate(&cur) + return context.WithValue(ctx, key{}, &cur) +} diff --git a/go/sdkctx/context_test.go b/go/sdkctx/context_test.go new file mode 100644 index 0000000..96606d0 --- /dev/null +++ b/go/sdkctx/context_test.go @@ -0,0 +1,38 @@ +package sdkctx + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCommunityOverrideAndFallback(t *testing.T) { + ctx := context.Background() + + // 未覆盖:Community 返回空串。 + assert.Equal(t, "", Community(ctx)) + + // 设置覆盖后可读回。 + ctx = WithCommunity(ctx, "openeuler") + assert.Equal(t, "openeuler", Community(ctx)) + + // 覆盖是叠加而非替换:保留其他字段。 + ctx = WithRequestID(ctx, "req-1") + r := From(ctx) + assert.Equal(t, "openeuler", r.Community) + assert.Equal(t, "req-1", r.RequestID) +} + +func TestFromZeroValue(t *testing.T) { + var r = From(nil) + assert.Equal(t, Request{}, r) + assert.Equal(t, "", r.Community) +} + +func TestTraceIDReserved(t *testing.T) { + // trace_id 预留注入位:本期可空,二期经 WithTraceID 注入即可见。 + ctx := WithTraceID(context.Background(), "trace-abc") + assert.Equal(t, "trace-abc", TraceID(ctx)) + assert.Equal(t, "", RequestID(ctx)) +} diff --git a/java/README.md b/java/README.md index d2b5100..e388a8c 100644 --- a/java/README.md +++ b/java/README.md @@ -1 +1,120 @@ -# java — obs-sdk-java:Java SDK。log/ + metrics/(micrometer/actuator 对齐)。待 #2061 实现。 +# obs-sdk-java + +opensourceways 微服务可观测薄封装 SDK 的 Java 实现,契约见根目录 [spec/](../spec/README.md)。 +语义与 Go / Python / Node SDK 对齐: + +- **community 双层注入**:`service/env/instance` 为部署级 const label,`community` 建模为普通可变 label, + 值取请求上下文覆盖(可信判定点写入),未覆盖回退部署默认 —— 「注册一次两用」。 +- **`trace_id` 预留**:首期只保证字段可写可透传,不落 span。 +- **日志**:结构化 JSON(logback + logstash JSON encoder,MDC 输出固定键)。 +- **指标**:Micrometer + Prometheus registry 薄封装(业务 counter/gauge/histogram); + HTTP 服务端指标不重复造轮子,Java 服务走 Spring Boot Actuator + Micrometer 官方 server instrumentation。 + +## 模块 + +| 组件 | 说明 | +| --- | --- | +| `context.RequestContext` | 请求级上下文(community/request_id/trace_id),ThreadLocal 作用域句柄,对齐其它语言的 sdkctx/contextvars/ALS | +| `log.ObsLogging` | 部署默认字段 + 请求覆盖字段写入 SLF4J MDC,由 JSON encoder 输出 | +| `ObsMetrics` | 业务指标装配(common tags + community 动态 label + namespace 前缀) | +| `middleware.ObsFilter` | 可选 Servlet Filter:注入 request_id + 可信判定点解析 community → RequestContext + MDC | + +## 构建与测试 + +JDK 17 + Maven: + +```bash +mvn test +``` + +> 当前开发机无 JDK/Maven,Java 代码未在本机编译运行;已按 Micrometer 1.13 公开 API 编写, +> 由仓库 CI(.github/workflows/ci.yml 的 java job)负责编译 + 跑 `mvn test` 验证。 +> 如 CI 暴露问题,以 CI 输出为准修复。 + +## 指标使用 + +```java +ObsSdkConfig cfg = ObsSdkConfig.builder() + .service("review") // 生产走 OBS_SERVICE 等环境变量,ObsSdkConfig.fromEnvironment() + .env("test") + .instance("pod-1") + .community("openeuler") // 部署级默认 community + .build(); +ObsMetrics m = ObsMetrics.of(cfg); + +// 业务 counter —— community 自动排首位,值取请求上下文覆盖,否则回退默认 +ObsMetrics.CounterVec built = m.counter("built_releases", "发布的构建数", "kind"); +built.inc(1, "tag"); + +// 请求上下文内再埋 → 该条 series 的 community 被覆盖为 mindspore +try (RequestContext.Scope scope = RequestContext.push("mindspore", "req-1", null)) { + built.inc(1, "tag"); +} + +// gauge / histogram(Timer 自动产出 _seconds_count/_sum,可配桶) +m.gauge("in_flight", "在飞请求数").set(3); +m.histogram("review_duration", "评审耗时", new double[]{0.1, 0.5}).observe(0.05); + +// Prometheus text 快照(供自检 / 测试) +String text = m.text(); +``` + +> **命名说明**:Micrometer 会自动为 Counter 追加 `_total`、为 Timer(秒基)追加 `_seconds`, +> 因此 Java 侧传基础名(不带 `_total`/`_seconds` 后缀),导出名与其它语言 SDK 一致 +> (如 `built_releases_total`、`review_duration_seconds_count`)。 +> 跨服务共享 SDK 时给 `namespace`(如 `"obs"`),导出名变为 `obs__total`。 + +### community 双层注入的日志侧 + +```java +// 服务启动: +ObsLogging.init(ObsSdkConfig.fromEnvironment()); + +// 请求处理:先 init() 后每次请求在可信判定点解析后 push 即可 +RequestContext.push(community, requestId, traceId); // try-with-resource 作用域 +``` + +## 集成:Spring Boot Actuator + Micrometer + +Java 服务的服务端指标交给 Actuator 暴露(stater 对齐,SDK 不重复埋 HTTP 指标): + +1. 依赖 `micrometer-registry-prometheus` + 引入 `spring-boot-starter-actuator`。 +2. 把 SDK 的 registry 暴露成 bean 让 Actuator 托管(`PrometheusMeterRegistry` 会被 + actuator 自动发现并挂到 `/actuator/prometheus`): + +```java +@Bean +public PrometheusMeterRegistry prometheusRegistry(ObsSdkConfig obsConfig) { + return ObsMetrics.of(obsConfig).meterRegistry(); +} +``` + +3. `application.yml`:`management.endpoints.web.exposure.include: prometheus`。 + +在接入服务里把 SDK 的 business 指标注册到同一个 `PrometheusMeterRegistry`(上面 bean 的实例), +即可与 Actuator 的服务端指标合并暴露给 AOM 抓取。 + +## 请求上下文中间件(可选 Servlet Filter) + +```java +// Spring Boot 注册,community resolver 必须来自可信判定点(路由前缀/认证主体/白名单), +// 不能裸读 URL/Header 当 community(见 spec/community-values.md) +@Bean +public FilterRegistrationBean obsFilter() { + FilterRegistrationBean reg = new FilterRegistrationBean<>(); + reg.setFilter(new ObsFilter(req -> + req.getRequestURI().startsWith("/mindspore") ? "mindspore" : null)); + reg.addUrlPatterns("/*"); + reg.setOrder(Ordered.HIGHEST_PRECEDENCE); + return reg; +} +``` + +## 日志 JSON 输出 + +把 `examples/logback-json.xml` 拷成接入服务的 logback 配置并引入 `logstash-logback-encoder`, +日志即输出单行 JSON(固定键 `service/env/instance/community/request_id/trace_id`),例: + +```json +{"@timestamp":"2026-09-08T09:00:00.000+08:00","level":"INFO","logger_name":"com.x.ReviewSvc","message":"hello","service":"review","env":"test","instance":"pod-1","community":"openeuler"} +``` diff --git a/java/examples/logback-json.xml b/java/examples/logback-json.xml new file mode 100644 index 0000000..5cfd942 --- /dev/null +++ b/java/examples/logback-json.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + service + env + instance + community + request_id + trace_id + + + + System.out + + + + + + diff --git a/java/log/README.md b/java/log/README.md deleted file mode 100644 index 4ef3b44..0000000 --- a/java/log/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# log package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/java/metrics/README.md b/java/metrics/README.md deleted file mode 100644 index bd09d8c..0000000 --- a/java/metrics/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# metrics package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 0000000..f044508 --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + io.opensourceways + obs-sdk-java + 0.1.0 + jar + + obs-sdk-java + opensourceways 微服务可观测薄封装 SDK (Java) —— 结构化 JSON 日志 + Micrometer/Prometheus 指标,契约见 ../spec + + + 17 + UTF-8 + 1.13.6 + 2.0.13 + 1.5.12 + 7.4 + 5.10.2 + + + + + + io.micrometer + micrometer-core + ${micrometer.version} + + + io.micrometer + micrometer-registry-prometheus + ${micrometer.version} + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + ch.qos.logback + logback-classic + ${logback.version} + provided + + + net.logstash.logback + logstash-logback-encoder + ${logstash-encoder.version} + provided + + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + diff --git a/java/src/main/java/io/opensourceways/obssdk/ObsMetrics.java b/java/src/main/java/io/opensourceways/obssdk/ObsMetrics.java new file mode 100644 index 0000000..07dde77 --- /dev/null +++ b/java/src/main/java/io/opensourceways/obssdk/ObsMetrics.java @@ -0,0 +1,247 @@ +package io.opensourceways.obssdk; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import io.micrometer.prometheusmetrics.PrometheusConfig; +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; +import io.opensourceways.obssdk.context.RequestContext; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * 指标装配:对 Micrometer + Prometheus registry 的薄封装,语义与 Go/Python/Node SDK 对齐 + * (见 spec/metrics-format.md、common-fields.md)。 + * + *
    + *
  • service/env/instance 三个部署级字段注册为 common tags(const label);
  • + *
  • community 建模为普通可变 label:值取请求上下文覆盖(可信判定点显式写入), + * 无覆盖时回退部署默认 —— 「注册一次两用」,单社区/多社区共用同一注册点。
  • + *
  • {@code namespace} 可选:给指标名加前缀(跨服务共享 SDK 时用)。
  • + *
+ * + *

注意:本 SDK 不重复造 HTTP 服务端指标 —— Java 服务通常走 Spring Boot Actuator + + * Micrometer 官方 server instrumentation 上报(starter 对齐),本类只提供业务 counter/gauge/histogram。 + */ +public final class ObsMetrics { + + private final PrometheusMeterRegistry registry; + private final String service; + private final String envName; + private final String instance; + private final String defaultCommunity; + private final String namespace; + + private ObsMetrics(ObsSdkConfig cfg) { + this.registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + this.service = cfg.service(); + this.envName = cfg.envName(); + this.instance = cfg.instance(); + this.defaultCommunity = cfg.community(); + this.namespace = cfg.namespace(); + + List common = new ArrayList<>(); + if (service != null) { + common.add(Tag.of("service", service)); + } + if (envName != null) { + common.add(Tag.of("env", envName)); + } + if (instance != null) { + common.add(Tag.of("instance", instance)); + } + if (!common.isEmpty()) { + registry.config().commonTags(Tags.of(common)); + } + } + + public static ObsMetrics of(ObsSdkConfig cfg) { + return new ObsMetrics(cfg); + } + + public PrometheusMeterRegistry meterRegistry() { + return registry; + } + + /** Prometheus text 格式快照(同其它语言 SDK 的 {@code text()},测试与自检用)。 */ + public String text() { + return registry.scrape(); + } + + /** + * 注册业务 counter。注意 Micrometer/Prometheus 会自动为 Counter 追加 {@code _total} 后缀, + * 这里传基础名(如 {@code "http_server_requests"})。 + * + * @param name 指标基础名 + * @param help 帮助文本 + * @param labelNames 业务 label 名(不含 community,community 由本 SDK 自动排在首位) + */ + public CounterVec counter(String name, String help, String... labelNames) { + return new CounterVec(meterName(name), help, labelNames); + } + + /** 注册 gauge(exported 名不带后缀)。 */ + public GaugeVec gauge(String name, String help, String... labelNames) { + return new GaugeVec(meterName(name), help, labelNames); + } + + /** + * 注册业务 histogram(基于 Micrometer Timer,自动追加 {@code _seconds} 后缀 + {@code _count/_sum})。 + * Timer 的 label 与 counter 相同语义:community 自动排在首位。 + * + * @param name 指标基础名,如 {@code "http_server_request_duration"} + * @param help 帮助文本 + * @param buckets 可选 SLA 桶(秒);不传则不产生 {@code _bucket} 系列 + * @param labelNames 业务 label 名 + */ + public HistogramVec histogram(String name, String help, double[] buckets, String... labelNames) { + return new HistogramVec(meterName(name), help, labelNames, buckets); + } + + public HistogramVec histogram(String name, String help, String... labelNames) { + return new HistogramVec(meterName(name), help, labelNames, null); + } + + private String meterName(String base) { + return (namespace == null || namespace.isEmpty()) ? base : namespace + "_" + base; + } + + // ---- 内部工具:community 值解析(覆盖优先于默认) ---- + + private static String communityValue(String defaultCommunity) { + return RequestContext.communityOverride().orElse(defaultCommunity); + } + + private static List tags(String defaultCommunity, String[] labelNames, String[] values) { + if (labelNames.length != values.length) { + throw new IllegalArgumentException("label 数量不匹配: names=" + labelNames.length + " values=" + values.length); + } + List tags = new ArrayList<>(labelNames.length + 1); + String community = communityValue(defaultCommunity); + if (community != null) { + tags.add(Tag.of("community", community)); + } + for (int i = 0; i < labelNames.length; i++) { + tags.add(Tag.of(labelNames[i], values[i] == null ? "" : values[i])); + } + return tags; + } + + // ---- vec 类型 ---- + + /** 业务 counter 句柄:{@code inc} 系列方法自动带 community(context 覆盖或默认)。 */ + public final class CounterVec { + private final String name; + private final String help; + private final String[] labelNames; + + private CounterVec(String name, String help, String[] labelNames) { + this.name = name; + this.help = help; + this.labelNames = labelNames; + } + + public void inc(String... labelValues) { + inc(1.0, labelValues); + } + + public void inc(double amount, String... labelValues) { + registry.counter(name, tags(ObsMetrics.this.defaultCommunity, labelNames, labelValues)).increment(amount); + } + } + + /** 业务 gauge 句柄:按 (community + 业务 label) 组合各自维护一个可写状态。 */ + public final class GaugeVec { + private final String name; + private final String help; + private final String[] labelNames; + private final ConcurrentHashMap> states = new ConcurrentHashMap<>(); + + private GaugeVec(String name, String help, String[] labelNames) { + this.name = name; + this.help = help; + this.labelNames = labelNames; + } + + public void set(double value, String... labelValues) { + child(labelValues).set(value); + } + + private AtomicReference child(String... labelValues) { + if (labelValues.length != labelNames.length) { + throw new IllegalArgumentException( + "label 数量不匹配: names=" + labelNames.length + " values=" + labelValues.length); + } + String community = communityValue(ObsMetrics.this.defaultCommunity); + List tags = new ArrayList<>(labelNames.length + 1); + List keyParts = new ArrayList<>(labelNames.length + 1); + keyParts.add(community == null ? "" : community); + if (community != null) { + tags.add(Tag.of("community", community)); + } + for (int i = 0; i < labelNames.length; i++) { + String v = labelValues[i] == null ? "" : labelValues[i]; + tags.add(Tag.of(labelNames[i], v)); + keyParts.add(v); + } + String key = keyParts.toString(); + AtomicReference state = states.get(key); + if (state == null) { + // JDK 无 AtomicDouble(Guava 才有),用 AtomicReference 存可写 gauge 值,默认 0.0 + AtomicReference created = new AtomicReference<>(0.0d); + Gauge.builder(name, created, ref -> ref.get()) + .description(help) + .tags(tags) + .register(registry); + AtomicReference raced = states.putIfAbsent(key, created); + state = raced == null ? created : raced; + } + return state; + } + } + + /** 业务 histogram 句柄:基于 Timer,自动带 community。 */ + public final class HistogramVec { + private final String name; + private final String help; + private final String[] labelNames; + private final double[] buckets; + + private HistogramVec(String name, String help, String[] labelNames, double[] buckets) { + this.name = name; + this.help = help; + this.labelNames = labelNames; + this.buckets = buckets; + } + + /** 观测一次耗时,单位秒。 */ + public void observe(double seconds, String... labelValues) { + Timer timer = timer(labelValues); + timer.record(Duration.ofNanos(Math.round(seconds * 1_000_000_000d))); + } + + public void observe(Duration duration, String... labelValues) { + timer(labelValues).record(duration); + } + + private Timer timer(String... labelValues) { + List tags = tags(ObsMetrics.this.defaultCommunity, labelNames, labelValues); + Timer.Builder builder = Timer.builder(name).description(help).tags(tags); + if (buckets != null && buckets.length > 0) { + Duration[] sla = Arrays.stream(buckets) + .mapToObj(b -> Duration.ofNanos(Math.round(b * 1_000_000_000d))) + .toArray(Duration[]::new); + builder.serviceLevelObjectives(sla); + } + return builder.register(registry); + } + } +} diff --git a/java/src/main/java/io/opensourceways/obssdk/ObsSdkConfig.java b/java/src/main/java/io/opensourceways/obssdk/ObsSdkConfig.java new file mode 100644 index 0000000..d28e332 --- /dev/null +++ b/java/src/main/java/io/opensourceways/obssdk/ObsSdkConfig.java @@ -0,0 +1,112 @@ +package io.opensourceways.obssdk; + +/** + * SDK 静态配置(部署级默认字段)。 + * + *

与 spec/common-fields.md 对齐:统一从环境变量读取默认值 + * {@code OBS_SERVICE / OBS_ENV / OBS_INSTANCE / OBS_COMMUNITY}, + * 语义与 Go/Python/Node SDK 的 Config 一致。 + */ +public final class ObsSdkConfig { + + public static final String ENV_SERVICE = "OBS_SERVICE"; + public static final String ENV_ENV = "OBS_ENV"; + public static final String ENV_INSTANCE = "OBS_INSTANCE"; + public static final String ENV_COMMUNITY = "OBS_COMMUNITY"; + + private final String service; + private final String envName; + private final String instance; + private final String community; + private final String namespace; + + private ObsSdkConfig(Builder b) { + this.service = b.service; + this.envName = b.envName; + this.instance = b.instance; + this.community = b.community; + this.namespace = b.namespace; + } + + /** 按规范优先级读取环境变量;调用方显式设置的值优先。 */ + public static Builder builder() { + return new Builder(); + } + + /** 只从环境变量构造(测试外入口)。 */ + public static ObsSdkConfig fromEnvironment() { + return builder() + .service(System.getenv(ENV_SERVICE)) + .env(System.getenv(ENV_ENV)) + .instance(System.getenv(ENV_INSTANCE)) + .community(System.getenv(ENV_COMMUNITY)) + .build(); + } + + public String service() { + return service; + } + + public String envName() { + return envName; + } + + public String instance() { + return instance; + } + + public String community() { + return community; + } + + /** 可选命名空间前缀(跨服务共享 SDK 埋点时用 obs_ 等前缀区分,见 spec/metrics-format.md)。 */ + public String namespace() { + return namespace; + } + + public Builder toBuilder() { + return new Builder() + .service(service) + .env(envName) + .instance(instance) + .community(community) + .namespace(namespace); + } + + public static final class Builder { + private String service; + private String envName; + private String instance; + private String community; + private String namespace; + + public Builder service(String v) { + this.service = v; + return this; + } + + public Builder env(String v) { + this.envName = v; + return this; + } + + public Builder instance(String v) { + this.instance = v; + return this; + } + + public Builder community(String v) { + this.community = v; + return this; + } + + public Builder namespace(String v) { + this.namespace = v; + return this; + } + + public ObsSdkConfig build() { + return new ObsSdkConfig(this); + } + } +} diff --git a/java/src/main/java/io/opensourceways/obssdk/context/RequestContext.java b/java/src/main/java/io/opensourceways/obssdk/context/RequestContext.java new file mode 100644 index 0000000..aa42bf5 --- /dev/null +++ b/java/src/main/java/io/opensourceways/obssdk/context/RequestContext.java @@ -0,0 +1,135 @@ +package io.opensourceways.obssdk.context; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +/** + * 请求级上下文:承载 {@code community / request_id / trace_id} 三个请求字段, + * 语义与其它语言 SDK 对齐 —— Go sdkctx(context.Context)、Python contextvars、Node AsyncLocalStorage。 + * + *

community 双层注入(见 spec/common-fields.md、community-values.md): + * 第一层是部署级默认(静态,来自 OBS_* 环境变量);第二层是请求上下文动态覆盖。 + * 中间件在服务自己的可信判定点(路由前缀 / 认证主体 / 白名单)解析出请求级 community + * 后,通过 {@link #push} 写入本上下文;指标与日志读取 {@link #communityOverride()}, + * 未覆盖时回退部署默认。 + * + *

{@code trace_id} 为预留位(首期不做 trace):本次只保证字段在上下文中可写入、可透传, + * 供后续 trace 接入时读取,不产生任何 span。 + */ +public final class RequestContext { + + /** 当前线程请求字段。继承式 ThreadLocal:子线程能读到父线程(HTTP 场景足够)。 */ + private static final InheritableThreadLocal HOLDER = new InheritableThreadLocal<>(); + + private final String community; + private final String requestId; + private final String traceId; + + private RequestContext(String community, String requestId, String traceId) { + this.community = community; + this.requestId = requestId; + this.traceId = traceId; + } + + /** 构造一个请求字段快照(通常由中间件调用)。 */ + public static RequestContext of(String community, String requestId, String traceId) { + return new RequestContext(community, requestId, traceId); + } + + /** 用给定字段绑定当前线程,返回作用域句柄;离开作用域(close)后自动还原。 */ + public static Scope push(String community, String requestId, String traceId) { + return push(of(community, requestId, traceId)); + } + + /** 同 {@link #push(String, String, String)},复用已有快照。 */ + public static Scope push(RequestContext ctx) { + final RequestContext prev = HOLDER.get(); + final AtomicBoolean closed = new AtomicBoolean(false); + HOLDER.set(ctx); + return () -> { + if (closed.compareAndSet(false, true)) { + if (prev == null) { + HOLDER.remove(); + } else { + HOLDER.set(prev); + } + } + }; + } + + /** 在当前线程请求上下文内执行 runnable,结束后自动还原线程原上下文。 */ + public static void run(RequestContext ctx, Runnable runnable) { + try (Scope ignored = push(ctx)) { + runnable.run(); + } + } + + /** 同 {@link #run},带返回值。 */ + public static T call(RequestContext ctx, Supplier supplier) { + try (Scope ignored = push(ctx)) { + return supplier.get(); + } + } + + /** 当前线程请求字段(可能为空)。 */ + public static Optional current() { + return Optional.ofNullable(HOLDER.get()); + } + + /** 清理当前线程请求字段(测试/长任务/线程池兜底用;正常请求走 {@link Scope#close()} 自动还原)。 */ + public static void clear() { + HOLDER.remove(); + } + + /** 请求级 community 覆盖值;无覆盖返回 {@link Optional#empty()}。 */ + public static Optional communityOverride() { + return current().map(ctx -> ctx.community); + } + + /** 请求级 request_id。 */ + public static Optional currentRequestId() { + return current().map(ctx -> ctx.requestId); + } + + /** 请求级 trace_id(预留)。 */ + public static Optional currentTraceId() { + return current().map(ctx -> ctx.traceId); + } + + /** 供日志装配读取的字段快照(写入 MDC)。 */ + public Map asMdcFields() { + Map fields = new HashMap<>(); + if (community != null) { + fields.put("community", community); + } + if (requestId != null) { + fields.put("request_id", requestId); + } + if (traceId != null) { + fields.put("trace_id", traceId); + } + return fields; + } + + public String community() { + return community; + } + + public String requestId() { + return requestId; + } + + public String traceId() { + return traceId; + } + + /** 作用域句柄:{@link RequestContext#push} 的返回,close 时还原线程上下文。 */ + @FunctionalInterface + public interface Scope extends AutoCloseable { + @Override + void close(); + } +} diff --git a/java/src/main/java/io/opensourceways/obssdk/log/ObsLogging.java b/java/src/main/java/io/opensourceways/obssdk/log/ObsLogging.java new file mode 100644 index 0000000..3a7125b --- /dev/null +++ b/java/src/main/java/io/opensourceways/obssdk/log/ObsLogging.java @@ -0,0 +1,79 @@ +package io.opensourceways.obssdk.log; + +import io.opensourceways.obssdk.ObsSdkConfig; +import io.opensourceways.obssdk.context.RequestContext; +import org.slf4j.MDC; + +import java.util.Optional; + +/** + * 日志结构化装配:把部署级默认字段 + 请求级覆盖字段写入 SLF4J MDC, + * 由接入服务的 logback JSON encoder(logstash-logback-encoder,见 + * {@code examples/logback-json.xml})输出为单行 JSON。 + * + *

固定键:{@code service / env / instance / community / request_id / trace_id} + * (与 spec/common-fields.md、spec/log-format.md 对齐)。 + * + *

community 双层注入与其它语言 SDK 一致:部署级默认来自 {@code OBS_*}/Config, + * 请求级由中间件在可信判定点解析后经 {@link RequestContext#push} 写入, + * 本助手在取数时用请求级覆盖默认(同 metrics 的 resolve 语义)。 + */ +public final class ObsLogging { + + public static final String MDC_SERVICE = "service"; + public static final String MDC_ENV = "env"; + public static final String MDC_INSTANCE = "instance"; + public static final String MDC_COMMUNITY = "community"; + public static final String MDC_REQUEST_ID = "request_id"; + public static final String MDC_TRACE_ID = "trace_id"; + + private static volatile ObsSdkConfig cfg; + + private ObsLogging() { + } + + /** 全局初始化一次:登记部署级默认字段(通常服务启动时调用)。 */ + public static void init(ObsSdkConfig config) { + cfg = config; + MDC.put(MDC_SERVICE, nvl(config.service())); + MDC.put(MDC_ENV, nvl(config.envName())); + MDC.put(MDC_INSTANCE, nvl(config.instance())); + MDC.put(MDC_COMMUNITY, nvl(config.community())); + } + + /** + * 请求进入时刷新请求级字段:request_id 取显式值(无则保持已有的), + * community 取请求覆盖,未覆盖回退部署默认;随后交由 JSON encoder 输出。 + * 返回前先写入 MDC;{@link RequestContext} 的 scope 由中间件管理。 + */ + public static void enrich(RequestContext request) { + Optional community = request != null && request.community() != null + ? Optional.of(request.community()) + : Optional.empty(); + Optional requestId = request != null && request.requestId() != null + ? Optional.of(request.requestId()) + : Optional.empty(); + Optional traceId = request != null && request.traceId() != null + ? Optional.of(request.traceId()) + : Optional.empty(); + + String base = cfg != null ? cfg.community() : null; + MDC.put(MDC_COMMUNITY, community.orElse(base != null ? base : "")); + if (requestId.isPresent()) { + MDC.put(MDC_REQUEST_ID, requestId.get()); + } + if (traceId.isPresent()) { + MDC.put(MDC_TRACE_ID, traceId.get()); + } + } + + /** 请求结束时清理请求级字段,避免线程复用串染(MDC 由框架在线程回收时兜底)。 */ + public static void clearRequestScope() { + MDC.remove(MDC_REQUEST_ID); + MDC.remove(MDC_TRACE_ID); + } + + private static String nvl(String v) { + return v == null ? "" : v; + } +} diff --git a/java/src/main/java/io/opensourceways/obssdk/middleware/ObsFilter.java b/java/src/main/java/io/opensourceways/obssdk/middleware/ObsFilter.java new file mode 100644 index 0000000..ff84ee0 --- /dev/null +++ b/java/src/main/java/io/opensourceways/obssdk/middleware/ObsFilter.java @@ -0,0 +1,78 @@ +package io.opensourceways.obssdk.middleware; + +import io.opensourceways.obssdk.context.RequestContext; +import io.opensourceways.obssdk.log.ObsLogging; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.UUID; +import java.util.function.Function; + +/** + * Java 版请求上下文中间件(对齐 Go http/gin、Python、Node 的 middleware)。 + * + *

职责与其它语言 SDK 一致:注入 {@code request_id}(沿用入站 {@code X-Request-Id} 或生成), + * 在可信判定点解析请求级 community 后写入 {@link RequestContext},并刷新日志 MDC。 + * + *

安全约束(spec/community-values.md):community 不从不加鉴别的入参盲取, + * 必须经 {@code resolveCommunity} —— 由服务自己基于路由前缀 / 认证主体 / 白名单判定; + * 不提供 resolver 时退化为部署级单社区(不做请求级覆盖)。 + * + *

不做 HTTP 服务端指标埋点:Java 服务一般走 Spring Boot Actuator + + * Micrometer 官方 server instrumentation(starter 对齐),SDK 不重复造轮子。 + * + *

Spring Boot 注册示例: + *

{@code
+ * @Bean
+ * public FilterRegistrationBean obsFilter() {
+ *     FilterRegistrationBean reg = new FilterRegistrationBean<>();
+ *     reg.setFilter(new ObsFilter(req -> req.getRequestURI().startsWith("/mindspore") ? "mindspore" : null));
+ *     reg.addUrlPatterns("/*");
+ *     reg.setOrder(Ordered.HIGHEST_PRECEDENCE);
+ *     return reg;
+ * }
+ * }
+ */ +public class ObsFilter implements Filter { + + public static final String HEADER_REQUEST_ID = "X-Request-Id"; + public static final String HEADER_TRACE_ID = "X-Trace-Id"; + + private final Function resolveCommunity; + + public ObsFilter() { + this(req -> null); + } + + public ObsFilter(Function resolveCommunity) { + this.resolveCommunity = resolveCommunity; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest http = (HttpServletRequest) request; + HttpServletResponse res = (HttpServletResponse) response; + + String requestId = http.getHeader(HEADER_REQUEST_ID); + if (requestId == null || requestId.isEmpty()) { + requestId = UUID.randomUUID().toString().replace("-", ""); + } + // trace_id 预留:只在可信入站头存在时透传,不主动生成(首期不落 span) + String traceId = http.getHeader(HEADER_TRACE_ID); + String community = resolveCommunity.apply(http); + + try (RequestContext.Scope scope = RequestContext.push(community, requestId, traceId)) { + ObsLogging.enrich(RequestContext.current().orElse(null)); + chain.doFilter(request, response); + } finally { + ObsLogging.clearRequestScope(); + } + } +} diff --git a/java/src/test/java/io/opensourceways/obssdk/ObsMetricsTest.java b/java/src/test/java/io/opensourceways/obssdk/ObsMetricsTest.java new file mode 100644 index 0000000..a8d355f --- /dev/null +++ b/java/src/test/java/io/opensourceways/obssdk/ObsMetricsTest.java @@ -0,0 +1,121 @@ +package io.opensourceways.obssdk; + +import io.opensourceways.obssdk.context.RequestContext; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ObsMetricsTest { + + private static ObsSdkConfig cfg(String... kv) { + ObsSdkConfig.Builder b = ObsSdkConfig.builder(); + for (int i = 0; i + 1 < kv.length; i += 2) { + switch (kv[i]) { + case "service" -> b.service(kv[i + 1]); + case "env" -> b.env(kv[i + 1]); + case "instance" -> b.instance(kv[i + 1]); + case "community" -> b.community(kv[i + 1]); + case "namespace" -> b.namespace(kv[i + 1]); + default -> throw new IllegalArgumentException("unknown cfg key: " + kv[i]); + } + } + return b.build(); + } + + /** 抓取 scrape 文本里某一 series 家族的数值行(非 # 注释行)。 */ + private static List samples(String text, String family) { + List out = new ArrayList<>(); + for (String line : text.split("\n")) { + if (line.startsWith(family) && !line.startsWith("#")) { + out.add(line); + } + } + return out; + } + + private static double value(String sampleLine) { + int brace = sampleLine.indexOf('}'); + return Double.parseDouble(sampleLine.substring(brace + 1).trim()); + } + + @Test + void counter_commonTag加community双层注入() { + ObsMetrics m = ObsMetrics.of(cfg("service", "review", "env", "test", + "instance", "pod-1", "community", "openeuler")); + ObsMetrics.CounterVec c = m.counter("events", "事件数", "kind"); + + c.inc(1, "pr"); + try (RequestContext.Scope ignored = RequestContext.push("mindspore", "r-1", "t-1")) { + c.inc(2, "pr"); + } + + String text = m.text(); + List ev = samples(text, "events_total"); + assertEquals(2, ev.size(), "默认与覆盖两种 community 应各自成一条 series"); + + String defLine = lineWith(ev, "community=\"openeuler\""); + assertTrue(defLine.contains("service=\"review\""), "const label service 应在: " + defLine); + assertTrue(defLine.contains("env=\"test\"")); + assertTrue(defLine.contains("instance=\"pod-1\"")); + assertTrue(defLine.contains("kind=\"pr\"")); + assertEquals(1.0, value(defLine)); + + String ovLine = lineWith(ev, "community=\"mindspore\""); + assertTrue(ovLine.contains("service=\"review\"")); + assertEquals(2.0, value(ovLine)); + } + + @Test + void gauge与histogram() { + ObsMetrics m = ObsMetrics.of(cfg("service", "s", "community", "openeuler")); + ObsMetrics.GaugeVec g = m.gauge("in_flight", "在飞请求数"); + g.set(3); + + ObsMetrics.HistogramVec h = m.histogram("latency", "处理延迟"); + h.observe(0.1); + h.observe(0.3); + + String text = m.text(); + // gauge 名不带后缀 + List gauges = samples(text, "in_flight"); + assertEquals(1, gauges.size()); + assertTrue(gauges.get(0).contains("community=\"openeuler\"")); + assertEquals(3.0, value(gauges.get(0))); + + // Timer 自动追加 _seconds,并产出 _count/_sum + List counts = samples(text, "latency_seconds_count"); + assertEquals(1, counts.size()); + assertEquals(2.0, value(counts.get(0))); + List sums = samples(text, "latency_seconds_sum"); + assertEquals(1, sums.size()); + assertTrue(Math.abs(value(sums.get(0)) - 0.4) < 1e-6); + } + + @Test + void namespace前缀() { + ObsMetrics m = ObsMetrics.of(cfg("service", "s", "community", "openeuler", "namespace", "obs")); + m.counter("events", "事件数").inc(1); + assertTrue(m.text().contains("obs_events_total"), m.text()); + } + + @Test + void histogram自定义桶() { + ObsMetrics m = ObsMetrics.of(cfg("service", "s", "community", "openeuler")); + m.histogram("latency", "处理延迟", new double[]{0.1, 0.5}).observe(0.05); + String text = m.text(); + assertTrue(text.contains("latency_seconds_bucket{"), text); + assertTrue(text.contains("le=\"0.1\""), text); + assertTrue(text.contains("le=\"0.5\""), text); + } + + private static String lineWith(List lines, String sub) { + return lines.stream() + .filter(l -> l.contains(sub)) + .findFirst() + .orElseThrow(() -> new AssertionError("缺少包含 " + sub + " 的 series,实际: " + lines)); + } +} diff --git a/java/src/test/java/io/opensourceways/obssdk/RequestContextTest.java b/java/src/test/java/io/opensourceways/obssdk/RequestContextTest.java new file mode 100644 index 0000000..6ee0336 --- /dev/null +++ b/java/src/test/java/io/opensourceways/obssdk/RequestContextTest.java @@ -0,0 +1,56 @@ +package io.opensourceways.obssdk; + +import io.opensourceways.obssdk.context.RequestContext; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RequestContextTest { + + @Test + void push之后读到覆盖值_close后还原为空() { + assertFalse(RequestContext.communityOverride().isPresent()); + + try (RequestContext.Scope s = RequestContext.push("openeuler", "req-1", "trace-1")) { + assertEquals("openeuler", RequestContext.communityOverride().orElse(null)); + assertEquals("req-1", RequestContext.currentRequestId().orElse(null)); + assertEquals("trace-1", RequestContext.currentTraceId().orElse(null)); + } + + assertFalse(RequestContext.communityOverride().isPresent()); + assertFalse(RequestContext.currentRequestId().isPresent()); + } + + @Test + void 嵌套push_内层关闭后还原外层() { + try (RequestContext.Scope outer = RequestContext.push("openeuler", "r-outer", null)) { + try (RequestContext.Scope inner = RequestContext.push("mindspore", "r-inner", null)) { + assertEquals("mindspore", RequestContext.communityOverride().orElse(null)); + } + assertEquals("openeuler", RequestContext.communityOverride().orElse(null)); + assertEquals("r-outer", RequestContext.currentRequestId().orElse(null)); + } + assertFalse(RequestContext.communityOverride().isPresent()); + } + + @Test + void call返回结果并还原上下文() { + try (RequestContext.Scope s = RequestContext.push("openeuler", null, null)) { + String result = RequestContext.call( + RequestContext.of("mindspore", "r-1", null), + () -> RequestContext.communityOverride().orElse(null)); + assertEquals("mindspore", result); + // call 结束应还原到外层 + assertEquals("openeuler", RequestContext.communityOverride().orElse(null)); + } + } + + @Test + void 无请求上下文时无覆盖() { + RequestContext.clear(); + assertTrue(RequestContext.current().isEmpty()); + assertFalse(RequestContext.communityOverride().isPresent()); + } +} diff --git a/node/README.md b/node/README.md index 59088fc..d553344 100644 --- a/node/README.md +++ b/node/README.md @@ -1 +1,52 @@ -# node — obs-sdk-node:Node SDK。log/ + metrics/(prom-client)。待 #2061 实现。 +# obs-sdk-node + +opensourceways 微服务可观测薄封装 SDK 的 Node 实现,契约见 [spec/](../spec/README.md)。 + +- **日志**:`lib/log` —— 单行 JSON 写 stream(默认 stdout) +- **指标**:`lib/metrics` —— prom-client 薄封装(counter/gauge/histogram) +- **请求上下文**:`lib/context` —— `AsyncLocalStorage` 承载 `community/request_id/trace_id` +- **中间件**:`lib/middleware` —— Express/通用 HTTP 中间件(注入 request_id + 可信判定点解析 community + 记 `obs_http_server_*`) +- **community 双层注入**:`service/env/instance` 常驻;`community` 可变 —— 请求上下文覆盖,未覆盖回退部署默认(`OBS_*` 环境变量) + +## 用法 + +```js +const obs = require('obs-sdk-node'); // index.js:{ log, metrics, context, middleware } + +// ---- 日志 ---- +obs.log.init({ service: 'review', env: 'test', instance: 'pod-1', community: 'openeuler' }); +obs.log.info('job done', { event: 'release', issue: '2061' }); + +// ---- 指标(prom-client 需显式 _total 等最终名,本 SDK 不做改名) ---- +const m = new obs.metrics.Metrics({ service: 'review', env: 'test', + instance: 'pod-1', community: 'openeuler' }); +const built = m.counter('built_releases_total', '发布的构建数', ['kind']); +built.inc(1, { kind: 'tag' }); +m.gauge('in_flight', '在飞请求数').set(3); +m.histogram('review_duration_seconds', '评审耗时').observe(0.2); + +// ---- HTTP 服务端指标(中间件自动登记) ---- +const { makeMiddleware, metricsRouteHandler } = obs.middleware; +const mw = makeMiddleware({ + metrics: m, // 不传则用默认 Metrics + // community 必须在可信判定点解析(路由前缀/认证主体/白名单),见 spec/community-values.md + resolveCommunity: (req) => (req.url.startsWith('/mindspore') ? 'mindspore' : undefined), +}); +// 业务 app 里 use(mw) 即可:注入 request_id + push 请求上下文 + 请求结束记 obs_http_server_* 指标 +// /metrics 暴露:app.get('/metrics', metricsRouteHandler(m)); +``` + +请求级覆盖(context 作用域内日志 / 指标自动带覆盖 community 与 request_id/trace_id): + +```js +obs.context.bindRequest({ community: 'mindspore', requestId: 'req-1', traceId: 'trace-x' }, () => { + obs.log.info('scoped'); // 单行 JSON 带 community="mindspore", request_id="req-1" + built.inc(1, { kind: 'tag' }); // 该 series community="mindspore" +}); +``` + +## 验证 + +```bash +cd node && npm install && npm test +``` diff --git a/node/index.js b/node/index.js new file mode 100644 index 0000000..50e8c5f --- /dev/null +++ b/node/index.js @@ -0,0 +1,8 @@ +'use strict'; + +const log = require('./lib/log'); +const metrics = require('./lib/metrics'); +const context = require('./lib/context'); +const middleware = require('./lib/middleware'); + +module.exports = { log, metrics, context, middleware }; diff --git a/node/lib/context.js b/node/lib/context.js new file mode 100644 index 0000000..5235f47 --- /dev/null +++ b/node/lib/context.js @@ -0,0 +1,41 @@ +'use strict'; + +// 请求级通用字段(Node 版 sdkctx)。用 AsyncLocalStorage 实现线程/并发隔离, +// 语义同 Python contextvars(见 spec/common-fields.md)。 +// +// 框架适配器在入口用 bindRequest 包裹请求处理,SDK log/metrics 读取当前 +// 上下文;未绑定则回退部署级默认。 + +const { AsyncLocalStorage } = require('async_hooks'); + +const storage = new AsyncLocalStorage(); + +function current() { + return storage.getStore() || {}; +} + +function community() { + return current().community || null; +} + +function requestId() { + return current().requestId || null; +} + +function traceId() { + return current().traceId || null; +} + +// bindRequest(store, fn):在 store({community?, requestId?, traceId?})内执行 fn。 +// 会与已有上下文合并(缺省字段继承外层)。 +function bindRequest(fields, fn) { + const prev = storage.getStore() || {}; + const merged = { + community: fields.community !== undefined ? fields.community : prev.community, + requestId: fields.requestId !== undefined ? fields.requestId : prev.requestId, + traceId: fields.traceId !== undefined ? fields.traceId : prev.traceId, + }; + return storage.run(merged, fn); +} + +module.exports = { current, community, requestId, traceId, bindRequest }; diff --git a/node/lib/env.js b/node/lib/env.js new file mode 100644 index 0000000..14917d5 --- /dev/null +++ b/node/lib/env.js @@ -0,0 +1,21 @@ +'use strict'; + +// OBS_* 环境变量解析(与其他语言 SDK 一致,见 spec/common-fields.md)。 + +const os = require('os'); + +function resolve(explicit, envName, fallback) { + if (explicit) return explicit; + const v = process.env[envName]; + return v || fallback; +} + +function service(explicit) { return resolve(explicit, 'OBS_SERVICE', 'unknown'); } +function env(explicit) { return resolve(explicit, 'OBS_ENV', 'unknown'); } +function instance(explicit) { + if (explicit) return explicit; + return process.env.OBS_INSTANCE || os.hostname() || 'unknown'; +} +function community(explicit) { return resolve(explicit, 'OBS_COMMUNITY', 'unknown'); } + +module.exports = { service, env, instance, community }; diff --git a/node/lib/log.js b/node/lib/log.js new file mode 100644 index 0000000..108761c --- /dev/null +++ b/node/lib/log.js @@ -0,0 +1,58 @@ +'use strict'; + +// 结构化 JSON 日志(obs-sdk-node 的 log 部分)。单行 JSON 写 stdout, +// 字段规范见 spec/log-format.md;请求级字段来自 ./context。 + +const env = require('./env'); +const context = require('./context'); + +const LEVELS = { debug: 10, info: 20, warn: 30, error: 40 }; + +const defaults = { + service: 'unknown', + env: 'unknown', + instance: 'unknown', + community: 'unknown', +}; +let minLevel = LEVELS.info; +let stream = process.stdout; + +// init({service, env, instance, community, level, stream}):进程启动时装配。 +function init(opts = {}) { + defaults.service = env.service(opts.service); + defaults.env = env.env(opts.env); + defaults.instance = env.instance(opts.instance); + defaults.community = env.community(opts.community); + minLevel = LEVELS[opts.level] !== undefined ? LEVELS[opts.level] : LEVELS.info; + if (opts.stream) stream = opts.stream; + return api; +} + +function log(levelName, msg, fields) { + const lv = LEVELS[levelName]; + if (lv < minLevel) return; + + const req = context.current(); + const record = Object.assign({}, fields || {}, { + service: defaults.service, + env: defaults.env, + instance: defaults.instance, + community: req.community || defaults.community, + level: levelName, + msg, + time: new Date().toISOString(), + }); + if (req.requestId) record.request_id = req.requestId; + if (req.traceId) record.trace_id = req.traceId; + + const line = JSON.stringify(record); + stream.write(line + '\n'); +} + +function debug(msg, fields) { log('debug', msg, fields); } +function info(msg, fields) { log('info', msg, fields); } +function warn(msg, fields) { log('warn', msg, fields); } +function error(msg, fields) { log('error', msg, fields); } + +const api = { init, debug, info, warn, error }; +module.exports = api; diff --git a/node/lib/metrics.js b/node/lib/metrics.js new file mode 100644 index 0000000..8b62821 --- /dev/null +++ b/node/lib/metrics.js @@ -0,0 +1,91 @@ +'use strict'; + +// Prometheus 指标装配(obs-sdk-node 的 metrics 部分)。底层用官方库 prom-client, +// 不自研 instrumentation,只做装配与 label 对齐(见 spec/metrics-format.md): +// - 每个指标带 service/env/instance/community label; +// - community 是唯一可动态项(ctx 覆盖优先,否则部署默认,双层注入); +// - Metrics.text() 输出 Prometheus text 供 /metrics。 +// +// prom-client 的 labelNames 声明后值必须全给,故返回薄 wrapper 自动填公共维度, +// 业务只传业务 label。 + +const { Registry, Counter, Gauge, Histogram } = require('prom-client'); +const env = require('./env'); +const context = require('./context'); + +const COMMON = ['service', 'env', 'instance', 'community']; + +class Metrics { + constructor({ service, env: envName, instance, community, namespace } = {}) { + this._registry = new Registry(); + this._service = env.service(service); + this._env = env.env(envName); + this._instance = env.instance(instance); + this._defaultCommunity = env.community(community); + this._namespace = namespace || ''; + // 不含 process 默认采集器;业务若需要 process/内存指标自行 registry.collectDefaultMetrics。 + } + + get registry() { return this._registry; } + get defaultCommunity() { return this._defaultCommunity; } + + _fq(name) { return this._namespace ? `${this._namespace}_${name}` : name; } + + _commonVals(extra) { + const c = context.community() || this._defaultCommunity; + return Object.assign({ service: this._service, env: this._env, + instance: this._instance, community: c }, extra); + } + + // counter(name, help, labelNames?) → { inc(amount, labels) } + counter(name, help, labelNames = []) { + const metric = new Counter({ + name: this._fq(name), help, + labelNames: [...COMMON, ...labelNames], + registers: [this._registry], + }); + return { + inc: (amount = 1, labels = {}) => metric.inc(this._commonVals(labels), amount), + }; + } + + gauge(name, help, labelNames = []) { + const metric = new Gauge({ + name: this._fq(name), help, + labelNames: [...COMMON, ...labelNames], + registers: [this._registry], + }); + return { + set: (value, labels = {}) => metric.set(this._commonVals(labels), value), + inc: (amount = 1, labels = {}) => metric.inc(this._commonVals(labels), amount), + }; + } + + histogram(name, help, labelNames = []) { + const metric = new Histogram({ + name: this._fq(name), help, + labelNames: [...COMMON, ...labelNames], + registers: [this._registry], + }); + return { + observe: (value, labels = {}) => metric.observe(this._commonVals(labels), value), + }; + } + + async text() { + return this._registry.metrics(); + } +} + +// 便捷单例(默认读 OBS_* 环境变量)。 +let _default = null; +function init(opts = {}) { + if (!_default) _default = new Metrics(opts); + return _default; +} +function defaultMetrics() { + if (!_default) return init(); + return _default; +} + +module.exports = { Metrics, init, defaultMetrics }; diff --git a/node/lib/middleware.js b/node/lib/middleware.js new file mode 100644 index 0000000..c54c210 --- /dev/null +++ b/node/lib/middleware.js @@ -0,0 +1,62 @@ +'use strict'; + +// Express/通用中间件(obs-sdk-node)。 +// +// 职责(薄装配,见 spec/common-fields.md):注入 request_id(沿用可信入站头 +// X-Request-Id 或生成);可选从可信判定点解析 community;可选记录服务器指标 +// obs_http_server_requests_total / obs_http_server_request_duration_seconds。 + +const { randomUUID } = require('crypto'); +const context = require('./context'); +const { Metrics } = require('./metrics'); + +const HEADER_REQUEST_ID = 'X-Request-Id'; +const DEFAULT_METRIC_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; + +// makeMiddleware({metrics, resolveCommunity, collectServerMetrics}) → express middleware。 +function makeMiddleware({ metrics, resolveCommunity, collectServerMetrics = true } = {}) { + let counterTotal = null; + let duration = null; + if (metrics && collectServerMetrics) { + counterTotal = metrics.counter('http_server_requests_total', 'HTTP requests handled', ['method', 'path', 'status_code']); + duration = metrics.histogram('http_server_request_duration_seconds', 'HTTP request latency', ['method', 'path', 'status_code']); + } + + return function obsMiddleware(req, res, next) { + const community = resolveCommunity ? resolveCommunity(req) : undefined; + const inboundRid = req.headers[HEADER_REQUEST_ID.toLowerCase()]; + const fields = { + community, + requestId: inboundRid || randomUUID(), + }; + + context.bindRequest(fields, () => { + const startHr = process.hrtime(); + res.on('finish', () => { + if (counterTotal) { + const durSec = process.hrtime(startHr)[0] + process.hrtime(startHr)[1] / 1e9; + const labels = { + method: req.method, + path: req.originalUrl ? req.originalUrl.split('?')[0] : req.url, + status_code: String(res.statusCode), + }; + counterTotal.inc(1, labels); + duration.observe(durSec, labels); + } + }); + next(); + }); + }; +} + +// metricsRouteHandler(metrics):给 express 挂 /metrics 用。 +async function metricsRouteHandler(metrics) { + const body = await metrics.text(); + return { + statusCode: 200, + contentType: 'text/plain; version=0.0.4; charset=utf-8', + body, + }; +} + +module.exports = { makeMiddleware, metricsRouteHandler, HEADER_REQUEST_ID, DEFAULT_METRIC_BUCKETS, Metrics }; diff --git a/node/log/README.md b/node/log/README.md deleted file mode 100644 index 4ef3b44..0000000 --- a/node/log/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# log package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/node/metrics/README.md b/node/metrics/README.md deleted file mode 100644 index bd09d8c..0000000 --- a/node/metrics/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# metrics package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/node/package-lock.json b/node/package-lock.json new file mode 100644 index 0000000..879429f --- /dev/null +++ b/node/package-lock.json @@ -0,0 +1,57 @@ +{ + "name": "obs-sdk-node", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "obs-sdk-node", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "prom-client": "^15.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + } + } +} diff --git a/node/package.json b/node/package.json new file mode 100644 index 0000000..2bfa0ea --- /dev/null +++ b/node/package.json @@ -0,0 +1,22 @@ +{ + "name": "obs-sdk-node", + "version": "0.1.0", + "description": "opensourceways 微服务可观测薄封装 SDK (Node)——结构化 JSON 日志 + Prometheus 指标,契约见 ../spec", + "license": "Apache-2.0", + "main": "index.js", + "type": "commonjs", + "files": [ + "index.js", + "lib/", + "README.md" + ], + "scripts": { + "test": "node --test test/" + }, + "engines": { + "node": ">=18" + }, + "dependencies": { + "prom-client": "^15.1.0" + } +} diff --git a/node/test/log.test.js b/node/test/log.test.js new file mode 100644 index 0000000..037fa09 --- /dev/null +++ b/node/test/log.test.js @@ -0,0 +1,60 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert'); +const { Writable } = require('node:stream'); +const log = require('../lib/log'); +const context = require('../lib/context'); + +function capture() { + const chunks = []; + const stream = new Writable({ + write(c, _e, cb) { chunks.push(c.toString()); cb(); }, + }); + return { stream, lines: () => chunks.map(JSON.parse) }; +} + +test('静态字段注入 + community 双层注入', () => { + const { stream, lines } = capture(); + log.init({ service: 'review', env: 'test', instance: 'pod-1', + community: 'openeuler', level: 'info', stream }); + + log.info('hello', { event: 'pr' }); + context.bindRequest({ community: 'mindspore', requestId: 'req-1' }, () => { + log.info('scoped'); + }); + + const out = lines(); + assert.strictEqual(out.length, 2); + + const a = out[0]; + assert.strictEqual(a.service, 'review'); + assert.strictEqual(a.env, 'test'); + assert.strictEqual(a.instance, 'pod-1'); + assert.strictEqual(a.community, 'openeuler'); + assert.strictEqual(a.level, 'info'); + assert.strictEqual(a.msg, 'hello'); + assert.strictEqual(a.event, 'pr'); + assert.ok(!('request_id' in a)); + + const b = out[1]; + assert.strictEqual(b.community, 'mindspore'); + assert.strictEqual(b.request_id, 'req-1'); +}); + +test('level 过滤 + trace_id 预留注入', () => { + const { stream, lines } = capture(); + log.init({ service: 'srv', community: 'openeuler', level: 'warn', stream }); + + log.info('dropped'); + log.warn('kept'); + context.bindRequest({ traceId: 'trace-xyz' }, () => { + log.warn('with trace'); + }); + + const out = lines(); + assert.strictEqual(out.length, 2); + assert.strictEqual(out[0].msg, 'kept'); + assert.strictEqual(out[1].trace_id, 'trace-xyz'); + assert.strictEqual(out[1].community, 'openeuler'); +}); diff --git a/node/test/metrics.test.js b/node/test/metrics.test.js new file mode 100644 index 0000000..6473d98 --- /dev/null +++ b/node/test/metrics.test.js @@ -0,0 +1,52 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert'); +const { Metrics } = require('../lib/metrics'); +const context = require('../lib/context'); + +test('counter 带公共 label + community 双层注入', async () => { + const m = new Metrics({ service: 'review', env: 'test', instance: 'pod-1', + community: 'openeuler' }); + const c = m.counter('events_total', 'events', ['kind']); + c.inc(1, { kind: 'pr' }); + context.bindRequest({ community: 'mindspore' }, () => { + c.inc(1, { kind: 'pr' }); + }); + + const text = await m.text(); + const rows = text.split('\n').filter((l) => /^events_total\{/.test(l)); + assert.strictEqual(rows.length, 2); + const byComm = {}; + for (const r of rows) { + const matched = r.match(/community="([^"]+)"/); + byComm[matched[1]] = r; + } + assert.ok(byComm.openeuler.endsWith(' 1')); + assert.ok(byComm.mindspore.endsWith(' 1')); + assert.ok(byComm.openeuler.includes('service="review"')); +}); + +test('gauge + histogram', async () => { + const m = new Metrics({ service: 'review', community: 'openeuler' }); + const g = m.gauge('in_flight', 'in flight'); + g.set(3); + g.inc(2); + const h = m.histogram('latency_seconds', 'latency'); + h.observe(0.1); + h.observe(0.2); + + const text = await m.text(); + const gaugeRows = text.split('\n').filter((l) => /^in_flight\{/.test(l)); + assert.ok(gaugeRows[0].endsWith(' 5')); + assert.ok(text.includes('latency_seconds_count')); + const countRows = text.split('\n').filter((l) => /^latency_seconds_count\{/.test(l)); + assert.ok(countRows[0].endsWith(' 2')); +}); + +test('namespace 前缀', async () => { + const m = new Metrics({ service: 'review', namespace: 'obs' }); + m.counter('events_total', 'events').inc(1); + const text = await m.text(); + assert.ok(text.includes('obs_events_total')); +}); diff --git a/node/test/middleware.test.js b/node/test/middleware.test.js new file mode 100644 index 0000000..7234e3b --- /dev/null +++ b/node/test/middleware.test.js @@ -0,0 +1,82 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert'); +const { Metrics } = require('../lib/metrics'); +const context = require('../lib/context'); +const { makeMiddleware } = require('../lib/middleware'); + +// 极简 req/res 模拟。handler 为中间件的下游(在中间件 bind 的作用域内执行), +// 结束后触发 finish(中间件靠它记账)再 resolve。 +function run(mw, { method = 'GET', url = '/jobs', headers = {}, status = 200 } = {}, handler) { + return new Promise((resolve) => { + const req = { method, url, originalUrl: url, headers }; + const listeners = {}; + const res = { + statusCode: status, + on(ev, cb) { (listeners[ev] = listeners[ev] || []).push(cb); return this; }, + }; + mw(req, res, () => { + res.statusCode = status; + if (handler) handler(); + for (const cb of listeners.finish || []) cb(); + resolve(); + }); + }); +} + +test('注入 request_id(沿用入站头)', async () => { + let got = null; + const mw = makeMiddleware({}); + await run(mw, { headers: { 'x-request-id': 'inbound-1' } }, () => { + got = context.requestId(); + }); + assert.strictEqual(got, 'inbound-1'); +}); + +test('无入站头则生成 request_id 且允许 community resolver', async () => { + let got = null; + const mw = makeMiddleware({ resolveCommunity: () => 'openeuler' }); + await run(mw, {}, () => { + got = { rid: context.requestId(), comm: context.community() }; + }); + assert.ok(got.rid); + assert.strictEqual(got.comm, 'openeuler'); +}); + +test('服务器指标带公共 label 与 community', async () => { + const m = new Metrics({ service: 'review', env: 'test', instance: 'pod-1', + community: 'openeuler' }); + const mw = makeMiddleware({ metrics: m, resolveCommunity: () => 'openeuler' }); + + await run(mw, { method: 'POST', url: '/jobs', status: 201 }); + + const text = await m.text(); + const rows = text.split('\n').filter((l) => /^http_server_requests_total\{/.test(l)); + assert.strictEqual(rows.length, 1); + assert.ok(rows[0].includes('service="review"')); + assert.ok(rows[0].includes('community="openeuler"')); + assert.ok(rows[0].includes('method="POST"')); + assert.ok(rows[0].includes('path="/jobs"')); + assert.ok(rows[0].includes('status_code="201"')); + assert.ok(rows[0].endsWith(' 1')); +}); + +test('community 双层注入:resolver 覆盖 > 部署默认', async () => { + const m = new Metrics({ service: 'review', community: 'openeuler' }); + // resolver 按 path 判定(演示可信判定点);/mindspore 走覆盖,其余默认。 + const mw = makeMiddleware({ + metrics: m, + resolveCommunity: (req) => (req.url.startsWith('/mindspore') ? 'mindspore' : undefined), + }); + + await run(mw, { url: '/jobs' }); + await run(mw, { url: '/mindspore/jobs' }); + + const text = await m.text(); + const rows = text.split('\n').filter((l) => /^http_server_requests_total\{/.test(l)); + const byComm = {}; + for (const r of rows) byComm[r.match(/community="([^"]+)"/)[1]] = r; + assert.ok(byComm.openeuler); // 无覆盖 path → 部署默认 + assert.ok(byComm.mindspore); // /mindspore → 覆盖 +}); diff --git a/python/README.md b/python/README.md index a4ff167..0fbf73e 100644 --- a/python/README.md +++ b/python/README.md @@ -1 +1,72 @@ -# python — obs-sdk-python:Python SDK。log/ + metrics/(prometheus-client)。待 #2061 实现。 +# obs-sdk-python + +opensourceways 微服务可观测薄封装 SDK 的 Python 实现,契约见 [spec/](../spec/README.md)。 + +- **日志**:`obs_sdk.log` —— 结构化 JSON(root logger 挂唯一 JsonHandler,字段规范见 spec/log-format.md) +- **指标**:`obs_sdk.metrics` —— prometheus-client 薄封装,自带独立 CollectorRegistry +- **请求上下文**:`obs_sdk._context` —— `contextvars` 承载 `community/request_id/trace_id` +- **框架适配**:`obs_sdk.middleware` —— FastAPI / Flask / Django 中间件(注入 request_id + 可信判定点解析 community) +- **community 双层注入**:`service/env/instance` 常驻 const;`community` 可变 —— 请求上下文覆盖(`_context.bind`),未覆盖回退部署默认(`OBS_*` 环境变量) + +## 日志用法 + +```python +import logging +from obs_sdk import log + +# 字段空则回退 OBS_SERVICE / OBS_ENV / OBS_INSTANCE / OBS_COMMUNITY;进程内幂等 +log.init(service="review", env="test", instance="pod-1", community="openeuler") + +logger = log.get_logger(__name__) # 命名 logger,propagate 到 root 的 JSON handler +logger.info("job done", extra={"event": "release", "issue": "2061"}) +# 或便捷函数:log.info("job done", extra=...) +``` + +单条 JSON 行固定键 `service/env/instance/community/level/time/msg` + extra;请求级字段经上下文覆盖。 + +## 指标用法 + +```python +from obs_sdk import metrics + +metrics.init(service="review", env="test", instance="pod-1", community="openeuler") + +built = metrics.counter("built_releases", "发布的构建数", ["kind"]) # 业务 label;community 自动补 +built.inc(1, kind="tag") +metrics.gauge("in_flight", "在飞请求数").set(3) +metrics.histogram("review_duration", "评审耗时", ["api"]).observe(0.2) + +# FastAPI 暴露 /metrics: +# from obs_sdk import metrics +# return Response(content=metrics.generate_text(), media_type=metrics.content_type()) +``` + +`metrics.init()` 默认单例读 `OBS_*` 环境变量;多注册表场景直接 `Metrics(...)`。 + +## 请求上下文 / 中间件(community 双层注入) + +请求级 community 只在**服务可信判定点**(路由前缀 / 认证主体 / 白名单)解析后显式 bind, +不从不加鉴别的 URL / Header 盲取(见 spec/community-values.md)。 + +```python +from obs_sdk.middleware import fastapi_wrap, flask_middleware, DjangoMiddleware + +# FastAPI:wrap 原 app +app = fastapi_wrap(app, resolver=lambda req: "mindspore" if req.url.path.startswith("/mindspore") else None) + +# Flask +flask_middleware(app, resolver=lambda: "openeuler") # 单社区可不传 resolver + +# Django(MIDDLEWARE 加 ObsMiddleware,子类里可覆写 resolve_community) +MIDDLEWARE = [..., "obs_sdk.middleware.DjangoMiddleware"] +``` + +中间件会注入 `request_id`(沿用 `X-Request-Id` 或生成)并 push 请求上下文;请求内日志 / 指标自动带覆盖值, +处理结束上下文还原。 + +## 验证 + +```bash +pip install -e "python[test,fastapi,flask,django]" # 或 virtualenv 装 obs_sdk + 框架 +cd python && pytest +``` diff --git a/python/log/README.md b/python/log/README.md deleted file mode 100644 index 4ef3b44..0000000 --- a/python/log/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# log package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/python/metrics/README.md b/python/metrics/README.md deleted file mode 100644 index bd09d8c..0000000 --- a/python/metrics/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# metrics package -> 骨架占位。具体实现见 [#2061](https://github.com/opensourceways/backlog/issues/2061)。 diff --git a/python/obs_sdk/__init__.py b/python/obs_sdk/__init__.py new file mode 100644 index 0000000..9ce37e9 --- /dev/null +++ b/python/obs_sdk/__init__.py @@ -0,0 +1,12 @@ +"""obs-sdk-python:opensourceways 微服务可观测薄封装 SDK(log + metrics)。 + +契约见仓库顶层 `spec/`,本包按 spec 装配: + - `obs_sdk.log` 结构化 JSON 日志(字段规范 spec/log-format.md) + - `obs_sdk.metrics` Prometheus 指标(label 规范 spec/metrics-format.md) + - `obs_sdk._context` 请求级字段 bind(community 双层注入,spec/common-fields.md) + - `obs_sdk.middleware` 框架适配器(FastAPI / Flask / Django) +""" + +from . import _context, log, metrics, middleware + +__all__ = ["log", "metrics", "middleware"] diff --git a/python/obs_sdk/_context.py b/python/obs_sdk/_context.py new file mode 100644 index 0000000..c3847a4 --- /dev/null +++ b/python/obs_sdk/_context.py @@ -0,0 +1,69 @@ +"""请求级通用字段的 context 读写(Python 版 sdkctx)。 + +用 contextvars 实现(线程 / async 各自隔离,与 spec/common-fields.md 一致): + - community 覆盖值 / request_id / trace_id 存于当前 Context; + - 框架适配器在入口把可信解析出的字段 bind 进 Context,退出时 reset; + - log / metrics 读取当前 Context,未绑定则回退部署级默认。 +""" + +from __future__ import annotations + +import contextvars +import dataclasses +from contextlib import contextmanager +from typing import Iterator + + +@dataclasses.dataclass +class Request: + """请求级通用字段。空串 / None 表示未设置。""" + + community: str | None = None + request_id: str | None = None + trace_id: str | None = None + + +_current: contextvars.ContextVar[Request] = contextvars.ContextVar( + "obs_request", default=Request() +) + + +def get() -> Request: + """读取当前请求级字段;未设置返回空 Request。""" + return _current.get() + + +def community() -> str | None: + """当前 community 覆盖值;未设置返回 None。""" + return _current.get().community + + +def request_id() -> str | None: + return _current.get().request_id + + +def trace_id() -> str | None: + return _current.get().trace_id + + +@contextmanager +def bind(*, community: str | None = None, request_id: str | None = None, + trace_id: str | None = None) -> Iterator[None]: + """把请求级字段 bind 进当前 Context;退出自动 reset。 + + 用于框架适配器入口 / 业务可信判定点: + + with bind(community="openeuler", request_id=req_id): + logger.info("...") # 自动带 community/request_id + """ + prev = _current.get() + merged = Request( + community=community if community is not None else prev.community, + request_id=request_id if request_id is not None else prev.request_id, + trace_id=trace_id if trace_id is not None else prev.trace_id, + ) + token = _current.set(merged) + try: + yield + finally: + _current.reset(token) diff --git a/python/obs_sdk/_env.py b/python/obs_sdk/_env.py new file mode 100644 index 0000000..bfe62c4 --- /dev/null +++ b/python/obs_sdk/_env.py @@ -0,0 +1,40 @@ +"""OBS_* 环境变量解析辅助(与 go/internal/env 语义一致,见 spec/common-fields.md)。 + +四语言 SDK 读同一套环境变量:OBS_SERVICE / OBS_ENV / OBS_INSTANCE / OBS_COMMUNITY。 +解析优先级:显式参数 > 环境变量 > 内置默认。 +""" + +from __future__ import annotations + +import os +import socket + + +def _resolve(explicit: str | None, env_name: str, default: str) -> str: + if explicit: + return explicit + value = os.getenv(env_name) + if value: + return value + return default + + +def service(explicit: str | None = None) -> str: + return _resolve(explicit, "OBS_SERVICE", "unknown") + + +def env(explicit: str | None = None) -> str: + return _resolve(explicit, "OBS_ENV", "unknown") + + +def instance(explicit: str | None = None) -> str: + if explicit: + return explicit + value = os.getenv("OBS_INSTANCE") + if value: + return value + return socket.gethostname() or "unknown" + + +def community(explicit: str | None = None) -> str: + return _resolve(explicit, "OBS_COMMUNITY", "unknown") diff --git a/python/obs_sdk/log.py b/python/obs_sdk/log.py new file mode 100644 index 0000000..07f6cd3 --- /dev/null +++ b/python/obs_sdk/log.py @@ -0,0 +1,165 @@ +"""结构化 JSON 日志(obs-sdk-python 的 log 部分)。 + +格式遵循 spec/log-format.md: + - 单行 JSON,经 stdout 进 LTS; + - 常驻字段 service/env/instance/community 在 init 时注入; + - 请求级 community 覆盖 / request_id / trace_id 从 _context 读取; + - trace_id 预留位(有值才输出,二期经 _context.bind(trace_id=...) 注入)。 + +用法: + + from obs_sdk import log + log.init(service="meeting-center") + logger = log.get_logger(__name__) + logger.info("hello", extra={"event": "xxx"}) + +请求级覆盖(框架适配器已在入口 bind): + + with _context.bind(community="openeuler", request_id="req-1"): + logger.info("scoped") +""" + +from __future__ import annotations + +import json +import logging +import sys +import time +from typing import Any, Optional + +from . import _context, _env + +# 标准字段,不当作业务字段输出。 +_RESERVED = { + "name", "msg", "args", "levelname", "levelno", "pathname", "filename", + "module", "exc_info", "exc_text", "stack_info", "lineno", "funcName", + "created", "msecs", "relativeCreated", "thread", "threadName", + "processName", "process", "taskName", "message", +} + +# 标识本 SDK 挂在 logger 上的 handler。 +_SDK_HANDLER_NAME = "obs-sdk-json" + + +class JsonFormatter(logging.Formatter): + """把日志记录格式化为单行 JSON。""" + + def __init__(self, *, service: str, env: str, instance: str, + community: str) -> None: + super().__init__() + self._service = service + self._env = env + self._instance = instance + self._default_community = community + + def format(self, record: logging.LogRecord) -> str: + # 先收业务 extra(不得覆盖常驻字段,见下)。 + fields: dict[str, Any] = {} + for key, value in record.__dict__.items(): + if key in _RESERVED or not isinstance(key, str) or key.startswith("_"): + continue + fields[key] = _serialize(value) + + # 请求级字段优先于静态默认。 + req = _context.get() + community = req.community or self._default_community + if req.request_id: + fields["request_id"] = req.request_id + if req.trace_id: + fields["trace_id"] = req.trace_id + + # 常驻字段最后写入 → 覆盖同名 extra,保证统一。 + fields.update({ + "service": self._service, + "env": self._env, + "instance": self._instance, + "community": community, + "level": record.levelname.lower(), + "msg": record.getMessage(), + "time": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + + f".{int(record.msecs):03d}Z", + }) + + # 异常堆栈附 error 字段。 + if record.exc_info: + fields["error"] = self.formatException(record.exc_info) + + return json.dumps(fields, ensure_ascii=False, default=str) + + +def _serialize(value: Any) -> Any: + try: + json.dumps(value) + return value + except (TypeError, ValueError): + return str(value) + + +class JsonHandler(logging.StreamHandler): + """把日志写到 stream 的 handler,自动用 JsonFormatter。""" + + def __init__(self, stream: Any, *, service: str, env: str, instance: str, + community: str) -> None: + super().__init__(stream) + self.name = _SDK_HANDLER_NAME + self.setFormatter(JsonFormatter( + service=service, env=env, instance=instance, community=community)) + + +# 进程级配置。get_logger 延迟 init 到首次调用。 +_defaults: dict[str, str] | None = None +_log_level = logging.INFO + + +def init(*, service: Optional[str] = None, env: Optional[str] = None, + instance: Optional[str] = None, community: Optional[str] = None, + level: str = "info", stream: Any = sys.stdout) -> None: + """初始化日志:注入常驻字段,在 root logger 挂 JSON handler。进程内幂等。 + + 重复 init 会用新配置重建(替换旧 SDK handler)。 + """ + global _defaults, _log_level + _defaults = { + "service": _env.service(service), + "env": _env.env(env), + "instance": _env.instance(instance), + "community": _env.community(community), + } + _log_level = getattr(logging, level.upper(), logging.INFO) + + root = logging.getLogger() + root.setLevel(logging.DEBUG) # 过滤交给 formatter 层 SDK 自己的 handler 级别控制 + + # 移除旧 SDK handler,挂新配置的。 + for h in list(root.handlers): + if getattr(h, "name", None) == _SDK_HANDLER_NAME: + root.removeHandler(h) + handler = JsonHandler(stream, **_defaults) + handler.setLevel(_log_level) + root.addHandler(handler) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """返回一个 logger(命名或 root)。命名 logger 经 propagate 落到 root 的 + JSON handler,单条日志只输出一次。""" + if _defaults is None: + init() + return logging.getLogger(name) + + +# --- 便捷函数(命名 = 调用方模块名) --- + +def debug(msg: str, *args: Any, **kwargs: Any) -> None: + get_logger().debug(msg, *args, **kwargs) + + +def info(msg: str, *args: Any, **kwargs: Any) -> None: + get_logger().info(msg, *args, **kwargs) + + +def warn(msg: str, *args: Any, **kwargs: Any) -> None: + get_logger().warning(msg, *args, **kwargs) + + +def error(msg: str, *args: Any, **kwargs: Any) -> None: + get_logger().error(msg, *args, **kwargs) diff --git a/python/obs_sdk/metrics.py b/python/obs_sdk/metrics.py new file mode 100644 index 0000000..2de0956 --- /dev/null +++ b/python/obs_sdk/metrics.py @@ -0,0 +1,164 @@ +"""Prometheus 指标装配(obs-sdk-python 的 metrics 部分)。 + +底层用官方库 prometheus-client,SDK 不自研 instrumentation,只做装配与 +命名/label 对齐(见 spec/metrics-format.md): + - 所有指标自动带固定 label:service / env / instance / community; + - community 是唯一可动态的公共 label:ctx 覆盖优先,否则部署默认 + (双层注入,"注册一次,两用"); + - 统一暴露文本输出(metrics.text() / 便捷 generate_text())。 + +用法: + + from obs_sdk import metrics + metrics.init(service="meeting-center") # 或读 OBS_* 环境变量 + c = metrics.counter("meeting_created_total", "created meetings", "kind") + c.inc(kind="scheduled") # community = ctx 覆盖或部署默认 + c.inc(2, kind="cancelled") # 业务 label 走关键字/位置均可 + + # /metrics 文本: + from obs_sdk import metrics + text = metrics.generate_text() +""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +from prometheus_client import (CONTENT_TYPE_LATEST, CollectorRegistry, + Counter, Gauge, Histogram, generate_latest) + +from . import _context, _env + +# 常驻公共 label 键。 +_COMMON = ["service", "env", "instance", "community"] + + +class _Vec: + """Vec 薄封装:注册时声明业务 label,打点时自动填公共维度。""" + + def __init__(self, metric, common_values: Dict[str, str]) -> None: + self._metric = metric + self._common = common_values # service/env/instance 值(community 动态取) + + def _labels(self, labelvalues: Dict[str, str]) -> object: + req = _context.get() + community = req.community or self._common["community"] + vals = dict(self._common) + vals["community"] = community + vals.update(labelvalues) + return self._metric.labels(**vals) + + # --- counter --- + + def inc(self, amount: float = 1.0, **labelvalues: str) -> None: + self._labels(labelvalues).inc(amount) + + # --- gauge --- + + def set(self, value: float, **labelvalues: str) -> None: + self._labels(labelvalues).set(value) + + # --- histogram --- + + def observe(self, value: float, **labelvalues: str) -> None: + self._labels(labelvalues).observe(value) + + +class Metrics: + """装配入口:持有独立 CollectorRegistry,避免全局注册表相互污染。""" + + def __init__(self, *, service: str, env: str, instance: str, + community: str, namespace: str = "") -> None: + self._common = {"service": service, "env": env, "instance": instance, + "community": community} + self._namespace = namespace + self._registry = CollectorRegistry(auto_describe=True) + + @property + def registry(self) -> CollectorRegistry: + return self._registry + + @property + def default_community(self) -> str: + return self._common["community"] + + def _fq(self, name: str) -> str: + return f"{self._namespace}_{name}" if self._namespace else name + + def _collector(self, cls, name: str, documentation: str, + labelnames: List[str], **kw) -> _Vec: + metric = cls(self._fq(name), documentation, + labelnames=[*_COMMON, *labelnames], + registry=self._registry, **kw) + return _Vec(metric, self._common) + + def counter(self, name: str, documentation: str, + labelnames: Optional[List[str]] = None) -> _Vec: + return self._collector(Counter, name, documentation, list(labelnames or [])) + + def gauge(self, name: str, documentation: str, + labelnames: Optional[List[str]] = None) -> _Vec: + return self._collector(Gauge, name, documentation, list(labelnames or [])) + + def histogram(self, name: str, documentation: str, + labelnames: Optional[List[str]] = None, + buckets: Optional[List[float]] = None) -> _Vec: + kw: Dict = {} + if buckets is not None: + kw["buckets"] = buckets + return self._collector(Histogram, name, documentation, + list(labelnames or []), **kw) + + def text(self) -> bytes: + """输出该注册表 Prometheus text 格式。""" + return generate_latest(self._registry) + + +# 进程级便捷单例(默认读 OBS_* 环境变量)。 +_default: Optional[Metrics] = None + + +def init(*, service: Optional[str] = None, env: Optional[str] = None, + instance: Optional[str] = None, community: Optional[str] = None, + namespace: str = "") -> Metrics: + """初始化(进程内单例)。空字段读 OBS_* 环境变量。""" + global _default + if _default is None: + _default = Metrics( + service=_env.service(service), + env=_env.env(env), + instance=_env.instance(instance), + community=_env.community(community), + namespace=namespace, + ) + return _default + + +def default() -> Metrics: + if _default is None: + return init() + return _default + + +def counter(name: str, documentation: str, + labelnames: Optional[List[str]] = None) -> _Vec: + return default().counter(name, documentation, labelnames) + + +def gauge(name: str, documentation: str, + labelnames: Optional[List[str]] = None) -> _Vec: + return default().gauge(name, documentation, labelnames) + + +def histogram(name: str, documentation: str, + labelnames: Optional[List[str]] = None) -> _Vec: + return default().histogram(name, documentation, labelnames) + + +def generate_text() -> bytes: + """输出默认注册表 Prometheus text(供 /metrics 路由)。""" + return default().text() + + +def content_type() -> str: + return CONTENT_TYPE_LATEST diff --git a/python/obs_sdk/middleware.py b/python/obs_sdk/middleware.py new file mode 100644 index 0000000..c6b95e6 --- /dev/null +++ b/python/obs_sdk/middleware.py @@ -0,0 +1,138 @@ +"""框架适配器:在入口把可信解析的请求级字段 bind 进 context,并挂 /metrics。 + +设计约束(spec/common-fields.md):community 请求覆盖必须来自可信判定点 +(路由前缀 / 认证主体 / 白名单)。SDK 提供 bind 入口,由业务提供 +community_resolver;SDK 不裸透传外部入参。 + +三个适配器共享同一套「取入站 request_id + 可选 community」逻辑,区别仅在 +框架挂载 API。框架未安装时导入对应函数会 ImportError,业务按其实际依赖 +选装(见 pyproject optional-dependencies)。 + +/metrics 路由不在本文件实现:业务按各自框架加一条返回 +metrics.generate_text() 的路由即可(见各函数 docstring 示例)。 +""" + +from __future__ import annotations + +import uuid +from typing import Callable, Optional + +from . import _context + +# 可信入站请求 ID 头(沿用 Go 版常量)。 +HEADER_REQUEST_ID = "X-Request-Id" + +# community 解析函数:入参为各框架 request 对象,返回社区字符串或 None。 +Resolver = Callable[[object], Optional[str]] + + +def make_request_id(inbound: Optional[str]) -> str: + """沿用入站 request_id,否则生成。""" + return inbound or uuid.uuid4().hex + + +def bind_request(request, community: Optional[str]) -> "_context._GeneratorContextManager": + """构造 _context.bind 上下文管理器(含 request_id 生成)。 + + request 需提供 .headers(dict 兼容,含 .get)。 + """ + rid = request.headers.get(HEADER_REQUEST_ID) + return _context.bind(community=community, request_id=make_request_id(rid)) + + +# --------------------------------------------------------------------------- +# FastAPI(ASGI,基于 starlette BaseHTTPMiddleware)。 +# --------------------------------------------------------------------------- + + +def fastapi_wrap(app, resolver: Optional[Resolver] = None): + """为 FastAPI app 挂载观测中间件,返回原 app(已加中间件)。 + + 用法: + + from fastapi import FastAPI, Response + from obs_sdk import metrics, middleware as mw + + app = FastAPI() + mw.fastapi_wrap(app, resolver=lambda request: "openeuler") + + @app.get("/metrics") + def metrics_route(): + return Response(metrics.generate_text(), media_type=metrics.content_type()) + """ + from starlette.middleware.base import BaseHTTPMiddleware + + class ObsMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + community = resolver(request) if resolver else None + with bind_request(request, community): + return await call_next(request) + + app.add_middleware(ObsMiddleware) + return app + + +# --------------------------------------------------------------------------- +# Flask(WSGI,before/teardown)。 +# --------------------------------------------------------------------------- + + +def flask_middleware(app, resolver: Optional[Resolver] = None): + """为 Flask app 挂载观测 before/after 钩子。 + + 用法: + + from flask import Flask, Response + from obs_sdk import metrics, middleware as mw + + app = Flask(__name__) + mw.flask_middleware(app, resolver=lambda request: "openeuler") + + @app.get("/metrics") + def metrics_route(): + return Response(metrics.generate_text(), mimetype=metrics.content_type()) + """ + import flask + + @app.before_request + def _obs_before(): + community = resolver(flask.request) if resolver else None + ctx = bind_request(flask.request, community) + flask.g._obs_ctx = ctx + ctx.__enter__() + + @app.teardown_request + def _obs_teardown(exc=None): + ctx = getattr(flask.g, "_obs_ctx", None) + if ctx is not None: + ctx.__exit__(None, None, None) + + return app + + +# --------------------------------------------------------------------------- +# Django(WSGI middleware)。 +# --------------------------------------------------------------------------- + + +class DjangoMiddleware: + """Django 观测中间件(请求级字段 bind + request_id 注入)。 + + 用法(settings.MIDDLEWARE 追加): + + "obs_sdk.middleware.DjangoMiddleware", + + 多社区中心化服务覆写 resolve_community(request) 按可信来源返回社区。 + """ + + def __init__(self, get_response: Callable): + self.get_response = get_response + + def __call__(self, request): + community = self.resolve_community(request) + with bind_request(request, community): + return self.get_response(request) + + def resolve_community(self, request) -> Optional[str]: + """覆写点:按可信来源返回社区;默认不覆盖(用部署级默认)。""" + return None diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..b52e6b6 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "obs-sdk-python" +version = "0.1.0" +description = "opensourceways 微服务可观测薄封装 SDK(Python)——结构化 JSON 日志 + Prometheus 指标,契约见 ../spec" +readme = "README.md" +requires-python = ">=3.9" +license = { text = "Apache-2.0" } +dependencies = [ + "prometheus-client>=0.20", +] + +[project.optional-dependencies] +fastapi = ["fastapi>=0.100"] +flask = ["flask>=2.0"] +django = ["django>=4.0"] +test = [ + "pytest>=8.0", + # starlette.testclient(FastAPI 中间件单测)的传输依赖;不装则 testclient 抛 RuntimeError + "httpx2>=2.0.0", +] + +[tool.setuptools.packages.find] +include = ["obs_sdk*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/python/tests/test_log.py b/python/tests/test_log.py new file mode 100644 index 0000000..f70d90f --- /dev/null +++ b/python/tests/test_log.py @@ -0,0 +1,119 @@ +"""log 模块单测:静态字段注入、community 双层注入、level 过滤、trace_id 预留。""" + +import io +import json +import logging +from typing import Dict, List + +import pytest + +from obs_sdk import log + + +@pytest.fixture(autouse=True) +def _reset_log(): + # 每个用例前重置进程级配置,保证测试隔离。 + log._defaults = None + log._log_level = logging.INFO + root = logging.getLogger() + for h in list(root.handlers): + if getattr(h, "name", None) == log._SDK_HANDLER_NAME: + root.removeHandler(h) + yield + # 恢复默认 stdout,避免污染后续。 + log._defaults = None + for h in list(root.handlers): + if getattr(h, "name", None) == log._SDK_HANDLER_NAME: + root.removeHandler(h) + + +def _capture(service: str = "srv", community: str = "openeuler", + level: str = "info") -> List[Dict]: + buf = io.StringIO() + log.init(service=service, community=community, level=level, stream=buf) + return buf + + +def _lines(buf: io.StringIO) -> List[Dict]: + return [json.loads(line) for line in buf.getvalue().strip().splitlines()] + + +def test_static_fields_injected(): + buf = _capture(service="review", community="openeuler") + logger = log.get_logger("t") + logger.info("hello", extra={"event": "pull_request"}) + + lines = _lines(buf) + assert len(lines) == 1 + f = lines[0] + assert f["service"] == "review" + assert f["community"] == "openeuler" + assert f["level"] == "info" + assert f["msg"] == "hello" + assert f["event"] == "pull_request" + # 无请求上下文时不输出 request_id / trace_id。 + assert "request_id" not in f + assert "trace_id" not in f + + +def test_business_extra_overrides_nothing_common(): + buf = _capture(service="review") + logger = log.get_logger("t") + logger.info("x", extra={"service": "fake", "k": 1}) + f = _lines(buf)[0] + # 业务 extra 不得覆盖常驻字段。 + assert f["service"] == "review" + assert f["k"] == 1 + + +def test_request_community_override(): + from obs_sdk import _context + buf = _capture(community="openeuler") + logger = log.get_logger("t") + with _context.bind(community="mindspore", request_id="req-1"): + logger.info("scoped") + f = _lines(buf)[0] + assert f["community"] == "mindspore" + assert f["request_id"] == "req-1" + + +def test_level_filter(): + buf = _capture(service="srv", level="warn") + logger = log.get_logger("t") + logger.info("dropped") + logger.warning("kept") + lines = _lines(buf) + assert len(lines) == 1 + assert lines[0]["msg"] == "kept" + + +def test_trace_id_reserved_inject(): + from obs_sdk import _context + buf = _capture(community="openeuler") + logger = log.get_logger("t") + with _context.bind(trace_id="trace-xyz"): + logger.info("with trace") + f = _lines(buf)[0] + assert f["trace_id"] == "trace-xyz" + assert f["community"] == "openeuler" + + +def test_error_field_on_exception(): + buf = _capture(service="srv") + logger = log.get_logger("t") + try: + raise ValueError("boom") + except ValueError: + logger.error("failed", exc_info=True) + f = _lines(buf)[0] + assert f["level"] == "error" + assert "boom" in f["error"] + + +def test_json_time_format(): + buf = _capture(service="srv") + log.get_logger("t").info("x") + f = _lines(buf)[0] + # RFC3339 UTC,毫秒级:2026-09-08T07:12:34.123Z + import re + assert re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$", f["time"]) diff --git a/python/tests/test_metrics.py b/python/tests/test_metrics.py new file mode 100644 index 0000000..043e81d --- /dev/null +++ b/python/tests/test_metrics.py @@ -0,0 +1,83 @@ +"""metrics 模块单测:公共 label 注入、community 双层注入、text 输出。""" + +import re + +import pytest + +from obs_sdk import _context, metrics + + +@pytest.fixture(autouse=True) +def _clean_default(): + # 每个用例重置全局单例,避免注册表重复名称冲突。 + metrics._default = None + yield + metrics._default = None + + +def _new(**cfg): + defaults = dict(service="srv", env="test", instance="pod-1", + community="openeuler", namespace="") + defaults.update(cfg) + return metrics.Metrics(**defaults) + + +def test_counter_common_labels(): + m = _new() + c = m.counter("events_total", "events", ["kind"]) + c.inc(kind="pr") + + text = m.text().decode() + assert "# HELP events_total events" in text + rows = [l for l in text.splitlines() + if re.match(r"^events_total\{", l)] + assert len(rows) == 1 + assert 'service="srv"' in rows[0] + assert 'env="test"' in rows[0] + assert 'instance="pod-1"' in rows[0] + assert 'community="openeuler"' in rows[0] + assert 'kind="pr"' in rows[0] + assert rows[0].endswith(" 1.0") + + +def test_community_double_layer(): + m = _new(community="openeuler") + c = m.counter("events_total", "events", ["kind"]) + + c.inc(kind="pr") # 默认 openeuler + with _context.bind(community="mindspore"): + c.inc(kind="pr") # ctx 覆盖 → mindspore + + rows = [l for l in m.text().decode().splitlines() + if re.match(r"^events_total\{", l)] + assert len(rows) == 2 + by_comm = {r.split('community="')[1].split('"')[0]: r for r in rows} + assert by_comm["openeuler"].endswith(" 1.0") + assert by_comm["mindspore"].endswith(" 1.0") + + +def test_gauge_and_histogram(): + m = _new() + g = m.gauge("in_flight", "in flight") + g.set(3) + g.inc(2) + rows = [l for l in m.text().decode().splitlines() + if re.match(r"^in_flight\{", l)] + assert rows[0].endswith(" 5.0") + + h = m.histogram("latency_seconds", "latency") + h.observe(0.1) + h.observe(0.2) + text = m.text().decode() + assert "latency_seconds_count" in text + count = [l for l in text.splitlines() if re.match(r"^latency_seconds_count\{", l)] + assert count[0].endswith(" 2.0") + + +def test_metrics_singleton_env(): + # 便捷单例 init 幂等。 + metrics.init(service="srv", community="ascend") + a = metrics.default() + metrics.init(service="ignored") # 已建不重建 + assert metrics.default() is a + assert metrics.default().default_community == "ascend" diff --git a/python/tests/test_middleware.py b/python/tests/test_middleware.py new file mode 100644 index 0000000..2482d28 --- /dev/null +++ b/python/tests/test_middleware.py @@ -0,0 +1,124 @@ +"""middleware 适配器单测:入口 bind community / request_id,请求内日志可见。""" + +import io +import json +from typing import Dict, List + +import pytest + +from obs_sdk import _context, log, middleware as mw + + +def _fresh_log(buf) -> None: + log._defaults = None + log._log_level = 0 # INFO 实际由 init 重设 + import logging + root = logging.getLogger() + for h in list(root.handlers): + if getattr(h, "name", None) == log._SDK_HANDLER_NAME: + root.removeHandler(h) + log.init(service="review", community="openeuler", stream=buf) + + +def _captured(buf: io.StringIO) -> List[Dict]: + return [json.loads(l) for l in buf.getvalue().strip().splitlines() + if l.strip()] + + +# --- FastAPI --- + +fastapi = pytest.importorskip("fastapi") + + +def test_fastapi_middleware_binds_context(): + buf = io.StringIO() + _fresh_log(buf) + + from fastapi import FastAPI + from starlette.testclient import TestClient + + app = FastAPI() + mw.fastapi_wrap(app, resolver=lambda req: "openeuler") + + @app.get("/ping") + def ping(): + log.get_logger("t").info("inside handler") + return {"ok": True} + + client = TestClient(app) + resp = client.get("/ping", headers={mw.HEADER_REQUEST_ID: "req-fastapi"}) + assert resp.status_code == 200 + + lines = [l for l in _captured(buf) if l.get("msg") == "inside handler"] + assert len(lines) == 1 + assert lines[0]["community"] == "openeuler" + assert lines[0]["request_id"] == "req-fastapi" + + +# --- Flask --- + +flask = pytest.importorskip("flask") + + +def test_flask_middleware_binds_context(): + buf = io.StringIO() + _fresh_log(buf) + + app = flask.Flask(__name__) + mw.flask_middleware(app, resolver=lambda req: "mindspore") + + @app.get("/ping") + def ping(): + log.get_logger("t").info("inside handler") + return "ok" + + with app.test_client() as c: + r = c.get("/ping", headers={mw.HEADER_REQUEST_ID: "req-flask"}) + assert r.status_code == 200 + + lines = [l for l in _captured(buf) if l.get("msg") == "inside handler"] + assert len(lines) == 1 + assert lines[0]["community"] == "mindspore" + assert lines[0]["request_id"] == "req-flask" + + +# --- Django --- + +django = pytest.importorskip("django") + + +def test_django_middleware_binds_context(): + buf = io.StringIO() + _fresh_log(buf) + + from django.conf import settings + if not settings.configured: + settings.configure( + DEBUG=True, + ALLOWED_HOSTS=["testserver"], + ROOT_URLCONF=__name__, + MIDDLEWARE=["obs_sdk.middleware.DjangoMiddleware"], + ) + import django + django.setup() + + from django.http import HttpResponse + + def ping(request): + log.get_logger("t").info("inside django view") + return HttpResponse("ok") + + from django.urls import path, resolve + from django.test import RequestFactory + + req = RequestFactory().get("/ping", HTTP_X_REQUEST_ID="req-django") + # 直接经中间件调用视图。 + get_response = lambda r: ping(r) # noqa: E731 + mw_instance = mw.DjangoMiddleware(get_response) + resp = mw_instance(req) + assert resp.status_code == 200 + + lines = [l for l in _captured(buf) if l.get("msg") == "inside django view"] + assert len(lines) == 1 + assert lines[0]["request_id"] == "req-django" + assert lines[0]["community"] == "openeuler" # 未覆写 → 部署默认 diff --git a/spec/README.md b/spec/README.md index 4ea510b..cf20394 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,9 +1,35 @@ -# spec — 契约层(单一事实来源) +# spec — 可观测契约层(单一事实来源) -> 骨架占位。内容待 [#2061](https://github.com/opensourceways/backlog/issues/2061) 实现。 +> 需求:[opensourceways/backlog#1938](https://github.com/opensourceways/backlog/issues/1938) · 子任务 A:[#2061](https://github.com/opensourceways/backlog/issues/2061) +> 本目录是 **log / metrics 输出格式的唯一权威定义**。`go/` `python/` `java/` `node/` 四个语言 SDK 必须按此实现;不一致时以本目录为准。 -定义: -- 日志 JSON schema(结构、级别) -- 通用字段:`service` / `env` / `instance` / `community` / `request_id` / `trace_id` -- 指标命名 prefix 与 label 规范(含 `community` 双层注入:部署级默认 + 请求级覆盖) -- community 取值枚举(遵循 service.md:Ascend / CANN / MindSpore / openEuler / openGauss 等) +## 设计原则(摘自 #2061 范围边界与设计约束) + +1. **首期只做 log + metrics,不含 trace**;`trace_id` 字段预留注入位,二期经 context 平滑接入。 +2. **SDK 是薄封装,不自研 instrumentation**:指标底层引用各语言官方库(Go→client_golang、Python→prometheus-client、Java→micrometer/actuator、Node→prom-client),SDK 只做装配(中间件 / 通用字段注入 / 命名对齐)。 +3. **接口抽象 + Init() 一行装配**:业务代码依赖 SDK 暴露的接口/方法,不依赖具体 instrumentation 实现。 +4. **community 双层注入**:日志字段与指标 label 均含 `community`;部署级默认 + 请求级动态覆盖两套场景共用同一注册。 + +## 目录 + +| 文件 | 内容 | +| --- | --- | +| [log-format.md](log-format.md) | 结构化日志 JSON 格式、级别、通用字段、`trace_id` 预留位 | +| [metrics-format.md](metrics-format.md) | 指标命名前缀、label 规范、`community` 双层注入建模 | +| [common-fields.md](common-fields.md) | 通用字段 `service` / `env` / `instance` / `community` / `request_id` / `trace_id` 的来源与注入规则 | +| [community-values.md](community-values.md) | `community` 取值枚举(service.md 各社区段,随 service.md 更新) | + +## 语言 SDK 对应关系 + +| 语言目录 | SDK | log 底层 | metrics 底层 | 用法文档 | +| --- | --- | --- | --- | --- | +| `go/` | obs-sdk-go | stdlib `encoding/json`(加锁单行 JSON → stdout) | `client_golang` | [go/README](../go/README.md) | +| `python/` | obs-sdk-python | stdlib `logging`(自定 JSON Formatter) | `prometheus-client` | [python/README](../python/README.md) | +| `java/` | obs-sdk-java | `logback` + logstash JSON encoder | `micrometer` + prometheus registry | [java/README](../java/README.md) | +| `node/` | obs-sdk-node | 自定 JSON serializer(console → stdout) | `prom-client` | [node/README](../node/README.md) | + +## community 取值要点(详见 [community-values.md](community-values.md)) + +- 遵循 `opensourceways/infra-common` 的 `service.md` 社区枚举:`Ascend / BoostKit / CANN / HPCKit / MindSpore / openEuler / OpenFuyao / openGauss / OpenJiuwen / OpenUBMC / OpenPangu / HiFloat / UnifiedBus / Infrastructure / openLookeng / Xihe` 等。 +- 单社区独立部署的服务:`community` = 部署级静态值(环境变量 `OBS_COMMUNITY` 或 Init 配置),日志常驻字段、指标常驻 label。 +- 中心化单实例服务多社区:`community` 由请求上下文动态覆盖(路由按社区分发时设置),日志按请求打印、指标按请求打点。 diff --git a/spec/common-fields.md b/spec/common-fields.md new file mode 100644 index 0000000..f7de44b --- /dev/null +++ b/spec/common-fields.md @@ -0,0 +1,67 @@ +# 通用字段契约 — service / env / instance / community / request_id / trace_id + +> 6 个通用字段是所有 log 与 metrics 的公共标识维度。字段语义、来源、注入规则在此统一。 + +## 字段总表 + +| 字段 | 语义 | 类型 | 注入层级 | 静态/动态 | +| --- | --- | --- | --- | --- | +| `service` | 服务名(微服务名 / module 子目录名) | string | Init(进程级) | 静态 | +| `env` | 部署环境 | string | Init(进程级) | 静态 | +| `instance` | 实例标识(pod 名 / IP / hostname) | string | Init(进程级,推荐 k8s `metadata.name` 或 hostname) | 静态 | +| `community` | 社区标识 | string | Init(进程级默认) **+** 请求上下文(覆盖) | 双态 | +| `request_id` | 单请求关联 ID | string | 请求上下文 | 动态 | +| `trace_id` | 分布式 trace ID(二期接入) | string | 请求上下文(预留) | 动态/预留 | + +## 静态字段来源与默认值 + +进程启动 Init 时注入,进程生命周期不变。推荐按优先级读取: + +1. SDK Init 显式参数(最高优先级,测试/单实例可控); +2. 部署环境变量(见下); +3. 内置默认(`service=unknown`、`env=unknown`、`instance=hostname`)。 + +| 字段 | 推荐环境变量名 | 说明 | +| --- | --- | --- | +| `service` | `OBS_SERVICE` | 部署时由编排(helm/kustomize)写入 | +| `env` | `OBS_ENV` | prod / test / preview / staging | +| `instance` | `OBS_INSTANCE`(缺省取 hostname) | k8s 下可用 `metadata.name`(=pod 名)经环境变量注入 | +| `community` | `OBS_COMMUNITY` | **部署级默认 community**:单社区服务常驻该值 | + +> 变量名统一 `OBS_*` 前缀;四语言 SDK 均读同一套环境变量名,保证部署配置模板(helm values)不因语言而异。 + +## community 双层注入(本需求核心) + +两个场景,共用同一套字段/label 注册,靠取值来源区分: + +### 第一层:部署级默认注入(静态) + +- 场景:**服务按社区拆实例**(如 openEuler 一份、MindSpore 一份各自独立部署),每实例只服务一个社区。 +- 行为:Init 从 `OBS_COMMUNITY`(或显式参数)注入默认 community。 + - **日志**:作为常驻字段打底,出现在该进程所有日志行。 + - **指标**:作为**常驻 const label** 打进该进程所有时间序列。 +- 请求处理中若无请求级覆盖,一律使用该默认值。 + +### 第二层:请求上下文动态覆盖(动态) + +- 场景:**中心化单实例服务多社区**(一个部署同时服务 openEuler / MindSpore 等多个社区,靠路由/鉴权识别请求归属社区)。典型:community-robots 类中心 hub、message-bus。 +- 行为:SDK 中间件或业务代码在**请求上下文**里设置覆盖值 `community=xxx`;处理该请求时: + - **日志**:该请求关联的所有日志行 community 字段 = 覆盖值。 + - **指标**:该请求触发打点的时间序列其 community label = 覆盖值(需用动态 label,见 metrics-format.md)。 +- 覆盖值缺失 → 回退第一层默认值。 + +### 注入路径约束(安全相关) + +- **请求级 community 永不从不可信入参盲取**(如裸读 URL/Header 即当 community)。必须来自:服务自己按可信来源(路由路径前缀、认证后的主体、白名单表)判定后显式放入上下文的**类型化值**。SDK 只负责读上下文里**已经放好的类型化值**,不负责从外部请求原样透传。 +- 四个语言 SDK 提供一致的「往上下文放 community / 从上下文读 community」API,供业务在可信判定点写入。 + +## request_id 注入 + +- 来源优先级:可信入站头(如 `X-Request-Id`,**在服务已校验可信时**)> 中间件自动生成(UUID/雪花)> 无。 +- SDK 中间件:入口中间件若上下文无 request_id 则生成并写入;日志绑定上下文时带上。 +- 出站调用传播:作为头/字段传给下游服务(语言 SDK 提供 outbound 侧 helper),保证全链路同 ID。 + +## trace_id 预留位 + +- 二期接 OpenTelemetry 后,trace 经 context 注入;日志上下文里的 `trace_id` 即取自该 context。 +- 首期:`trace_id` 注入位必须存在(log 上下文 API 有对应字段位置、metrics label 有对应位但可省略),值恒空。目标:二期 trace 接入时**零日志格式返工**。 diff --git a/spec/community-values.md b/spec/community-values.md new file mode 100644 index 0000000..fb2abed --- /dev/null +++ b/spec/community-values.md @@ -0,0 +1,16 @@ +# community 取值枚举 + +> 权威来源:`opensourceways/infra-common` 仓 [`service.md`](https://github.com/opensourceways/infra-common/blob/master/service.md) 的**社区段**(section)。service.md 新增/改名社区时,本文件随服务接入同步更新。 +> `community` 字段/label 取值**必须落在此枚举内**(小写用连字符的取值为部署时约定别名,见下)。 + +## 取值(跟随 service.md 常用社区段) + +`Ascend` · `BoostKit` · `CANN` · `HPCKit` · `MindSpore` · `openEuler` · `OpenFuyao` · `openGauss` · `OpenJiuwen` · `OpenUBMC` · `OpenPangu` · `HiFloat` · `UnifiedBus` · `Infrastructure` · `openLookeng` · `Xihe` + +> 上表大小写跟随 service.md 原样。**通用约定:log/label 里统一转小写**以保跨服务可聚合(查询聚合大小写敏感)。例:service.md `Ascend` → 字段值 `ascend`;`openEuler` → `openeuler`;`MindSpore` → `mindspore`。 + +## 多社区 / 中心化部署 + +- 单社区独立部署:`OBS_COMMUNITY=<小写值>`。 +- 中心化单实例多社区:进程级无单一 community,请求级覆盖值为上表枚举内小写值之一。 +- 默认/未知:`unknown`(保留值,表示尚未判定归属,不应出现在正常运营数据里)。 diff --git a/spec/log-format.md b/spec/log-format.md new file mode 100644 index 0000000..702591c --- /dev/null +++ b/spec/log-format.md @@ -0,0 +1,53 @@ +# 日志格式契约 — 结构化 JSON + +> 唯一权威定义:各语言 SDK 的 log 输出必须与本文件一致。 +> 采集链路:服务 stdout(结构化 JSON)→ 华为云 log-agent(`allContainers: true` 全量采集)→ LTS。格式统一是 LTS 内可检索/结构化/告警的前提。 + +## 输出形态 + +- 每行一条日志,单行 JSON(**不 pretty**,不换行),通过 stdout 输出。 +- 字符集 UTF-8。 +- 时间字段 UTC,RFC 3339 / ISO-8601,带毫秒以上精度:`2026-09-08T07:12:34.567Z` 或 `.567890Z`(尽力而为,纳秒精度佳)。 + +## 顶层字段 + +| 字段 | 类型 | 必填 | 说明 | +| --- | --- | --- | --- | +| `time` | string | 是 | RFC 3339 UTC,输出时刻 | +| `level` | string | 是 | 枚举见下 | +| `msg` | string | 是 | 人类可读消息(错误场景为该错误摘要) | +| `service` | string | 是 | 服务名(= service.md 微服务名 / go module 子目录名),Init 时注入,常驻 | +| `env` | string | 是 | 部署环境 `prod/test/preview/staging`,Init 注入,常驻 | +| `instance` | string | 是 | 实例标识(k8s pod 名/IP),Init 注入,常驻 | +| `community` | string | 是 | 社区标识。部署级默认注入;请求级覆盖见 common-fields.md | +| `request_id` | string | 否* | 单请求关联 ID;无则输出空串或省略(*见下) | +| `trace_id` | string | 否* | **预留位**:二期 trace 接入前始终为空/省略,见下 | +| `logger` | string | 否 | 产生日志的 logger/包名(调试定位用) | +| `error` | string/object | 否 | 错误信息(仅 error 级推荐) | +| 其余 | — | 否 | 业务字段平铺在顶层(扁平 JSON),键名小写下划线(`snake_case`),禁止嵌套对象优先平铺 | + +> **request_id / trace_id 预留约定**:契约字段表里两者语义必留,但取值可为空。两种实现都接受:(a) 输出 `"request_id":""` / `"trace_id":""` 空串;(b) 为空时**省略该键**。SDK 内部统一为「有值才输出」可减少噪音——但 `trace_id` 语义位必须存在(SDK 须有注入该字段的能力),二期 trace 一接即可见。 + +## level 枚举 + +`debug` / `info` / `warn` / `error`。(`fatal` 由调用方决定是否需要 os.Exit,日志层不强制) + +## 示例(合法行) + +```json +{"time":"2026-09-08T07:12:34.567890Z","level":"info","msg":"handle github webhook","service":"robot-universal-review","env":"test","instance":"review-7f9c5d8b66-abcde","community":"openEuler","request_id":"req_01J8XK","trace_id":"","event":"pull_request","action":"opened"} +``` + +## 推荐:从上下文附加请求级字段 + +所有语言 SDK 都应支持从请求上下文(Go `context.Context` 等价物)读取并附加 `request_id`、`community`、`trace_id`(预留)三个字段。SDK 暴露两种调用风格之一或兼具: + +1. **绑定式**:`logger.WithRequestContext(ctx).Info(...)` —— 用 context 值绑定一个带请求字段的 logger,此后调用自动带上。 +2. **参数式**:`logger.Info(ctx, msg, kv...)` —— 每次显式传 context。 + +要求:**同一条日志内 `request_id`/`community`/`trace_id` 必须唯一、确定**;context 缺省时回退到 Init 注入的静态默认值(community 尤其如此)。 + +## LTS 检索约定 + +- LTS 结构化字段名 = 本表顶层字段名;日志检索时按 `service` / `community` / `level` / `request_id` 过滤。 +- 服务内业务字段建议在 LTS 里再做一次字段提取(正则/JSON 解析),键名同日志顶层键。 diff --git a/spec/metrics-format.md b/spec/metrics-format.md new file mode 100644 index 0000000..01ef473 --- /dev/null +++ b/spec/metrics-format.md @@ -0,0 +1,53 @@ +# 指标格式契约 — 命名与 label 规范 + +> 采集链路:服务暴露 `/metrics`(Prometheus text 格式)→ AOM 2.0 Prometheus 实例(ServiceMonitor 抓取)→ 大盘/告警。格式统一是跨服务聚合的前提。 + +## 指标命名前缀 + +Prometheus 指标命名规则基础上追加组织约束: + +- 指标名全小写,`snake_case`,单位后缀遵循 Prometheus 约定(`_total` / `_seconds` / `_bytes` / `_count` / `_sum`)。 +- **强制前缀**:`_`(service 短名,服务唯一)。例:`robot-universal-review_http_requests_total` → 简写 `review_http_requests_total`。 + - 也可按服务内子系统细分:`___`。 +- 若部署跨多个服务的共享 SDK 中间件统一暴露公共指标,则用 SDK 保留前缀 `obs_`(见 middleware 指标),避免与服务业务指标混名。例:`obs_http_server_requests_total`、`obs_http_server_request_duration_seconds`。 + - 中间件指标前缀统一 `obs_`,**不以 service 名开头**,因为同一条时间序列已带 `service` label;查询按 label 过滤。 + +## 通用 label 集(每个时间序列必须带) + +SDK 统一为所有注册的指标自动附加以下 **const label**(值来自 Init 注入的静态字段): + +| label | 值来源 | 说明 | +| --- | --- | --- | +| `service` | Init | 服务名 | +| `env` | Init | 环境 | +| `instance` | Init | 实例标识 | +| `community` | Init 默认 **+ 请求级覆盖** | 见下节——唯一一个可动态的公共 label | + +> 其余业务维度 label(`endpoint` / `method` / `status` / `code` 等)由业务/中间件按需声明。 +> **禁止**把 `request_id` 等高基数值当 label——会撑爆时序基数。 + +## community 双层注入在指标上的实现 + +同一个注册、两套取值来源: + +- **静态默认**:初始化时把默认 community 作为 const label 打进注册表。单社区独立部署服务到此为止,全部时间序列带固定 `community`。 +- **动态覆盖**:中心化服务需要区分社区时——SDK 指标 API 的**打点函数接受可选 community 覆盖**(与日志绑定式/参数式一致): + - 实现方式(推荐,兼容官方库):对需要区分 community 的业务指标,注册时把 `community` 声明为**普通可变 label**(同时保留 service/env/instance 作 const label),打点时由 SDK 从上下文/覆盖参数取出 community 值填 label。 + - 即:**同一条指标**,单社区场景填默认值即可;多社区场景填覆盖值。注册一次,两用。 +- 若官方库只支持 const label(无法按请求覆盖),SDK 应暴露「注册时声明该指标 community 是否动态」的两套方法,内部按官方库能力映射到 const label 或普通 label。 + +## 默认暴露的中间件指标(可选,按服务依赖挑) + +| 指标 | 类型 | label | +| --- | --- | --- | +| `obs_http_server_requests_total` | Counter | service, env, instance, community, method, path(可选,注意基数), status_code | +| `obs_http_server_request_duration_seconds` | Histogram | service, env, instance, community, method, path(可选) | +| `obs_log_entries_total` | Counter | service, env, instance, community, level | + +> 不强制:服务只要保证**自己注册的业务指标**带通用 label 即可;中间件公共指标(若启用)用 `obs_` 前缀,且**路径不入 label 或按低基数路由模板入 label**。 + +## 服务要求 + +- 每个服务暴露一个统一 `/metrics` HTTP 端点(Prometheus text exposition)。 +- 抓取协议:HTTP `GET /metrics`,官方库默认 behavior,content-type `text/plain; version=0.0.4`。 +- 多语言同构:四个语言 SDK 都要能输出**语义等价**的 `service/env/instance/community` 组合,供 ServiceMonitor 统一抓取、AOM/Cortex 统一查询。