From 21706ef6b25a0468a53c982b7ec7ea8d271691f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Sun, 4 Oct 2026 14:42:03 +0200 Subject: [PATCH] feat: Add loki and grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Müller --- README.md | 1 + docker-compose.yml | 59 + docker/configs/alloy/config.alloy | 330 ++++ .../grafana/dashboards/container-logs.json | 805 ++++++++ .../grafana/dashboards/nextcloud-access.json | 1632 +++++++++++++++++ .../grafana/dashboards/nextcloud-logs.json | 997 ++++++++++ .../provisioning/dashboards/default.yml | 10 + .../grafana/provisioning/datasources/loki.yml | 20 + docker/configs/loki/config.yml | 60 + docs/basics/troubleshooting.md | 1 + docs/services/more.md | 9 + docs/tools/logs.md | 223 +++ example.env | 7 + 13 files changed, 4154 insertions(+) create mode 100644 docker/configs/alloy/config.alloy create mode 100644 docker/configs/grafana/dashboards/container-logs.json create mode 100644 docker/configs/grafana/dashboards/nextcloud-access.json create mode 100644 docker/configs/grafana/dashboards/nextcloud-logs.json create mode 100644 docker/configs/grafana/provisioning/dashboards/default.yml create mode 100644 docker/configs/grafana/provisioning/datasources/loki.yml create mode 100644 docker/configs/loki/config.yml create mode 100644 docs/tools/logs.md diff --git a/README.md b/README.md index e1f5677c..af001435 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Nextcloud's development environment using Docker Compose providing a large varie - 👥 LDAP with example user data, Keycloak - ✉ Mailhog for testing mail sending - 🚀 Blackfire, Xdebug for profiling and debugging +- 🔍 Loki, Alloy and Grafana for collecting and searching the logs of the whole setup - 📄 Lots of integrating service containers: Collabora Online, Onlyoffice, Elasticsearch, ... ## Tutorial diff --git a/docker-compose.yml b/docker-compose.yml index eff92fbb..fa57cd71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: DHPARAM_GENERATION: "false" HTTPS_METHOD: "noredirect" HSTS: "off" + # logfmt with duration and request id for config.alloy, $ escapes $ for compose + LOG_FORMAT: 'time=$$time_iso8601 vhost=$$host method=$$request_method path=$$uri query="$$args" status=$$status bytes=$$body_bytes_sent duration=$$request_time upstream_time=$$upstream_response_time reqId=$$upstream_http_x_request_id remote_addr=$$remote_addr referer="$$http_referer" user_agent="$$http_user_agent"' cap_add: - SYS_ADMIN networks: @@ -68,6 +70,8 @@ services: - talk-signaling${DOMAIN_SUFFIX} - talk-recording${DOMAIN_SUFFIX} - authentik${DOMAIN_SUFFIX} + - grafana${DOMAIN_SUFFIX} + - alloy${DOMAIN_SUFFIX} extra_hosts: - host.docker.internal:host-gateway @@ -1405,6 +1409,58 @@ services: - ./authentik-custom-templates:/templates - ./authentik-certs:/certs + + # Log aggregation, see docs/tools/logs.md + loki: + image: grafana/loki:3.7.0 + command: -config.file=/etc/loki/config.yml -config.expand-env=true + environment: + LOKI_RETENTION_PERIOD: ${LOKI_RETENTION_PERIOD:-168h} + expose: + - 3100 + volumes: + - ./docker/configs/loki/config.yml:/etc/loki/config.yml:ro + - loki_data:/loki + + alloy: + image: grafana/alloy:v1.18.1 + # reading the docker socket requires root + user: root + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + environment: + # config.alloy scopes the log collection with it + COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-master} + VIRTUAL_HOST: "alloy${DOMAIN_SUFFIX}" + VIRTUAL_PORT: 12345 + volumes: + - ./docker/configs/alloy/config.alloy:/etc/alloy/config.alloy:ro + - ${DOCKER_SOCKET-/var/run/docker.sock}:/var/run/docker.sock:ro + - alloy_data:/var/lib/alloy/data + depends_on: + - loki + + grafana: + image: grafana/grafana:13.2.2 + environment: + VIRTUAL_HOST: "grafana${DOMAIN_SUFFIX}" + GF_SERVER_ROOT_URL: ${PROTOCOL:-http}://grafana${DOMAIN_SUFFIX}/ + GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin} + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + volumes: + - ./docker/configs/grafana/provisioning:/etc/grafana/provisioning:ro + - ./docker/configs/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - loki + - alloy volumes: data: config: @@ -1428,6 +1484,9 @@ volumes: elasticsearch_data: clam: mariadb_primary_data: + loki_data: + alloy_data: + grafana_data: authentik-database: driver: local diff --git a/docker/configs/alloy/config.alloy b/docker/configs/alloy/config.alloy new file mode 100644 index 00000000..b584206b --- /dev/null +++ b/docker/configs/alloy/config.alloy @@ -0,0 +1,330 @@ +// The nextcloud containers mix nextcloud.log, cron, xdebug and apache output on stdout +// (docker/bin/bootstrap.sh), the log_type label separates these formats again. + +logging { + level = "warn" + format = "logfmt" +} + +// shows what every pipeline stage does to a line in the alloy web interface +livedebugging { + enabled = true +} + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + refresh_interval = "5s" + + // COMPOSE_PROJECT_NAME is passed in by docker-compose.yml + filter { + name = "label" + values = ["com.docker.compose.project=" + sys.env("COMPOSE_PROJECT_NAME")] + } +} + +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // docker reports the name with a leading slash + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/?(.*)" + target_label = "container" + } + + // the service name survives a recreate, the container name does not + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.containers.output + labels = {"job" = "docker"} + forward_to = [loki.process.containers.receiver] +} + +loki.process "containers" { + forward_to = [loki.write.loki.receiver] + + // `tail --follow` prints a "==> /path <==" banner when it switches file + stage.drop { + expression = `^==> .* <==$` + drop_counter_reason = "tail_file_header" + } + + stage.match { + selector = `{container=~".+"} |~ "^\\{.*\"reqId\""` + + stage.json { + expressions = { + time = "time", + level = "to_string(level)", + app = "app", + reqId = "reqId", + user = "user", + method = "method", + url = "url", + scriptName = "scriptName", + remoteAddr = "remoteAddr", + userAgent = "userAgent", + version = "version", + exception = "exception.Exception", + } + } + + // nextcloud writes numeric levels, grafana colours by name + stage.template { + source = "level" + template = `{{ if eq .Value "0" }}debug{{ else if eq .Value "1" }}info{{ else if eq .Value "2" }}warn{{ else if eq .Value "3" }}error{{ else if eq .Value "4" }}fatal{{ else }}unknown{{ end }}` + } + + stage.static_labels { + values = {log_type = "nextcloud"} + } + + // only bounded values become labels, everything else multiplies the streams + stage.labels { + values = { + level = "", + app = "", + } + } + + stage.structured_metadata { + values = { + reqId = "", + user = "", + method = "", + url = "", + scriptName = "", + remoteAddr = "", + userAgent = "", + version = "", + exception = "", + } + } + + stage.timestamp { + source = "time" + format = "RFC3339" + fallback_formats = ["RFC3339Nano"] + action_on_failure = "fudge" + } + } + + // proxy access log in logfmt (LOG_FORMAT in docker-compose.yml), the only one with a duration + stage.match { + selector = `{service="proxy"} |~ "time=[0-9]{4}-"` + + // nginx runs under forego there, which prefixes every line with the process name + stage.replace { + expression = `^\S+\s+\| ` + replace = "" + } + + stage.logfmt { + mapping = { + time = "", + vhost = "", + method = "", + path = "", + query = "", + status = "", + bytes = "", + duration = "", + upstream_time = "", + reqId = "", + remote_addr = "", + user_agent = "", + } + } + + // the raw path is unique per room, file and user, route replaces only those parts + stage.template { + source = "route" + template = "{{ .path }}" + } + + stage.replace { + source = "route" + expression = `^(?:/remote\.php/dav/[a-z]+/)(.+)$` + replace = "{path}" + } + + stage.replace { + source = "route" + expression = `^(?:/remote\.php/webdav/)(.+)$` + replace = "{path}" + } + + // listed explicitly, a blanket rule would also rewrite fixed endpoints like /signaling/backend + stage.replace { + source = "route" + expression = `^(?:/ocs/v[0-9]\.php/apps/spreed/api/v[0-9]+/(?:room|chat|call|reaction|poll|bot|breakout-rooms|recording)/)([^/]+)` + replace = "{token}" + } + + stage.replace { + source = "route" + expression = `^(?:/ocs/v[0-9]\.php/apps/spreed/api/v[0-9]+/signaling/)([a-z0-9]{8,})$` + replace = "{token}" + } + + stage.replace { + source = "route" + expression = `^(?:/s/)([^/]+)` + replace = "{token}" + } + + stage.replace { + source = "route" + expression = `^(?:/index\.php/avatar/)([^/]+)` + replace = "{user}" + } + + stage.replace { + source = "route" + expression = `/([0-9]+)(?:/|$)` + replace = "{id}" + } + + stage.replace { + source = "route" + expression = `^(.+)\.(?:js|mjs|css|woff2?|ttf|otf|png|jpe?g|gif|svg|ico|map)$` + replace = "{static}" + } + + stage.static_labels { + values = {log_type = "access"} + } + + // the proxy serves every hostname, so the vhost is what separates the instances + stage.labels { + values = {vhost = ""} + } + + stage.structured_metadata { + values = { + method = "", + path = "", + query = "", + route = "", + status = "", + bytes = "", + duration = "", + upstream_time = "", + reqId = "", + remote_addr = "", + user_agent = "", + } + } + + stage.timestamp { + source = "time" + format = "RFC3339" + action_on_failure = "fudge" + } + } + + // includes internal traffic that never passes the proxy, e.g. from the signaling server + stage.match { + selector = `{container=~".+"} !~ "^\\{" |~ "\" [0-9]{3} [0-9-]+"` + + stage.regex { + expression = `"(?P[A-Z]+) (?P[^ "]+)[^"]*" (?P\d{3}) (?P\S+)` + } + + stage.static_labels { + values = {log_type = "apache_access"} + } + + stage.structured_metadata { + values = { + method = "", + path = "", + status = "", + bytes = "", + } + } + } + + // the apache error log carries php warnings and fatals, which never reach nextcloud.log + stage.match { + selector = `{container=~".+"} |~ "^\\[[^\\]]+\\] \\[[a-z_]+:[a-z0-9]+\\]"` + + stage.regex { + expression = `^\[[^\]]+\] \[(?P[a-z_]+):(?P[a-z0-9]+)\]` + } + + stage.template { + source = "level" + template = `{{ if or (eq .Value "emerg") (eq .Value "alert") (eq .Value "crit") }}fatal{{ else if eq .Value "notice" }}info{{ else if hasPrefix "trace" .Value }}debug{{ else }}{{ .Value }}{{ end }}` + } + + stage.static_labels { + values = {log_type = "apache_error"} + } + + stage.labels { + values = {level = ""} + } + + stage.structured_metadata { + values = {module = ""} + } + } + + // xdebug prefixes its lines with the process id + stage.match { + selector = `{container=~".+"} |~ "^\\[[0-9]+\\] "` + + stage.static_labels { + values = {log_type = "xdebug"} + } + } + + // janus prefixes everything but info, mapped to the nextcloud level names so {level="error"} spans the setup + stage.match { + selector = `{service="talk-janus"} |~ "\\[(FATAL|ERR|WARN)\\]"` + + stage.regex { + expression = `\[(?PFATAL|ERR|WARN)\]` + } + + stage.template { + source = "level" + template = `{{ if eq .Value "ERR" }}error{{ else }}{{ ToLower .Value }}{{ end }}` + } + + stage.labels { + values = {level = ""} + } + } + + // the recording server logs in pythons default format, "LEVEL:logger:message" + stage.match { + selector = `{service="talk-recording"} |~ "^(DEBUG|INFO|WARNING|ERROR|CRITICAL):"` + + stage.regex { + expression = `^(?PDEBUG|INFO|WARNING|ERROR|CRITICAL):` + } + + stage.template { + source = "level" + template = `{{ if eq .Value "WARNING" }}warn{{ else if eq .Value "CRITICAL" }}fatal{{ else }}{{ ToLower .Value }}{{ end }}` + } + + stage.labels { + values = {level = ""} + } + } +} + +loki.write "loki" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/docker/configs/grafana/dashboards/container-logs.json b/docker/configs/grafana/dashboards/container-logs.json new file mode 100644 index 00000000..ae023cf6 --- /dev/null +++ b/docker/configs/grafana/dashboards/container-logs.json @@ -0,0 +1,805 @@ +{ + "uid": "container-logs", + "title": "Container logs", + "description": "Everything but the Nextcloud application and request logs: databases, Redis, the Talk backends, Apache errors and the tooling containers.", + "tags": [ + "nextcloud", + "logs" + ], + "timezone": "browser", + "editable": true, + "schemaVersion": 39, + "version": 2, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "links": [ + { + "type": "dashboards", + "title": "Logs", + "tags": [ + "logs" + ], + "asDropdown": true, + "includeVars": false, + "keepTime": true, + "icon": "external link", + "tooltip": "", + "url": "" + } + ], + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({service=~\".+\", log_type!~\"nextcloud|access|apache_access\"}, service)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "search", + "label": "Search", + "description": "Only lines containing this text, applies to every panel", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + }, + { + "name": "level", + "label": "Log: level", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({service=~\".+\", log_type!~\"nextcloud|access|apache_access\"}, level)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "description": "Only applies to the log" + } + ] + }, + "panels": [ + { + "id": 23, + "type": "stat", + "title": "Errors", + "description": "Entries with level error or fatal, only services that log a level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#d03b3b" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=~\"error|fatal\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 24, + "type": "stat", + "title": "Warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#fab219" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"warn\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 25, + "type": "stat", + "title": "Services with errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#fab219" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (service) (count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=~\"error|fatal\"} |= `$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 26, + "type": "stat", + "title": "Log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 27, + "type": "timeseries", + "title": "Log volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "#666b75" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "fatal" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#9085e9" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d03b3b" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#fab219" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#86b6ef" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no level" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#666b75" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"fatal\"} |= `$search` [$__auto]))", + "legendFormat": "fatal" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"error\"} |= `$search` [$__auto]))", + "legendFormat": "error" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"warn\"} |= `$search` [$__auto]))", + "legendFormat": "warn" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"info\"} |= `$search` [$__auto]))", + "legendFormat": "info" + }, + { + "refId": "E", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"debug\"} |= `$search` [$__auto]))", + "legendFormat": "debug" + }, + { + "refId": "F", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=\"\"} |= `$search` [$__auto]))", + "legendFormat": "no level" + } + ] + }, + { + "id": 28, + "type": "table", + "title": "Errors by service", + "description": "Entries with level error or fatal, only services that log a level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 12 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Errors", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "fatal": { + "color": "#9085e9", + "index": 0 + }, + "error": { + "color": "#d03b3b", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(15, sum by (service, level) (count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=~\"error|fatal\"} |= `$search` [$__range])))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "Errors" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "service": 0, + "level": 1, + "Errors": 2 + }, + "renameByName": { + "service": "Service", + "level": "Level" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Errors", + "desc": true + } + ] + } + } + ] + }, + { + "id": 29, + "type": "status-history", + "title": "Log volume by service", + "description": "Lines per interval, one row per service", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "maxDataPoints": 60, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#a9cdf5" + }, + { + "value": 50, + "color": "#5598e7" + }, + { + "value": 500, + "color": "#2f6fc0" + } + ] + }, + "custom": { + "lineWidth": 0, + "fillOpacity": 100 + } + }, + "overrides": [] + }, + "options": { + "showValue": "never", + "rowHeight": 0.85, + "colWidth": 0.95, + "legend": { + "showLegend": false, + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (service) (count_over_time({service=~\"$service\", log_type!~\"nextcloud|access|apache_access\"} |= `$search` [$__interval]))", + "queryType": "range", + "legendFormat": "{{service}}" + } + ] + }, + { + "id": 30, + "type": "logs", + "title": "Log", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 29 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service=~\"$service\", log_type!~\"nextcloud|access|apache_access\", level=~\"$level\"} |= `$search`", + "queryType": "range", + "maxLines": 1000 + } + ] + } + ] +} diff --git a/docker/configs/grafana/dashboards/nextcloud-access.json b/docker/configs/grafana/dashboards/nextcloud-access.json new file mode 100644 index 00000000..677b14cc --- /dev/null +++ b/docker/configs/grafana/dashboards/nextcloud-access.json @@ -0,0 +1,1632 @@ +{ + "uid": "nextcloud-access", + "title": "Access log", + "description": "Every HTTP request as logged by the nginx proxy, grouped by normalized route.", + "tags": [ + "nextcloud", + "logs" + ], + "timezone": "browser", + "editable": true, + "schemaVersion": 39, + "version": 2, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "links": [ + { + "type": "dashboards", + "title": "Logs", + "tags": [ + "logs" + ], + "asDropdown": true, + "includeVars": false, + "keepTime": true, + "icon": "external link", + "tooltip": "", + "url": "" + } + ], + "templating": { + "list": [ + { + "name": "vhost", + "label": "Host", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({log_type=\"access\"}, vhost)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "class", + "label": "Log: status", + "type": "custom", + "multi": false, + "includeAll": false, + "description": "Only applies to the slow request table and the request log", + "query": "All : .*,2xx : 2..,3xx : 3..,4xx : 4..,5xx : 5..", + "options": [ + { + "selected": true, + "text": "All", + "value": ".*" + }, + { + "selected": false, + "text": "2xx", + "value": "2.." + }, + { + "selected": false, + "text": "3xx", + "value": "3.." + }, + { + "selected": false, + "text": "4xx", + "value": "4.." + }, + { + "selected": false, + "text": "5xx", + "value": "5.." + } + ], + "current": { + "selected": true, + "text": "All", + "value": ".*" + } + }, + { + "name": "slow", + "label": "Slow over (s)", + "type": "textbox", + "query": "1", + "current": { + "text": "1", + "value": "1" + } + }, + { + "name": "search", + "label": "Search", + "description": "Only lines containing this text, applies to every panel", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + } + ] + }, + "panels": [ + { + "id": 11, + "type": "stat", + "title": "5xx", + "description": "Share of requests that failed with a server error", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 0.001, + "color": "#fab219" + }, + { + "value": 5, + "color": "#d03b3b" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "100 * (sum(count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status=~\"5..\" [$__range])) or vector(0)) / sum(count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` [$__range]))", + "queryType": "instant" + } + ] + }, + { + "id": 12, + "type": "stat", + "title": "4xx", + "description": "Share of requests rejected with a client error, without 499 (client closed the connection, normal for long polling)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 10, + "color": "#fab219" + }, + { + "value": 25, + "color": "#d03b3b" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "100 * (sum(count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status=~\"4..\" | status!=\"499\" [$__range])) or vector(0)) / sum(count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` [$__range]))", + "queryType": "instant" + } + ] + }, + { + "id": 13, + "type": "stat", + "title": "p95 response time", + "description": "Without Talk long polling and websockets", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#fab219" + }, + { + "value": 3, + "color": "#d03b3b" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.95, {log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | unwrap duration | __error__=\"\" [$__range]) by ()", + "queryType": "instant" + } + ] + }, + { + "id": 14, + "type": "stat", + "title": "Requests", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Requests by status", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "#666b75" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#666b75" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#86b6ef" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "499" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#666b75" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#fab219" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d03b3b" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | label_format class=`{{ if eq .status \"499\" }}499{{ else }}{{ substr 0 1 .status }}xx{{ end }}` [$__auto]))", + "queryType": "range", + "legendFormat": "{{class}}" + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Response time", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Without Talk long polling and websockets", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "fixed", + "fixedColor": "#3987e5" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#fab219", + "value": 1 + }, + { + "color": "#d03b3b", + "value": 3 + } + ] + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "thresholdsStyle": { + "mode": "dashed" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#86b6ef" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "legendFormat": "p50", + "expr": "quantile_over_time(0.5, {log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | unwrap duration | __error__=\"\" [$__auto]) by ()" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "legendFormat": "p95", + "expr": "quantile_over_time(0.95, {log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | unwrap duration | __error__=\"\" [$__auto]) by ()" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "legendFormat": "p99", + "expr": "quantile_over_time(0.99, {log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | unwrap duration | __error__=\"\" [$__auto]) by ()" + } + ] + }, + { + "id": 17, + "type": "table", + "title": "Failing routes", + "description": "4xx and 5xx responses, without 499", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Count", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^5\\d\\d$", + "result": { + "color": "#d03b3b", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4\\d\\d$", + "result": { + "color": "#fab219", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Host" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(20, sum by (status, method, vhost, route) (count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status=~\"[45]..\" | status!=\"499\" [$__range])))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "Count" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "status": 0, + "method": 1, + "vhost": 2, + "route": 3, + "Count": 4 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "vhost": "Host", + "route": "Route" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Count", + "desc": true + } + ] + } + } + ] + }, + { + "id": 18, + "type": "table", + "title": "Slowest routes", + "description": "Without Talk long polling and websockets", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "p95", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Host" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#fab219", + "value": 1 + }, + { + "color": "#d03b3b", + "value": 3 + } + ] + } + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(20, quantile_over_time(0.95, {log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | unwrap duration | __error__=\"\" [$__range]) by (method, vhost, route))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "p95" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "method": 0, + "vhost": 1, + "route": 2, + "p95": 3 + }, + "renameByName": { + "method": "Method", + "vhost": "Host", + "route": "Route" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "p95", + "desc": true + } + ] + } + } + ] + }, + { + "id": 19, + "type": "table", + "title": "Busiest routes", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Requests", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Host" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(20, sum by (method, vhost, route) (count_over_time({log_type=\"access\", vhost=~\"$vhost\"} |= `$search` [$__range])))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "Requests" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "method": 0, + "vhost": 1, + "route": 2, + "Requests": 3 + }, + "renameByName": { + "method": "Method", + "vhost": "Host", + "route": "Route" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Requests", + "desc": true + } + ] + } + } + ] + }, + { + "id": 20, + "type": "table", + "title": "Requests slower than ${slow}s", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Without Talk long polling and websockets. The request id opens the matching Nextcloud log entries", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 36 + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Duration", + "desc": true + } + ], + "cellHeight": "sm", + "footer": { + "show": false + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^5\\d\\d$", + "result": { + "color": "#d03b3b", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4\\d\\d$", + "result": { + "color": "#fab219", + "index": 1 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Host" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request id" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Nextcloud log entries of this request", + "url": "/d/nextcloud-logs/nextcloud-logs?var-search=${__value.raw}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request id" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status=~\"$class\" | status!=\"101\" | query!~`.*lookIntoFuture=1.*` | route!~`.*/signaling/\\{token\\}` | duration > $slow", + "queryType": "range", + "maxLines": 200 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "format": "json", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "status", + "method", + "vhost", + "route", + "duration", + "reqId" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "duration", + "destinationType": "number" + } + ] + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "Time": 0, + "status": 1, + "method": 2, + "vhost": 3, + "route": 4, + "duration": 5, + "reqId": 6 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "vhost": "Host", + "route": "Route", + "duration": "Duration", + "reqId": "Request id" + } + } + } + ] + }, + { + "id": 21, + "type": "logs", + "title": "Requests", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 46 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{log_type=\"access\", vhost=~\"$vhost\"} |= `$search` | status=~\"$class\" | label_format level=`{{ if hasPrefix \"5\" .status }}error{{ else if and (hasPrefix \"4\" .status) (ne .status \"499\") }}warn{{ else }}info{{ end }}`, detected_level=`{{ if hasPrefix \"5\" .status }}error{{ else if and (hasPrefix \"4\" .status) (ne .status \"499\") }}warn{{ else }}info{{ end }}` | line_format `{{.status}} {{.method}} {{.vhost}}{{.path}}{{ if .query }}?{{.query}}{{ end }} · {{.duration}}s{{ if .reqId }} · {{.reqId}}{{ end }}`", + "queryType": "range", + "maxLines": 1000 + } + ] + } + ] +} diff --git a/docker/configs/grafana/dashboards/nextcloud-logs.json b/docker/configs/grafana/dashboards/nextcloud-logs.json new file mode 100644 index 00000000..5c50294f --- /dev/null +++ b/docker/configs/grafana/dashboards/nextcloud-logs.json @@ -0,0 +1,997 @@ +{ + "uid": "nextcloud-logs", + "title": "Nextcloud logs", + "description": "The application log of the Nextcloud instances, nextcloud.log parsed as JSON.", + "tags": [ + "nextcloud", + "logs" + ], + "timezone": "browser", + "editable": true, + "schemaVersion": 39, + "version": 2, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "links": [ + { + "type": "dashboards", + "title": "Logs", + "tags": [ + "logs" + ], + "asDropdown": true, + "includeVars": false, + "keepTime": true, + "icon": "external link", + "tooltip": "", + "url": "" + } + ], + "templating": { + "list": [ + { + "name": "container", + "label": "Instance", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({log_type=\"nextcloud\"}, container)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "search", + "label": "Search", + "description": "Only lines containing this text, applies to every panel", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + }, + { + "name": "app", + "label": "Log: app", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({log_type=\"nextcloud\"}, app)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "description": "Only applies to the log" + }, + { + "name": "level", + "label": "Log: level", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": "label_values({log_type=\"nextcloud\"}, level)", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "description": "Only applies to the log" + }, + { + "name": "exception", + "label": "Log: exception", + "description": "Only applies to the log, part of the exception class is enough", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Errors", + "description": "Entries with level error or fatal", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#d03b3b" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=~\"error|fatal\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Exceptions", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#fab219" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\"} |= `$search` | exception!=\"\" [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Apps with errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#fab219" + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (app) (count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=~\"error|fatal\"} |= `$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\"} |= `$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Log volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "#666b75" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "fatal" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#9085e9" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#d03b3b" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#fab219" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#86b6ef" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=\"fatal\"} |= `$search` [$__auto]))", + "legendFormat": "fatal" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=\"error\"} |= `$search` [$__auto]))", + "legendFormat": "error" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=\"warn\"} |= `$search` [$__auto]))", + "legendFormat": "warn" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=\"info\"} |= `$search` [$__auto]))", + "legendFormat": "info" + }, + { + "refId": "E", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "sum(count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=\"debug\"} |= `$search` [$__auto]))", + "legendFormat": "debug" + } + ] + }, + { + "id": 6, + "type": "status-history", + "title": "Exceptions over time", + "description": "One row per exception class, a new class shows up as a new row", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "maxDataPoints": 60, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "value": 1, + "color": "#a9cdf5" + }, + { + "value": 5, + "color": "#5598e7" + }, + { + "value": 20, + "color": "#2f6fc0" + } + ] + }, + "custom": { + "lineWidth": 0, + "fillOpacity": 100 + } + }, + "overrides": [] + }, + "options": { + "showValue": "never", + "rowHeight": 0.85, + "colWidth": 0.95, + "legend": { + "showLegend": false, + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (exception) (count_over_time({log_type=\"nextcloud\", container=~\"$container\"} |= `$search` | exception!=\"\" [$__interval]))", + "queryType": "range", + "legendFormat": "{{exception}}" + } + ] + }, + { + "id": 7, + "type": "table", + "title": "Top exceptions", + "description": "Click a class to show its entries in the log", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Count", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "App" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Exception" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show these exceptions in the log", + "url": "/d/nextcloud-logs/nextcloud-logs?${container:queryparam}&${search:queryparam}&${__url_time_range}&var-exception=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Exception" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(20, sum by (app, exception) (count_over_time({log_type=\"nextcloud\", container=~\"$container\"} |= `$search` | exception!=\"\" [$__range])))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "Count" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "app": 0, + "exception": 1, + "Count": 2 + }, + "renameByName": { + "app": "App", + "exception": "Exception" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Count", + "desc": true + } + ] + } + } + ] + }, + { + "id": 8, + "type": "table", + "title": "Where errors come from", + "description": "Entries with level error or fatal. Click an app to show its entries in the log", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Errors", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ] + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Entry point" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "basic", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#3987e5" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "App" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show the entries of this app in the log", + "url": "/d/nextcloud-logs/nextcloud-logs?${container:queryparam}&${search:queryparam}&${__url_time_range}&var-app=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Entry point" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(20, sum by (container, scriptName, app) (count_over_time({log_type=\"nextcloud\", container=~\"$container\", level=~\"error|fatal\"} |= `$search` [$__range])))", + "queryType": "instant" + } + ], + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "^Value.*", + "renamePattern": "Errors" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "container": 0, + "scriptName": 1, + "app": 2, + "Errors": 3 + }, + "renameByName": { + "container": "Instance", + "scriptName": "Entry point", + "app": "App" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Errors", + "desc": true + } + ] + } + } + ] + }, + { + "id": 9, + "type": "logs", + "title": "Log", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 36 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{log_type=\"nextcloud\", container=~\"$container\", app=~\"$app\", level=~\"$level\"} |= `$search` | exception=~`(?i).*${exception:regex}.*` | json message, exception_message=\"exception.Message\", exception_file=\"exception.File\", exception_line=\"exception.Line\" | line_format `{{.app}} · {{ if and .user (ne .user \"--\") }}{{.user}} · {{ end }}{{ if .exception }}{{.exception}}: {{ end }}{{.message}}{{ if .url }} · {{.method}} {{.url}}{{ end }}`", + "queryType": "range", + "maxLines": 1000 + } + ] + } + ] +} diff --git a/docker/configs/grafana/provisioning/dashboards/default.yml b/docker/configs/grafana/provisioning/dashboards/default.yml new file mode 100644 index 00000000..4db62453 --- /dev/null +++ b/docker/configs/grafana/provisioning/dashboards/default.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: nextcloud-docker-dev + type: file + disableDeletion: false + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/docker/configs/grafana/provisioning/datasources/loki.yml b/docker/configs/grafana/provisioning/datasources/loki.yml new file mode 100644 index 00000000..fa7bed26 --- /dev/null +++ b/docker/configs/grafana/provisioning/datasources/loki.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + editable: false + jsonData: + maxLines: 5000 + # clicking the request id of a log line opens all log entries of that request + derivedFields: + - name: reqId + matcherType: label + matcherRegex: reqId + datasourceUid: loki + url: '{log_type=~"nextcloud|access"} | reqId = `${__value.raw}`' + urlDisplayLabel: 'All log entries of this request' diff --git a/docker/configs/loki/config.yml b/docker/configs/loki/config.yml new file mode 100644 index 00000000..2c3f2df9 --- /dev/null +++ b/docker/configs/loki/config.yml @@ -0,0 +1,60 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +# structured metadata (reqId and the other high cardinality fields) requires tsdb and schema v13 +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + allow_structured_metadata: true + # log files are often replayed or backdated here + reject_old_samples: false + retention_period: ${LOKI_RETENTION_PERIOD:-168h} + max_global_streams_per_user: 10000 + per_stream_rate_limit: 100MB + per_stream_rate_limit_burst: 100MB + ingestion_rate_mb: 100 + ingestion_burst_size_mb: 100 + +# without the compactor the chunks directory grows forever +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +pattern_ingester: + enabled: true + +analytics: + reporting_enabled: false diff --git a/docs/basics/troubleshooting.md b/docs/basics/troubleshooting.md index 2393582a..09ec17c5 100644 --- a/docs/basics/troubleshooting.md +++ b/docs/basics/troubleshooting.md @@ -11,3 +11,4 @@ allowing you to run `docker compose up` again from a clean slate. - You can use `docker compose logs -f` to follow the logs of all containers - You can use `docker compose logs -f nextcloud` to follow the logs of the Nextcloud container +- For searching and visualizing the logs of all containers, the setup ships a Loki and Grafana stack, see [log aggregation](../tools/logs.md) diff --git a/docs/services/more.md b/docs/services/more.md index d96540e1..63b23804 100644 --- a/docs/services/more.md +++ b/docs/services/more.md @@ -17,3 +17,12 @@ docker compose up -d elasticsearch elasticsearch-ui - Address for accessing the UI: `sudo sysctl -w vm.max_map_count=262144` + +## Log aggregation + +``` +docker compose up -d grafana +``` + +Collects the logs of all containers of the setup into Loki and makes them searchable in Grafana at +. See [log aggregation](../tools/logs.md) for details. diff --git a/docs/tools/logs.md b/docs/tools/logs.md new file mode 100644 index 00000000..0d060569 --- /dev/null +++ b/docs/tools/logs.md @@ -0,0 +1,223 @@ +# Log aggregation + +All container logs can be collected into [Loki](https://grafana.com/oss/loki/) and browsed in +[Grafana](https://grafana.com/oss/grafana/), which is a lot more comfortable than `docker compose logs` +when several containers are involved or when you want to look at a request that happened an hour ago. + +```bash +docker compose up -d grafana +``` + +This starts three containers: + +| Container | Purpose | +| --------- | ------- | +| `alloy` | [Grafana Alloy](https://grafana.com/docs/alloy/latest/) collects the logs of all containers of this compose project through the docker socket and pushes them to Loki. It replaces Promtail, which reached end of life in March 2026. | +| `loki` | Stores the logs on a local volume | +| `grafana` | Web interface to search and visualize them | + +Afterwards open [http://grafana.local](http://grafana.local) and log in with `admin` / `admin`, +which can be changed with `GRAFANA_USER` and `GRAFANA_PASSWORD` in your `.env` file. + +## What is collected + +Alloy reads the docker logs of every container of this compose project, so nothing has to be +mounted or configured per service. Containers of other projects running on the same docker host are +not touched, the scoping is done with the `com.docker.compose.project` label and therefore relies on +`COMPOSE_PROJECT_NAME` being set in your `.env` file, which `bootstrap.sh` does for you. + +The Nextcloud containers already write `data/nextcloud.log`, the cron log and the xdebug log to +their stdout and Apache logs there as well, which means the full Nextcloud log ends up in Loki +without any further setup. One container log therefore mixes several formats, the `log_type` +label separates them again: + +| `log_type` | Content | +| ---------- | ------- | +| `nextcloud` | `nextcloud.log`, parsed as JSON | +| `access` | The access log of the nginx proxy, which sees every request of the whole setup | +| `apache_access` | The access log of Apache itself, which also covers internal requests that never pass the proxy | +| `apache_error` | Apache error log, this is where PHP fatals and warnings show up that never reach `nextcloud.log` | +| `xdebug` | Xdebug output | + +Everything without one of those formats, for example the databases, the signaling server or the +cron output, has no `log_type` and is stored as it is. + +## Labels + +Every log entry carries `container`, `service` and `job="docker"`. Loki queries start with a +stream selector, so only fields with few, known values are labels, everything with many possible +values is [structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/) +instead, which is filtered after a `|` and does not create a stream per value. + +Lines from `nextcloud.log` are parsed as JSON and get: + +- Labels: `level`, `app` +- Structured metadata: `reqId`, `user`, `method`, `url`, `scriptName`, `remoteAddr`, + `userAgent`, `version` and `exception`, the class name of a logged exception + +The proxy is configured with a `LOG_FORMAT` that writes logfmt instead of the combined format, so +its access lines carry `vhost` as a label and `method`, `path`, `query`, `route`, `status`, +`bytes`, `duration`, `upstream_time`, `remote_addr`, `user_agent` and `reqId` as structured +metadata. Status codes are deliberately not labels, a label per status code would multiply the +streams of every container. + +Grouping by `path` is pointless in Nextcloud because it contains conversation tokens, user names, +file paths and ids, so almost every request has its own. `route` is the same path with those parts +replaced, which is what the dashboard groups by: + +| Path | Route | +| ---- | ----- | +| `/ocs/v2.php/apps/spreed/api/v1/chat/jetukpyo` | `/ocs/v2.php/apps/spreed/api/v1/chat/{token}` | +| `/remote.php/dav/files/admin/Photos/image.jpg` | `/remote.php/dav/files/{path}` | +| `/index.php/avatar/alice/64` | `/index.php/avatar/{user}/{id}` | +| `/apps/files/js/merged-index.js` | `{static}.js` | + +The rules live at the end of the proxy block in `config.alloy` and are easy to extend, each one +replaces only its capture group and keeps the rest of the path. + +`duration` is the request time in seconds, which is what the response time panels are built on, +and `reqId` is the request id Nextcloud returns in its `X-Request-Id` response header. It ties an +access line to the `nextcloud.log` entries of the same request, so a slow or failing request can +be opened in the application log with one click. Responses that Nextcloud does not render through +its app framework, for example WebDAV through `remote.php` or static files, have no such header +and therefore no `reqId`. + +The `level` label uses the same five values everywhere, `debug`, `info`, `warn`, `error` and +`fatal`, so a single query covers the whole setup. Besides Nextcloud it is set for: + +- `talk-janus`, which prefixes everything but info with `[FATAL]`, `[ERR]` or `[WARN]` +- `talk-recording`, which logs in Pythons default `LEVEL:logger:message` format +- the Apache error log, whose `[php:error]` style tags are mapped to the same names + +The signaling server writes plain Go log lines without a level and there is nothing else worth +labelling in them, the same is true for the cron output and the databases. For those Loki still +adds its own guess as the `detected_level` structured metadata field. + +## Dashboards + +Three dashboards are provisioned, the dropdown in the top left switches between them and keeps +the selected time range: + +| Dashboard | Use it for | +| --------- | ---------- | +| **Nextcloud logs** | The application log. Errors and exceptions at the top, volume by level, every exception class over time so that a new one stands out, the most frequent exceptions, where errors come from and the log itself. Clicking an exception or app in a table filters the log to it. | +| **Access log** | HTTP traffic as seen by the proxy. Share of 5xx and 4xx responses and the p95 response time at the top, requests by status class over time, p50/p95/p99 response time, sortable tables of the failing, slowest and busiest routes, and every single request slower than a configurable threshold, whose request id opens the matching Nextcloud log entries. | +| **Container logs** | Everything else, the databases, Redis, the Talk backends, the Apache error log and the tooling containers, by service and level. | + +The tiles at the top only turn amber or red when something needs a look. Filters named `Log: …` only +apply to the log list at the bottom, so the overview above never changes unnoticed, while the instance +or host filter and `Search` apply to every panel. + +Response times leave out the requests Talk keeps open on purpose, long polling of the chat and the +internal signaling as well as websockets, otherwise they would dominate every latency panel. For the +same reason the 4xx share ignores `499`, which nginx logs when a client closes such a request. + +## Useful queries + +Errors anywhere in the setup, Nextcloud instances and Talk backends alike: + +```logql +{level=~"error|fatal"} +``` + +Only the application log of the Nextcloud instances, without access lines and cron output: + +```logql +{log_type="nextcloud"} +``` + +Requests that failed with a server error: + +```logql +{log_type="access"} | status=~"5.." +``` + +Requests that took longer than two seconds: + +```logql +{log_type="access"} | duration > 2 +``` + +The 95th percentile of the response time per host: + +```logql +quantile_over_time(0.95, {log_type="access"} | unwrap duration [5m]) by (vhost) +``` + +The busiest routes of the last hour: + +```logql +topk(10, sum by (route) (count_over_time({log_type="access"}[1h]))) +``` + +The most frequent exceptions of the last hour: + +```logql +topk(10, sum by (exception) (count_over_time({log_type="nextcloud"} | exception != "" [1h]))) +``` + +Everything a single container logged: + +```logql +{container="master-nextcloud-1"} +``` + +All log entries belonging to one request, the request id can also be copied from the +`X-Request-Id` response header: + +```logql +{service=~".+"} | reqId = `bJSJXYCTU0gBjVZq5Vtm` +``` + +Log entries of one Nextcloud app only: + +```logql +{app="spreed"} +``` + +Everything the Talk backends logged: + +```logql +{service=~"talk-.+"} +``` + +Number of errors per app over the last hour: + +```logql +sum by (app) (count_over_time({level=~"error|fatal"}[1h])) +``` + +In the log view of Grafana every entry has a `reqId` field. Clicking it opens all log entries of +that request, which is the fastest way to find out what else happened while a request failed. + +## Configuration + +- Collection pipeline: `docker/configs/alloy/config.alloy` +- Loki: `docker/configs/loki/config.yml` +- Grafana datasource and dashboards: `docker/configs/grafana/` + +Logs are kept for 7 days, this can be changed with `LOKI_RETENTION_PERIOD` in your `.env` file. + +Grafana asks for a login instead of allowing anonymous access, because an anonymous session cannot +save dashboard changes and makes Grafana show `Unauthorized` banners for the user specific parts +of its own API. Adding `GF_AUTH_ANONYMOUS_ENABLED: "true"` to the environment of the `grafana` +service in `docker-compose.yml` enables the login free variant with those limitations. + +To see which containers Alloy discovered and what the pipeline does to a log line, open the Alloy +interface at [http://alloy.local](http://alloy.local). + +After changing `config.alloy`, recreate the container instead of restarting it: + +```bash +docker compose up -d --force-recreate alloy +``` + +The file is bind mounted as a single file, so an editor that writes a new file instead of changing +the existing one leaves the running container with the old content. Alloy also remembers how far it +has read, which means only log lines written after the recreate carry new labels. + +!!! note + + Since all containers of the project are collected, `loki`, `alloy` and `grafana` log their own + output into Loki as well. If Loki is unreachable, Alloy keeps logging that error and picks it up again on the next + run, so it is normal to see a burst of Alloy errors after Loki was restarted. diff --git a/example.env b/example.env index 05ea40d7..f18d472c 100644 --- a/example.env +++ b/example.env @@ -74,3 +74,10 @@ PHP_XDEBUG_MODE=develop # EX_APPS_NET=ipv4@localhost # EX_APPS_COUNT=50 # ------------------------------------ + +# Log aggregation (loki, alloy and grafana, see docs/tools/logs.md) +# How long collected logs are kept +# LOKI_RETENTION_PERIOD=168h +# Grafana login +# GRAFANA_USER=admin +# GRAFANA_PASSWORD=admin