Add metrics enrichment table processor to Observability Pipelines

.generator/schemas/v2/openapi.yaml

@@ -76346,6 +76346,7 @@ components:
         - $ref: "#/components/schemas/ObservabilityPipelineThrottleProcessor"
         - $ref: "#/components/schemas/ObservabilityPipelineAddMetricTagsProcessor"
         - $ref: "#/components/schemas/ObservabilityPipelineAggregateProcessor"
+        - $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableProcessor"
         - $ref: "#/components/schemas/ObservabilityPipelineMetricTagsProcessor"
         - $ref: "#/components/schemas/ObservabilityPipelineRenameMetricTagsProcessor"
         - $ref: "#/components/schemas/ObservabilityPipelineTagCardinalityLimitProcessor"
@@ -78491,6 +78492,187 @@ components:
         - name
         - value
       type: object
+    ObservabilityPipelineMetricEnrichmentTableFile:
+      description: Defines a static enrichment table loaded from a CSV file for metric enrichment.
+      properties:
+        encoding:
+          $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFileEncoding"
+        key:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableFileKey"
+        path:
+          description: Path to the CSV file.
+          example: /etc/enrichment/lookup.csv
+          type: string
+      required:
+        - encoding
+        - key
+        - path
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableFileKey:
+      description: Defines how to map a metric lookup value to a CSV column during enrichment table lookups.
+      properties:
+        column:
+          description: The CSV column name or index to match against the lookup value.
+          example: service
+          type: string
+        source:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableLookupSource"
+      required:
+        - column
+        - source
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableFileProcessor:
+      description: An `enrichment_table` processor that enriches metrics using a static CSV file.
+      properties:
+        display_name:
+          $ref: "#/components/schemas/ObservabilityPipelineComponentDisplayName"
+        enabled:
+          description: Indicates whether the processor is enabled.
+          example: true
+          type: boolean
+        file:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableFile"
+        id:
+          description: |-
+            The unique identifier for this component. Used in other parts of the pipeline to reference this component
+            (for example, as the `input` to downstream components).
+          example: enrichment-table-processor
+          type: string
+        include:
+          description: A Datadog search query used to determine which metrics this processor targets.
+          example: "*"
+          type: string
+        type:
+          $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableProcessorType"
+      required:
+        - id
+        - type
+        - include
+        - file
+        - enabled
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableLookupSource:
+      description: |-
+        Specifies the source of the key value used for metric enrichment table lookups.
+        The lookup key can be either the metric name or a metric tag.
+      oneOf:
+        - $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableMetricNameLookup"
+        - $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableTagLookup"
+    ObservabilityPipelineMetricEnrichmentTableMetricNameLookup:
+      description: Uses the metric name as the lookup key for enrichment table matching.
+      properties:
+        type:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableMetricNameLookupType"
+      required:
+        - type
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableMetricNameLookupType:
+      default: metric_name
+      description: The lookup source type. The value should always be `metric_name`.
+      enum: [metric_name]
+      example: metric_name
+      type: string
+      x-enum-varnames:
+        - METRIC_NAME
+    ObservabilityPipelineMetricEnrichmentTableProcessor:
+      description: |-
+        The `enrichment_table` processor enriches metrics with tags from a static CSV file or a Datadog reference table.
+        It looks up a row using the metric name or a metric tag value. It then adds each column of the matching row as a
+        metric tag, overwriting any existing tag with the same key. Exactly one of `file` or `reference_table` must be
+        configured.
+
+        **Supported pipeline types:** metrics
+      oneOf:
+        - $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableFileProcessor"
+        - $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableReferenceTableProcessor"
+      x-pipeline-types: [metrics]
+    ObservabilityPipelineMetricEnrichmentTableReferenceKey:
+      description: Defines the metric lookup value used as the reference-table row ID.
+      properties:
+        source:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableLookupSource"
+      required:
+        - source
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableReferenceTable:
+      description: Uses a Datadog reference table to enrich metrics.
+      properties:
+        app_key_key:
+          description: |-
+            The name of the environment variable or secret that holds the Datadog application key used to access the
+            reference table.
+          example: DD_APP_KEY
+          type: string
+        columns:
+          description: |-
+            A list of column names to include from the reference table. If not provided, all columns are included.
+          items:
+            description: The name of a column to include from the reference table.
+            type: string
+          maxItems: 50
+          type: array
+        key:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableReferenceKey"
+        table_id:
+          description: The unique identifier of the reference table.
+          example: 550e8400-e29b-41d4-a716-446655440000
+          type: string
+      required:
+        - key
+        - table_id
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableReferenceTableProcessor:
+      description: An `enrichment_table` processor that enriches metrics using a Datadog reference table.
+      properties:
+        display_name:
+          $ref: "#/components/schemas/ObservabilityPipelineComponentDisplayName"
+        enabled:
+          description: Indicates whether the processor is enabled.
+          example: true
+          type: boolean
+        id:
+          description: |-
+            The unique identifier for this component. Used in other parts of the pipeline to reference this component
+            (for example, as the `input` to downstream components).
+          example: enrichment-table-processor
+          type: string
+        include:
+          description: A Datadog search query used to determine which metrics this processor targets.
+          example: "*"
+          type: string
+        reference_table:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableReferenceTable"
+        type:
+          $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableProcessorType"
+      required:
+        - id
+        - type
+        - include
+        - reference_table
+        - enabled
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableTagLookup:
+      description: Uses a metric tag as the lookup key for enrichment table matching.
+      properties:
+        name:
+          description: The Datadog tag key used as the lookup key.
+          example: service
+          maxLength: 200
+          type: string
+        type:
+          $ref: "#/components/schemas/ObservabilityPipelineMetricEnrichmentTableTagLookupType"
+      required:
+        - type
+        - name
+      type: object
+    ObservabilityPipelineMetricEnrichmentTableTagLookupType:
+      default: tag
+      description: The lookup source type. The value should always be `tag`.
+      enum: [tag]
+      example: tag
+      type: string
+      x-enum-varnames:
+        - TAG
     ObservabilityPipelineMetricTagsProcessor:
       description: |-
         The `metric_tags` processor filters metrics based on their tags using Datadog tag key patterns.

examples/v2/observability-pipelines/ValidatePipeline_1795564228.rb

@@ -0,0 +1,68 @@
+# Validate a metrics pipeline with enrichment table processor file lookup returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.access_token = ENV["DD_BEARER_TOKEN"]
+end
+api_instance = DatadogAPIClient::V2::ObservabilityPipelinesAPI.new
+
+body = DatadogAPIClient::V2::ObservabilityPipelineSpec.new({
+  data: DatadogAPIClient::V2::ObservabilityPipelineSpecData.new({
+    attributes: DatadogAPIClient::V2::ObservabilityPipelineDataAttributes.new({
+      config: DatadogAPIClient::V2::ObservabilityPipelineConfig.new({
+        pipeline_type: DatadogAPIClient::V2::ObservabilityPipelineConfigPipelineType::METRICS,
+        destinations: [
+          DatadogAPIClient::V2::ObservabilityPipelineDatadogMetricsDestination.new({
+            id: "datadog-metrics-destination",
+            inputs: [
+              "my-processor-group",
+            ],
+            type: DatadogAPIClient::V2::ObservabilityPipelineDatadogMetricsDestinationType::DATADOG_METRICS,
+          }),
+        ],
+        processor_groups: [
+          DatadogAPIClient::V2::ObservabilityPipelineConfigProcessorGroup.new({
+            enabled: true,
+            id: "my-processor-group",
+            include: "*",
+            inputs: [
+              "datadog-agent-source",
+            ],
+            processors: [
+              DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableFileProcessor.new({
+                enabled: true,
+                id: "enrichment-table-processor",
+                include: "*",
+                type: DatadogAPIClient::V2::ObservabilityPipelineEnrichmentTableProcessorType::ENRICHMENT_TABLE,
+                file: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableFile.new({
+                  encoding: DatadogAPIClient::V2::ObservabilityPipelineEnrichmentTableFileEncoding.new({
+                    delimiter: ",",
+                    type: DatadogAPIClient::V2::ObservabilityPipelineEnrichmentTableFileEncodingType::CSV,
+                    includes_headers: true,
+                  }),
+                  key: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableFileKey.new({
+                    column: "service",
+                    source: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableTagLookup.new({
+                      type: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableTagLookupType::TAG,
+                      name: "service",
+                    }),
+                  }),
+                  path: "/etc/enrichment/lookup.csv",
+                }),
+              }),
+            ],
+          }),
+        ],
+        sources: [
+          DatadogAPIClient::V2::ObservabilityPipelineDatadogAgentSource.new({
+            id: "datadog-agent-source",
+            type: DatadogAPIClient::V2::ObservabilityPipelineDatadogAgentSourceType::DATADOG_AGENT,
+          }),
+        ],
+      }),
+      name: "Metrics Pipeline with Enrichment Table File Lookup",
+    }),
+    type: "pipelines",
+  }),
+})
+p api_instance.validate_pipeline(body)

examples/v2/observability-pipelines/ValidatePipeline_4239139905.rb

@@ -0,0 +1,65 @@
+# Validate a metrics pipeline with enrichment table processor reference table returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.access_token = ENV["DD_BEARER_TOKEN"]
+end
+api_instance = DatadogAPIClient::V2::ObservabilityPipelinesAPI.new
+
+body = DatadogAPIClient::V2::ObservabilityPipelineSpec.new({
+  data: DatadogAPIClient::V2::ObservabilityPipelineSpecData.new({
+    attributes: DatadogAPIClient::V2::ObservabilityPipelineDataAttributes.new({
+      config: DatadogAPIClient::V2::ObservabilityPipelineConfig.new({
+        pipeline_type: DatadogAPIClient::V2::ObservabilityPipelineConfigPipelineType::METRICS,
+        destinations: [
+          DatadogAPIClient::V2::ObservabilityPipelineDatadogMetricsDestination.new({
+            id: "datadog-metrics-destination",
+            inputs: [
+              "my-processor-group",
+            ],
+            type: DatadogAPIClient::V2::ObservabilityPipelineDatadogMetricsDestinationType::DATADOG_METRICS,
+          }),
+        ],
+        processor_groups: [
+          DatadogAPIClient::V2::ObservabilityPipelineConfigProcessorGroup.new({
+            enabled: true,
+            id: "my-processor-group",
+            include: "*",
+            inputs: [
+              "datadog-agent-source",
+            ],
+            processors: [
+              DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableReferenceTableProcessor.new({
+                enabled: true,
+                id: "enrichment-table-processor",
+                include: "*",
+                type: DatadogAPIClient::V2::ObservabilityPipelineEnrichmentTableProcessorType::ENRICHMENT_TABLE,
+                reference_table: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableReferenceTable.new({
+                  table_id: "metric-enrichment",
+                  key: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableReferenceKey.new({
+                    source: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableMetricNameLookup.new({
+                      type: DatadogAPIClient::V2::ObservabilityPipelineMetricEnrichmentTableMetricNameLookupType::METRIC_NAME,
+                    }),
+                  }),
+                  columns: [
+                    "environment",
+                    "team",
+                  ],
+                }),
+              }),
+            ],
+          }),
+        ],
+        sources: [
+          DatadogAPIClient::V2::ObservabilityPipelineDatadogAgentSource.new({
+            id: "datadog-agent-source",
+            type: DatadogAPIClient::V2::ObservabilityPipelineDatadogAgentSourceType::DATADOG_AGENT,
+          }),
+        ],
+      }),
+      name: "Metrics Pipeline with Enrichment Table Reference Table",
+    }),
+    type: "pipelines",
+  }),
+})
+p api_instance.validate_pipeline(body)

features/v2/observability_pipelines.feature

@@ -149,6 +149,22 @@ Feature: Observability Pipelines
     And the response "data.attributes.config.destinations" has length 1
     And the response "data.attributes.config.destinations[0].id" is equal to "updated-datadog-logs-destination-id"
 
+  @team:DataDog/observability-pipelines
+  Scenario: Validate a metrics pipeline with enrichment table processor file lookup returns "OK" response
+    Given new "ValidatePipeline" request
+    And body with value {"data": {"attributes": {"config": {"pipeline_type": "metrics", "destinations": [{"id": "datadog-metrics-destination", "inputs": ["my-processor-group"], "type": "datadog_metrics"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "*", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "enrichment-table-processor", "include": "*", "type": "enrichment_table", "file": {"encoding": {"delimiter": ",", "type": "csv", "includes_headers": true}, "key": {"column": "service", "source": {"type": "tag", "name": "service"}}, "path": "/etc/enrichment/lookup.csv"}}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Metrics Pipeline with Enrichment Table File Lookup"}, "type": "pipelines"}}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "errors" has length 0
+
+  @team:DataDog/observability-pipelines
+  Scenario: Validate a metrics pipeline with enrichment table processor reference table returns "OK" response
+    Given new "ValidatePipeline" request
+    And body with value {"data": {"attributes": {"config": {"pipeline_type": "metrics", "destinations": [{"id": "datadog-metrics-destination", "inputs": ["my-processor-group"], "type": "datadog_metrics"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "*", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "enrichment-table-processor", "include": "*", "type": "enrichment_table", "reference_table": {"table_id": "metric-enrichment", "key": {"source": {"type": "metric_name"}}, "columns": ["environment", "team"]}}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Metrics Pipeline with Enrichment Table Reference Table"}, "type": "pipelines"}}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "errors" has length 0
+
   @team:DataDog/observability-pipelines
   Scenario: Validate a metrics pipeline with opentelemetry source returns "OK" response
     Given new "ValidatePipeline" request

generated-test/test-runner-data/manifest.json

@@ -7952,6 +7952,20 @@
       "scenario": "Update a pipeline returns \"OK\" response",
       "version": "v2"
     },
+    {
+      "feature": "Observability Pipelines",
+      "feature_file": "../../features/v2/observability_pipelines.feature",
+      "file": "v2/observability-pipelines/validate-a-metrics-pipeline-with-enrichment-table-processor-file-lookup-returns-ok-response.json",
+      "scenario": "Validate a metrics pipeline with enrichment table processor file lookup returns \"OK\" response",
+      "version": "v2"
+    },
+    {
+      "feature": "Observability Pipelines",
+      "feature_file": "../../features/v2/observability_pipelines.feature",
+      "file": "v2/observability-pipelines/validate-a-metrics-pipeline-with-enrichment-table-processor-reference-table-returns-ok-response.json",
+      "scenario": "Validate a metrics pipeline with enrichment table processor reference table returns \"OK\" response",
+      "version": "v2"
+    },
     {
       "feature": "Observability Pipelines",
       "feature_file": "../../features/v2/observability_pipelines.feature",