diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..f7fc82f3 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,50 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '24 1 * * 5' + +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 0c00c410..e36a9f1a 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -26,14 +26,14 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: - go-version: 1.23.x + go-version: 1.24.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 with: args: --verbose - version: v1.63.4 + version: v1.64.6 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..4793ce72 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,56 @@ +name: Scorecard supply-chain security + +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '26 18 * * 2' + push: + branches: [ "main" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4 + with: + sarif_file: results.sarif diff --git a/Makefile.common b/Makefile.common index d1576bb3..8cb38385 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.63.4 +GOLANGCI_LINT_VERSION ?= v1.64.6 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) diff --git a/README.md b/README.md index 954cc91b..f7d5342d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Common ![circleci](https://circleci.com/gh/prometheus/common/tree/main.svg?style=shield) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/common/badge)](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/common) + This repository contains Go libraries that are shared across Prometheus components and libraries. They are considered internal to Prometheus, without diff --git a/config/headers.go b/config/headers.go index 7276742e..9beaae26 100644 --- a/config/headers.go +++ b/config/headers.go @@ -24,9 +24,9 @@ import ( "strings" ) -// reservedHeaders that change the connection, are set by Prometheus, or can +// ReservedHeaders that change the connection, are set by Prometheus, or can // be changed otherwise. -var reservedHeaders = map[string]struct{}{ +var ReservedHeaders = map[string]struct{}{ "Authorization": {}, "Host": {}, "Content-Encoding": {}, @@ -72,7 +72,7 @@ func (h *Headers) SetDirectory(dir string) { // Validate validates the Headers config. func (h *Headers) Validate() error { for n := range h.Headers { - if _, ok := reservedHeaders[http.CanonicalHeaderKey(n)]; ok { + if _, ok := ReservedHeaders[http.CanonicalHeaderKey(n)]; ok { return fmt.Errorf("setting header %q is not allowed", http.CanonicalHeaderKey(n)) } } diff --git a/config/headers_test.go b/config/headers_test.go index 39c6f9ff..c807fbc3 100644 --- a/config/headers_test.go +++ b/config/headers_test.go @@ -22,10 +22,10 @@ import ( ) func TestReservedHeaders(t *testing.T) { - for k := range reservedHeaders { + for k := range ReservedHeaders { l := http.CanonicalHeaderKey(k) if k != l { - t.Errorf("reservedHeaders keys should be lowercase: got %q, expected %q", k, http.CanonicalHeaderKey(k)) + t.Errorf("ReservedHeaders keys should be lowercase: got %q, expected %q", k, http.CanonicalHeaderKey(k)) } } } diff --git a/expfmt/decode_test.go b/expfmt/decode_test.go index 10b12b66..759ff746 100644 --- a/expfmt/decode_test.go +++ b/expfmt/decode_test.go @@ -369,7 +369,7 @@ func TestProtoDecoder(t *testing.T) { var all model.Vector for { - model.NameValidationScheme = model.LegacyValidation + model.NameValidationScheme = model.LegacyValidation //nolint:staticcheck var smpls model.Vector err := dec.Decode(&smpls) if err != nil && errors.Is(err, io.EOF) { @@ -377,7 +377,7 @@ func TestProtoDecoder(t *testing.T) { } if scenario.legacyNameFail { require.Errorf(t, err, "Expected error when decoding without UTF-8 support enabled but got none") - model.NameValidationScheme = model.UTF8Validation + model.NameValidationScheme = model.UTF8Validation //nolint:staticcheck dec = &SampleDecoder{ Dec: &protoDecoder{r: strings.NewReader(scenario.in)}, Opts: &DecodeOptions{ diff --git a/go.mod b/go.mod index 4d62719b..b924a3f7 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,15 @@ go 1.21 require ( github.com/alecthomas/kingpin/v2 v2.4.0 - github.com/google/go-cmp v0.6.0 + github.com/google/go-cmp v0.7.0 github.com/julienschmidt/httprouter v1.3.0 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/prometheus/client_model v0.6.1 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.33.0 - golang.org/x/oauth2 v0.24.0 - google.golang.org/protobuf v1.36.1 + golang.org/x/net v0.35.0 + golang.org/x/oauth2 v0.25.0 + google.golang.org/protobuf v1.36.5 gopkg.in/yaml.v2 v2.4.0 ) @@ -27,8 +27,8 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index b5955f01..009bd785 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -43,16 +43,16 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/model/alert.go b/model/alert.go index bd3a39e3..460f554f 100644 --- a/model/alert.go +++ b/model/alert.go @@ -65,7 +65,7 @@ func (a *Alert) Resolved() bool { return a.ResolvedAt(time.Now()) } -// ResolvedAt returns true off the activity interval ended before +// ResolvedAt returns true iff the activity interval ended before // the given timestamp. func (a *Alert) ResolvedAt(ts time.Time) bool { if a.EndsAt.IsZero() { diff --git a/model/labels.go b/model/labels.go index 73b7aa3e..f4a38760 100644 --- a/model/labels.go +++ b/model/labels.go @@ -22,7 +22,7 @@ import ( ) const ( - // AlertNameLabel is the name of the label containing the an alert's name. + // AlertNameLabel is the name of the label containing the alert's name. AlertNameLabel = "alertname" // ExportedLabelPrefix is the prefix to prepend to the label names present in diff --git a/model/metric.go b/model/metric.go index 5766107c..a6b01755 100644 --- a/model/metric.go +++ b/model/metric.go @@ -27,13 +27,25 @@ import ( ) var ( - // NameValidationScheme determines the method of name validation to be used by - // all calls to IsValidMetricName() and LabelName IsValid(). Setting UTF-8 - // mode in isolation from other components that don't support UTF-8 may result - // in bugs or other undefined behavior. This value can be set to - // LegacyValidation during startup if a binary is not UTF-8-aware binaries. To - // avoid need for locking, this value should be set once, ideally in an - // init(), before multiple goroutines are started. + // NameValidationScheme determines the global default method of the name + // validation to be used by all calls to IsValidMetricName() and LabelName + // IsValid(). + // + // Deprecated: This variable should not be used and might be removed in the + // far future. If you wish to stick to the legacy name validation use + // `IsValidLegacyMetricName()` and `LabelName.IsValidLegacy()` methods + // instead. This variable is here as an escape hatch for emergency cases, + // given the recent change from `LegacyValidation` to `UTF8Validation`, e.g., + // to delay UTF-8 migrations in time or aid in debugging unforeseen results of + // the change. In such a case, a temporary assignment to `LegacyValidation` + // value in the `init()` function in your main.go or so, could be considered. + // + // Historically we opted for a global variable for feature gating different + // validation schemes in operations that were not otherwise easily adjustable + // (e.g. Labels yaml unmarshaling). That could have been a mistake, a separate + // Labels structure or package might have been a better choice. Given the + // change was made and many upgraded the common already, we live this as-is + // with this warning and learning for the future. NameValidationScheme = UTF8Validation // NameEscapingScheme defines the default way that names will be escaped when @@ -50,7 +62,7 @@ var ( type ValidationScheme int const ( - // LegacyValidation is a setting that requirets that metric and label names + // LegacyValidation is a setting that requires that all metric and label names // conform to the original Prometheus character requirements described by // MetricNameRE and LabelNameRE. LegacyValidation ValidationScheme = iota diff --git a/otlptranslator/constants.go b/otlptranslator/constants.go new file mode 100644 index 00000000..d719daa1 --- /dev/null +++ b/otlptranslator/constants.go @@ -0,0 +1,58 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package otlptranslator + +const ( + // MetricMetadataTypeKey is the key used to store the original Prometheus + // type in metric metadata: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata + MetricMetadataTypeKey = "prometheus.type" + // ExemplarTraceIDKey is the key used to store the trace ID in Prometheus + // exemplars: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#exemplars + ExemplarTraceIDKey = "trace_id" + // ExemplarSpanIDKey is the key used to store the Span ID in Prometheus + // exemplars: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#exemplars + ExemplarSpanIDKey = "span_id" + // ScopeInfoMetricName is the name of the metric used to preserve scope + // attributes in Prometheus format: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#instrumentation-scope + ScopeInfoMetricName = "otel_scope_info" + // ScopeNameLabelKey is the name of the label key used to identify the name + // of the OpenTelemetry scope which produced the metric: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#instrumentation-scope + ScopeNameLabelKey = "otel_scope_name" + // ScopeVersionLabelKey is the name of the label key used to identify the + // version of the OpenTelemetry scope which produced the metric: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#instrumentation-scope + ScopeVersionLabelKey = "otel_scope_version" + // TargetInfoMetricName is the name of the metric used to preserve resource + // attributes in Prometheus format: + // https://github.com/open-telemetry/opentelemetry-specification/blob/e6eccba97ebaffbbfad6d4358408a2cead0ec2df/specification/compatibility/prometheus_and_openmetrics.md#resource-attributes-1 + // It originates from OpenMetrics: + // https://github.com/OpenObservability/OpenMetrics/blob/1386544931307dff279688f332890c31b6c5de36/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems + TargetInfoMetricName = "target_info" +) + +type MetricType string + +const ( + MetricTypeNonMonotonicCounter MetricType = "non-monotonic-counter" + MetricTypeMonotonicCounter MetricType = "monotonic-counter" + MetricTypeGauge MetricType = "gauge" + MetricTypeHistogram MetricType = "histogram" + MetricTypeExponentialHistogram MetricType = "exponential-histogram" + MetricTypeSummary MetricType = "summary" + MetricTypeUnknown MetricType = "unknown" +) diff --git a/otlptranslator/doc.go b/otlptranslator/doc.go new file mode 100644 index 00000000..88a7fed4 --- /dev/null +++ b/otlptranslator/doc.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// otlptranslator is a dependency free package that contains the logic for translating information, such as metric name, unit and type, +// from OpenTelemetry metrics to valid Prometheus metric and label names. +// +// Use BuildCompliantMetricName to build a metric name that complies with traditional Prometheus naming conventions. +// Such conventions exist from a time when Prometheus didn't have support for full UTF-8 characters in metric names. +// For more details see: https://prometheus.io/docs/practices/naming/ +// +// Use BuildMetricName to build a metric name that will be accepted by Prometheus with full UTF-8 support. +// +// Use NormalizeLabel to normalize a label name to a valid format that can be used in Prometheus before UTF-8 characters were supported. +package otlptranslator diff --git a/otlptranslator/label_builder.go b/otlptranslator/label_builder.go new file mode 100644 index 00000000..deb3bbd0 --- /dev/null +++ b/otlptranslator/label_builder.go @@ -0,0 +1,54 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package otlptranslator + +import ( + "regexp" + "strings" + "unicode" +) + +var invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`) + +// Normalizes the specified label to follow Prometheus label names standard. +// +// See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels. +// +// Labels that start with non-letter rune will be prefixed with "key_". +// An exception is made for double-underscores which are allowed. +func NormalizeLabel(label string) string { + // Trivial case. + if len(label) == 0 { + return label + } + + label = SanitizeLabelName(label) + + // If label starts with a number, prepend with "key_". + if unicode.IsDigit(rune(label[0])) { + label = "key_" + label + } else if strings.HasPrefix(label, "_") && !strings.HasPrefix(label, "__") { + label = "key" + label + } + + return label +} + +// SanitizeLabelName replaces anything that doesn't match +// client_label.LabelNameRE with an underscore. +// Note: this does not handle all Prometheus label name restrictions (such as +// not starting with a digit 0-9), and hence should only be used if the label +// name is prefixed with a known valid string. +func SanitizeLabelName(name string) string { + return invalidLabelCharRE.ReplaceAllString(name, "_") +} diff --git a/otlptranslator/label_builder_bench_test.go b/otlptranslator/label_builder_bench_test.go new file mode 100644 index 00000000..6f2ba120 --- /dev/null +++ b/otlptranslator/label_builder_bench_test.go @@ -0,0 +1,35 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package otlptranslator + +import "testing" + +var labelBenchmarkInputs = []string{ + "", + "label:with:colons", + "LabelWithCapitalLetters", + "label!with&special$chars)", + "label_with_foreign_characters_字符", + "label.with.dots", + "123label", + "_label_starting_with_underscore", + "__label_starting_with_2underscores", +} + +func BenchmarkNormalizeLabel(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, input := range labelBenchmarkInputs { + NormalizeLabel(input) + } + } +} diff --git a/otlptranslator/label_builder_test.go b/otlptranslator/label_builder_test.go new file mode 100644 index 00000000..48856a21 --- /dev/null +++ b/otlptranslator/label_builder_test.go @@ -0,0 +1,44 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package otlptranslator + +import ( + "fmt" + "testing" +) + +func TestNormalizeLabel(t *testing.T) { + tests := []struct { + label string + expected string + }{ + {"", ""}, + {"label:with:colons", "label_with_colons"}, + {"LabelWithCapitalLetters", "LabelWithCapitalLetters"}, + {"label!with&special$chars)", "label_with_special_chars_"}, + {"label_with_foreign_characters_字符", "label_with_foreign_characters___"}, + {"label.with.dots", "label_with_dots"}, + {"123label", "key_123label"}, + {"_label_starting_with_underscore", "key_label_starting_with_underscore"}, + {"__label_starting_with_2underscores", "__label_starting_with_2underscores"}, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) { + result := NormalizeLabel(test.label) + if test.expected != result { + t.Errorf("expected %s, got %s", test.expected, result) + } + }) + } +} diff --git a/otlptranslator/metric_name_builder.go b/otlptranslator/metric_name_builder.go new file mode 100644 index 00000000..a6b7d275 --- /dev/null +++ b/otlptranslator/metric_name_builder.go @@ -0,0 +1,286 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package otlptranslator + +import ( + "regexp" + "slices" + "strings" + "unicode" +) + +// The map to translate OTLP units to Prometheus units +// OTLP metrics use the c/s notation as specified at https://ucum.org/ucum.html +// (See also https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/semantic_conventions/README.md#instrument-units) +// Prometheus best practices for units: https://prometheus.io/docs/practices/naming/#base-units +// OpenMetrics specification for units: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#units-and-base-units +var unitMap = map[string]string{ + // Time + "d": "days", + "h": "hours", + "min": "minutes", + "s": "seconds", + "ms": "milliseconds", + "us": "microseconds", + "ns": "nanoseconds", + + // Bytes + "By": "bytes", + "KiBy": "kibibytes", + "MiBy": "mebibytes", + "GiBy": "gibibytes", + "TiBy": "tibibytes", + "KBy": "kilobytes", + "MBy": "megabytes", + "GBy": "gigabytes", + "TBy": "terabytes", + + // SI + "m": "meters", + "V": "volts", + "A": "amperes", + "J": "joules", + "W": "watts", + "g": "grams", + + // Misc + "Cel": "celsius", + "Hz": "hertz", + "1": "", + "%": "percent", +} + +// The map that translates the "per" unit +// Example: s => per second (singular) +var perUnitMap = map[string]string{ + "s": "second", + "m": "minute", + "h": "hour", + "d": "day", + "w": "week", + "mo": "month", + "y": "year", +} + +var ( + nonMetricNameCharRE = regexp.MustCompile(`[^a-zA-Z0-9:]`) + // Regexp for metric name characters that should be replaced with _. + invalidMetricCharRE = regexp.MustCompile(`[^a-zA-Z0-9:_]`) + multipleUnderscoresRE = regexp.MustCompile(`__+`) +) + +// BuildMetricName builds a valid metric name but without following Prometheus naming conventions. +// It doesn't do any character transformation, it only prefixes the metric name with the namespace, if any, +// and adds metric type suffixes, e.g. "_total" for counters and unit suffixes. +// +// Differently from BuildCompliantMetricName, it doesn't check for the presence of unit and type suffixes. +// If "addMetricSuffixes" is true, it will add them anyway. +// +// Please use BuildCompliantMetricName for a metric name that follows Prometheus naming conventions. +func BuildMetricName(name, unit string, metricType MetricType, addMetricSuffixes bool) string { + if addMetricSuffixes { + mainUnitSuffix, perUnitSuffix := buildUnitSuffixes(unit) + if mainUnitSuffix != "" { + name = name + "_" + mainUnitSuffix + } + if perUnitSuffix != "" { + name = name + "_" + perUnitSuffix + } + + // Append _total for Counters + if metricType == MetricTypeMonotonicCounter { + name = name + "_total" + } + + // Append _ratio for metrics with unit "1" + // Some OTel receivers improperly use unit "1" for counters of objects + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aissue+some+metric+units+don%27t+follow+otel+semantic+conventions + // Until these issues have been fixed, we're appending `_ratio` for gauges ONLY + // Theoretically, counters could be ratios as well, but it's absurd (for mathematical reasons) + if unit == "1" && metricType == MetricTypeGauge { + name = name + "_ratio" + } + } + return name +} + +// BuildCompliantMetricName builds a Prometheus-compliant metric name for the specified metric. +// +// Metric name is prefixed with specified namespace and underscore (if any). +// Namespace is not cleaned up. Make sure specified namespace follows Prometheus +// naming convention. +// +// See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels, +// https://prometheus.io/docs/practices/naming/#metric-and-label-naming +// and https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus. +func BuildCompliantMetricName(name, unit string, metricType MetricType, addMetricSuffixes bool) string { + // Full normalization following standard Prometheus naming conventions + if addMetricSuffixes { + return normalizeName(name, unit, metricType) + } + + // Simple case (no full normalization, no units, etc.). + metricName := strings.Join(strings.FieldsFunc(name, func(r rune) bool { + return invalidMetricCharRE.MatchString(string(r)) + }), "_") + + // Metric name starts with a digit? Prefix it with an underscore. + if metricName != "" && unicode.IsDigit(rune(metricName[0])) { + metricName = "_" + metricName + } + + return metricName +} + +// Build a normalized name for the specified metric. +func normalizeName(metric, unit string, metricType MetricType) string { + // Split metric name into "tokens" (of supported metric name runes). + // Note that this has the side effect of replacing multiple consecutive underscores with a single underscore. + // This is part of the OTel to Prometheus specification: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus. + nameTokens := strings.FieldsFunc( + metric, + func(r rune) bool { return nonMetricNameCharRE.MatchString(string(r)) }, + ) + + mainUnitSuffix, perUnitSuffix := buildUnitSuffixes(unit) + nameTokens = addUnitTokens(nameTokens, CleanUpString(mainUnitSuffix), CleanUpString(perUnitSuffix)) + + // Append _total for Counters + if metricType == MetricTypeMonotonicCounter { + nameTokens = append(removeItem(nameTokens, "total"), "total") + } + + // Append _ratio for metrics with unit "1" + // Some OTel receivers improperly use unit "1" for counters of objects + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aissue+some+metric+units+don%27t+follow+otel+semantic+conventions + // Until these issues have been fixed, we're appending `_ratio` for gauges ONLY + // Theoretically, counters could be ratios as well, but it's absurd (for mathematical reasons) + if unit == "1" && metricType == MetricTypeGauge { + nameTokens = append(removeItem(nameTokens, "ratio"), "ratio") + } + + // Build the string from the tokens, separated with underscores + normalizedName := strings.Join(nameTokens, "_") + + // Metric name cannot start with a digit, so prefix it with "_" in this case + if normalizedName != "" && unicode.IsDigit(rune(normalizedName[0])) { + normalizedName = "_" + normalizedName + } + + return normalizedName +} + +// buildUnitSuffixes builds the main and per unit suffixes for the specified unit +// but doesn't do any special character transformation to accommodate Prometheus naming conventions. +// Removing trailing underscores or appending suffixes is done in the caller. +func buildUnitSuffixes(unit string) (mainUnitSuffix, perUnitSuffix string) { + // Split unit at the '/' if any + unitTokens := strings.SplitN(unit, "/", 2) + + if len(unitTokens) > 0 { + // Main unit + // Update if not blank and doesn't contain '{}' + mainUnitOTel := strings.TrimSpace(unitTokens[0]) + if mainUnitOTel != "" && !strings.ContainsAny(mainUnitOTel, "{}") { + mainUnitSuffix = unitMapGetOrDefault(mainUnitOTel) + } + + // Per unit + // Update if not blank and doesn't contain '{}' + if len(unitTokens) > 1 && unitTokens[1] != "" { + perUnitOTel := strings.TrimSpace(unitTokens[1]) + if perUnitOTel != "" && !strings.ContainsAny(perUnitOTel, "{}") { + perUnitSuffix = perUnitMapGetOrDefault(perUnitOTel) + } + if perUnitSuffix != "" { + perUnitSuffix = "per_" + perUnitSuffix + } + } + } + + return mainUnitSuffix, perUnitSuffix +} + +// Retrieve the Prometheus "basic" unit corresponding to the specified "basic" unit +// Returns the specified unit if not found in unitMap +func unitMapGetOrDefault(unit string) string { + if promUnit, ok := unitMap[unit]; ok { + return promUnit + } + return unit +} + +// Retrieve the Prometheus "per" unit corresponding to the specified "per" unit +// Returns the specified unit if not found in perUnitMap +func perUnitMapGetOrDefault(perUnit string) string { + if promPerUnit, ok := perUnitMap[perUnit]; ok { + return promPerUnit + } + return perUnit +} + +// addUnitTokens will add the suffixes to the nameTokens if they are not already present. +// It will also remove trailing underscores from the main suffix to avoid double underscores +// when joining the tokens. +// +// If the 'per' unit ends with underscore, the underscore will be removed. If the per unit is just +// 'per_', it will be entirely removed. +func addUnitTokens(nameTokens []string, mainUnitSuffix, perUnitSuffix string) []string { + if slices.Contains(nameTokens, mainUnitSuffix) { + mainUnitSuffix = "" + } + + if perUnitSuffix == "per_" { + perUnitSuffix = "" + } else { + perUnitSuffix = strings.TrimSuffix(perUnitSuffix, "_") + if slices.Contains(nameTokens, perUnitSuffix) { + perUnitSuffix = "" + } + } + + if perUnitSuffix != "" { + mainUnitSuffix = strings.TrimSuffix(mainUnitSuffix, "_") + } + + if mainUnitSuffix != "" { + nameTokens = append(nameTokens, mainUnitSuffix) + } + if perUnitSuffix != "" { + nameTokens = append(nameTokens, perUnitSuffix) + } + return nameTokens +} + +// CleanUpString cleans up a string so it matches model.LabelNameRE. +// CleanUpString is usually used to clean up unit strings, but can be used for any string, e.g. namespaces. +func CleanUpString(s string) string { + // Multiple consecutive underscores are replaced with a single underscore. + // This is part of the OTel to Prometheus specification: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.38.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus. + return strings.TrimPrefix(multipleUnderscoresRE.ReplaceAllString( + nonMetricNameCharRE.ReplaceAllString(s, "_"), + "_", + ), "_") +} + +// Remove the specified value from the slice +func removeItem(slice []string, value string) []string { + newSlice := make([]string, 0, len(slice)) + for _, sliceEntry := range slice { + if sliceEntry != value { + newSlice = append(newSlice, sliceEntry) + } + } + return newSlice +} diff --git a/otlptranslator/metric_name_builder_bench_test.go b/otlptranslator/metric_name_builder_bench_test.go new file mode 100644 index 00000000..bc0851d3 --- /dev/null +++ b/otlptranslator/metric_name_builder_bench_test.go @@ -0,0 +1,134 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package otlptranslator + +import ( + "fmt" + "testing" +) + +var benchmarkInputs = []struct { + name string + metricName string + unit string + metricType MetricType +}{ + { + name: "simple_metric", + metricName: "http_requests", + unit: "", + metricType: MetricTypeGauge, + }, + { + name: "compound_unit", + metricName: "request_throughput", + unit: "By/s", + metricType: MetricTypeMonotonicCounter, + }, + { + name: "complex_unit", + metricName: "disk_usage", + unit: "KiBy/m", + metricType: MetricTypeGauge, + }, + { + name: "ratio_metric", + metricName: "cpu_utilization", + unit: "1", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_dots", + metricName: "system.cpu.usage.idle", + unit: "%", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_unicode", + metricName: "メモリ使用率", + unit: "By", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_special_chars", + metricName: "error-rate@host{instance}/service#component", + unit: "ms", + metricType: MetricTypeMonotonicCounter, + }, + { + name: "metric_with_multiple_slashes", + metricName: "network/throughput/total", + unit: "By/s/min", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_spaces", + metricName: "api response time total", + unit: "ms", + metricType: MetricTypeMonotonicCounter, + }, + { + name: "metric_with_curly_braces", + metricName: "custom_{tag}_metric", + unit: "{custom}/s", + metricType: MetricTypeGauge, + }, + { + name: "metric_starting_with_digit", + metricName: "5xx_error_count", + unit: "1", + metricType: MetricTypeMonotonicCounter, + }, + { + name: "empty_metric", + metricName: "", + unit: "", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_SI_units", + metricName: "power_consumption", + unit: "W", + metricType: MetricTypeGauge, + }, + { + name: "metric_with_temperature", + metricName: "server_temperature", + unit: "Cel", + metricType: MetricTypeGauge, + }, +} + +func BenchmarkBuildMetricName(b *testing.B) { + for _, addSuffixes := range []bool{true, false} { + b.Run(fmt.Sprintf("with_metric_suffixes=%t", addSuffixes), func(b *testing.B) { + for _, input := range benchmarkInputs { + for i := 0; i < b.N; i++ { + BuildMetricName(input.metricName, input.unit, input.metricType, addSuffixes) + } + } + }) + } +} + +func BenchmarkBuildCompliantMetricName(b *testing.B) { + for _, addSuffixes := range []bool{true, false} { + b.Run(fmt.Sprintf("with_metric_suffixes=%t", addSuffixes), func(b *testing.B) { + for _, input := range benchmarkInputs { + for i := 0; i < b.N; i++ { + BuildCompliantMetricName(input.metricName, input.unit, input.metricType, addSuffixes) + } + } + }) + } +} diff --git a/otlptranslator/metric_name_builder_test.go b/otlptranslator/metric_name_builder_test.go new file mode 100644 index 00000000..49b729b4 --- /dev/null +++ b/otlptranslator/metric_name_builder_test.go @@ -0,0 +1,428 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package otlptranslator + +import ( + "reflect" + "testing" +) + +func TestBuildMetricName(t *testing.T) { + tests := []struct { + name string + metricName string + unit string + metricType MetricType + addMetricSuffixes bool + expected string + }{ + { + name: "simple metric without suffixes", + metricName: "http_requests", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "http_requests", + }, + { + name: "counter with total suffix", + metricName: "http_requests", + unit: "", + metricType: MetricTypeMonotonicCounter, + addMetricSuffixes: true, + expected: "http_requests_total", + }, + { + name: "gauge with time unit", + metricName: "request_duration", + unit: "s", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "request_duration_seconds", + }, + { + name: "counter with time unit", + metricName: "request_duration", + unit: "ms", + metricType: MetricTypeMonotonicCounter, + addMetricSuffixes: true, + expected: "request_duration_milliseconds_total", + }, + { + name: "gauge with compound unit", + metricName: "throughput", + unit: "By/s", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "throughput_bytes_per_second", + }, + { + name: "ratio metric", + metricName: "cpu_utilization", + unit: "1", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "cpu_utilization_ratio", + }, + { + name: "counter with unit 1 (no ratio suffix)", + metricName: "error_count", + unit: "1", + metricType: MetricTypeMonotonicCounter, + addMetricSuffixes: true, + expected: "error_count_total", + }, + { + name: "metric with byte units", + metricName: "memory_usage", + unit: "MiBy", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "memory_usage_mebibytes", + }, + { + name: "metric with SI units", + metricName: "temperature", + unit: "Cel", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "temperature_celsius", + }, + { + name: "metric with dots", + metricName: "system.cpu.usage", + unit: "1", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "system.cpu.usage_ratio", + }, + { + name: "metric with japanese characters (memory usage rate)", + metricName: "メモリ使用率", // memori shiyouritsu (memory usage rate) xD + unit: "By", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "メモリ使用率_bytes", + }, + { + name: "metric with mixed special characters (system.memory.usage.rate)", + metricName: "system.メモリ.usage.率", // system.memory.usage.rate + unit: "By/s", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "system.メモリ.usage.率_bytes_per_second", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := BuildMetricName(tt.metricName, tt.unit, tt.metricType, tt.addMetricSuffixes) + if tt.expected != result { + t.Errorf("expected %s, got %s", tt.expected, result) + } + }) + } +} + +func TestBuildUnitSuffixes(t *testing.T) { + tests := []struct { + name string + unit string + expectedMain string + expectedPerUnit string + }{ + { + name: "empty unit", + unit: "", + expectedMain: "", + expectedPerUnit: "", + }, + { + name: "simple time unit", + unit: "s", + expectedMain: "seconds", + expectedPerUnit: "", + }, + { + name: "compound unit", + unit: "By/s", + expectedMain: "bytes", + expectedPerUnit: "per_second", + }, + { + name: "complex compound unit", + unit: "KiBy/m", + expectedMain: "kibibytes", + expectedPerUnit: "per_minute", + }, + { + name: "unit with spaces", + unit: " ms / s ", + expectedMain: "milliseconds", + expectedPerUnit: "per_second", + }, + { + name: "invalid unit", + unit: "invalid", + expectedMain: "invalid", + expectedPerUnit: "", + }, + { + name: "unit with curly braces", + unit: "{custom}/s", + expectedMain: "", + expectedPerUnit: "per_second", + }, + { + name: "multiple slashes", + unit: "By/s/h", + expectedMain: "bytes", + expectedPerUnit: "per_s/h", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mainUnit, perUnit := buildUnitSuffixes(tt.unit) + if tt.expectedMain != mainUnit { + t.Errorf("expected main unit %s, got %s", tt.expectedMain, mainUnit) + } + if tt.expectedPerUnit != perUnit { + t.Errorf("expected per unit %s, got %s", tt.expectedPerUnit, perUnit) + } + }) + } +} + +func TestBuildCompliantMetricName(t *testing.T) { + tests := []struct { + name string + metricName string + unit string + metricType MetricType + addMetricSuffixes bool + expected string + }{ + { + name: "simple valid metric name", + metricName: "http_requests", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "http_requests", + }, + { + name: "metric name with invalid characters", + metricName: "http-requests@in_flight", + unit: "", + metricType: MetricTypeNonMonotonicCounter, + addMetricSuffixes: false, + expected: "http_requests_in_flight", + }, + { + name: "metric name starting with digit", + metricName: "5xx_errors", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "_5xx_errors", + }, + { + name: "metric name starting with digit, with suffixes", + metricName: "5xx_errors", + unit: "", + metricType: MetricTypeMonotonicCounter, + addMetricSuffixes: true, + expected: "_5xx_errors_total", + }, + { + name: "metric name with multiple consecutive invalid chars", + metricName: "api..//request--time", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "api_request_time", + }, + { + name: "full normalization with units and type", + metricName: "system.cpu-utilization", + unit: "ms/s", + metricType: MetricTypeMonotonicCounter, + addMetricSuffixes: true, + expected: "system_cpu_utilization_milliseconds_per_second_total", + }, + { + name: "metric with special characters and ratio", + metricName: "memory.usage%rate", + unit: "1", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "memory_usage_rate_ratio", + }, + { + name: "metric with unicode characters", + metricName: "error_rate_£_€_¥", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "error_rate_____", + }, + { + name: "metric with multiple spaces", + metricName: "api response time", + unit: "ms", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "api_response_time_milliseconds", + }, + { + name: "metric with colons (valid prometheus chars)", + metricName: "app:request:latency", + unit: "s", + metricType: MetricTypeGauge, + addMetricSuffixes: true, + expected: "app:request:latency_seconds", + }, + { + name: "empty metric name", + metricName: "", + unit: "", + metricType: MetricTypeGauge, + addMetricSuffixes: false, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := BuildCompliantMetricName(tt.metricName, tt.unit, tt.metricType, tt.addMetricSuffixes) + if tt.expected != result { + t.Errorf("expected %s, got %s", tt.expected, result) + } + }) + } +} + +func TestAddUnitTokens(t *testing.T) { + tests := []struct { + nameTokens []string + mainUnitSuffix string + perUnitSuffix string + expected []string + }{ + {[]string{}, "", "", []string{}}, + {[]string{"token1"}, "main", "", []string{"token1", "main"}}, + {[]string{"token1"}, "", "per", []string{"token1", "per"}}, + {[]string{"token1"}, "main", "per", []string{"token1", "main", "per"}}, + {[]string{"token1", "per"}, "main", "per", []string{"token1", "per", "main"}}, + {[]string{"token1", "main"}, "main", "per", []string{"token1", "main", "per"}}, + {[]string{"token1"}, "main_", "per", []string{"token1", "main", "per"}}, + {[]string{"token1"}, "main_unit", "per_seconds_", []string{"token1", "main_unit", "per_seconds"}}, // trailing underscores are removed + {[]string{"token1"}, "main_unit", "per_", []string{"token1", "main_unit"}}, // 'per_' is removed entirely + } + + for _, test := range tests { + result := addUnitTokens(test.nameTokens, test.mainUnitSuffix, test.perUnitSuffix) + if !reflect.DeepEqual(test.expected, result) { + t.Errorf("expected %v, got %v", test.expected, result) + } + } +} + +func TestRemoveItem(t *testing.T) { + if !reflect.DeepEqual([]string{}, removeItem([]string{}, "test")) { + t.Errorf("expected %v, got %v", []string{}, removeItem([]string{}, "test")) + } + if !reflect.DeepEqual([]string{}, removeItem([]string{}, "")) { + t.Errorf("expected %v, got %v", []string{}, removeItem([]string{}, "")) + } + if !reflect.DeepEqual([]string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "d")) { + t.Errorf("expected %v, got %v", []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "d")) + } + if !reflect.DeepEqual([]string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "")) { + t.Errorf("expected %v, got %v", []string{"a", "b", "c"}, removeItem([]string{"a", "b", "c"}, "")) + } + if !reflect.DeepEqual([]string{"a", "b"}, removeItem([]string{"a", "b", "c"}, "c")) { + t.Errorf("expected %v, got %v", []string{"a", "b"}, removeItem([]string{"a", "b", "c"}, "c")) + } + if !reflect.DeepEqual([]string{"a", "c"}, removeItem([]string{"a", "b", "c"}, "b")) { + t.Errorf("expected %v, got %v", []string{"a", "c"}, removeItem([]string{"a", "b", "c"}, "b")) + } + if !reflect.DeepEqual([]string{"b", "c"}, removeItem([]string{"a", "b", "c"}, "a")) { + t.Errorf("expected %v, got %v", []string{"b", "c"}, removeItem([]string{"a", "b", "c"}, "a")) + } +} + +func TestCleanUpStrings(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "already valid string", + input: "valid_metric_name", + expected: "valid_metric_name", + }, + { + name: "invalid characters", + input: "metric-name@with#special$chars", + expected: "metric_name_with_special_chars", + }, + { + name: "multiple consecutive invalid chars", + input: "metric---name###special", + expected: "metric_name_special", + }, + { + name: "leading invalid chars", + input: "@#$metric_name", + expected: "metric_name", + }, + { + name: "trailing invalid chars", + input: "metric_name@#$", + expected: "metric_name_", + }, + { + name: "multiple consecutive underscores", + input: "metric___name____test", + expected: "metric_name_test", + }, + { + name: "empty string", + input: "", + expected: "", + }, + { + name: "only invalid chars", + input: "@#$%^&", + expected: "", + }, + { + name: "colons are valid", + input: "system.cpu:usage.rate", + expected: "system_cpu:usage_rate", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := CleanUpString(tt.input) + if result != tt.expected { + t.Errorf("expected %q, got %q", tt.expected, result) + } + }) + } +} diff --git a/promslog/flag/flag.go b/promslog/flag/flag.go index 0a164fcc..85c67b25 100644 --- a/promslog/flag/flag.go +++ b/promslog/flag/flag.go @@ -42,12 +42,12 @@ var FormatFlagHelp = "Output format of log messages. One of: [" + strings.Join(p // AddFlags adds the flags used by this package to the Kingpin application. // To use the default Kingpin application, call AddFlags(kingpin.CommandLine) func AddFlags(a *kingpin.Application, config *promslog.Config) { - config.Level = &promslog.AllowedLevel{} + config.Level = promslog.NewLevel() a.Flag(LevelFlagName, LevelFlagHelp). Default("info").HintOptions(promslog.LevelFlagOptions...). SetValue(config.Level) - config.Format = &promslog.AllowedFormat{} + config.Format = promslog.NewFormat() a.Flag(FormatFlagName, FormatFlagHelp). Default("logfmt").HintOptions(promslog.FormatFlagOptions...). SetValue(config.Format) diff --git a/promslog/slog.go b/promslog/slog.go index 6e8fbabc..f9f89966 100644 --- a/promslog/slog.go +++ b/promslog/slog.go @@ -25,73 +25,43 @@ import ( "path/filepath" "strconv" "strings" + "time" ) +// LogStyle represents the common logging formats in the Prometheus ecosystem. type LogStyle string const ( SlogStyle LogStyle = "slog" GoKitStyle LogStyle = "go-kit" + + reservedKeyPrefix = "logged_" ) var ( - LevelFlagOptions = []string{"debug", "info", "warn", "error"} + // LevelFlagOptions represents allowed logging levels. + LevelFlagOptions = []string{"debug", "info", "warn", "error"} + // FormatFlagOptions represents allowed formats. FormatFlagOptions = []string{"logfmt", "json"} - callerAddFunc = false - defaultWriter = os.Stderr - goKitStyleReplaceAttrFunc = func(groups []string, a slog.Attr) slog.Attr { - key := a.Key - switch key { - case slog.TimeKey: - a.Key = "ts" - - // This timestamp format differs from RFC3339Nano by using .000 instead - // of .999999999 which changes the timestamp from 9 variable to 3 fixed - // decimals (.130 instead of .130987456). - t := a.Value.Time() - a.Value = slog.StringValue(t.UTC().Format("2006-01-02T15:04:05.000Z07:00")) - case slog.SourceKey: - a.Key = "caller" - src, _ := a.Value.Any().(*slog.Source) - - switch callerAddFunc { - case true: - a.Value = slog.StringValue(filepath.Base(src.File) + "(" + filepath.Base(src.Function) + "):" + strconv.Itoa(src.Line)) - default: - a.Value = slog.StringValue(filepath.Base(src.File) + ":" + strconv.Itoa(src.Line)) - } - case slog.LevelKey: - a.Value = slog.StringValue(strings.ToLower(a.Value.String())) - default: - } - - return a - } - defaultReplaceAttrFunc = func(groups []string, a slog.Attr) slog.Attr { - key := a.Key - switch key { - case slog.TimeKey: - t := a.Value.Time() - a.Value = slog.TimeValue(t.UTC()) - case slog.SourceKey: - src, _ := a.Value.Any().(*slog.Source) - a.Value = slog.StringValue(filepath.Base(src.File) + ":" + strconv.Itoa(src.Line)) - default: - } - - return a - } + defaultWriter = os.Stderr ) -// AllowedLevel is a settable identifier for the minimum level a log entry -// must be have. -type AllowedLevel struct { - s string +// Level controls a logging level, with an info default. +// It wraps slog.LevelVar with string-based level control. +// Level is safe to be used concurrently. +type Level struct { lvl *slog.LevelVar } -func (l *AllowedLevel) UnmarshalYAML(unmarshal func(interface{}) error) error { +// NewLevel returns a new Level. +func NewLevel() *Level { + return &Level{ + lvl: &slog.LevelVar{}, + } +} + +func (l *Level) UnmarshalYAML(unmarshal func(interface{}) error) error { var s string type plain string if err := unmarshal((*plain)(&s)); err != nil { @@ -100,55 +70,60 @@ func (l *AllowedLevel) UnmarshalYAML(unmarshal func(interface{}) error) error { if s == "" { return nil } - lo := &AllowedLevel{} - if err := lo.Set(s); err != nil { + if err := l.Set(s); err != nil { return err } - *l = *lo return nil } -func (l *AllowedLevel) String() string { - return l.s -} - -// Set updates the value of the allowed level. -func (l *AllowedLevel) Set(s string) error { - if l.lvl == nil { - l.lvl = &slog.LevelVar{} +// String returns the current level. +func (l *Level) String() string { + switch l.lvl.Level() { + case slog.LevelDebug: + return "debug" + case slog.LevelInfo: + return "info" + case slog.LevelWarn: + return "warn" + case slog.LevelError: + return "error" + default: + return "" } +} +// Set updates the logging level with the validation. +func (l *Level) Set(s string) error { switch strings.ToLower(s) { case "debug": l.lvl.Set(slog.LevelDebug) - callerAddFunc = true case "info": l.lvl.Set(slog.LevelInfo) - callerAddFunc = false case "warn": l.lvl.Set(slog.LevelWarn) - callerAddFunc = false case "error": l.lvl.Set(slog.LevelError) - callerAddFunc = false default: return fmt.Errorf("unrecognized log level %s", s) } - l.s = s return nil } -// AllowedFormat is a settable identifier for the output format that the logger can have. -type AllowedFormat struct { +// Format controls a logging output format. +// Not concurrency-safe. +type Format struct { s string } -func (f *AllowedFormat) String() string { +// NewFormat creates a new Format. +func NewFormat() *Format { return &Format{} } + +func (f *Format) String() string { return f.s } // Set updates the value of the allowed format. -func (f *AllowedFormat) Set(s string) error { +func (f *Format) Set(s string) error { switch s { case "logfmt", "json": f.s = s @@ -160,18 +135,113 @@ func (f *AllowedFormat) Set(s string) error { // Config is a struct containing configurable settings for the logger type Config struct { - Level *AllowedLevel - Format *AllowedFormat + Level *Level + Format *Format Style LogStyle Writer io.Writer } +func newGoKitStyleReplaceAttrFunc(lvl *Level) func(groups []string, a slog.Attr) slog.Attr { + return func(groups []string, a slog.Attr) slog.Attr { + key := a.Key + switch key { + case slog.TimeKey, "ts": + if t, ok := a.Value.Any().(time.Time); ok { + a.Key = "ts" + + // This timestamp format differs from RFC3339Nano by using .000 instead + // of .999999999 which changes the timestamp from 9 variable to 3 fixed + // decimals (.130 instead of .130987456). + a.Value = slog.StringValue(t.UTC().Format("2006-01-02T15:04:05.000Z07:00")) + } else { + // If we can't cast the any from the value to a + // time.Time, it means the caller logged + // another attribute with a key of `ts`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to `logged_ts`. + a.Key = reservedKeyPrefix + key + } + case slog.SourceKey, "caller": + if src, ok := a.Value.Any().(*slog.Source); ok { + a.Key = "caller" + switch lvl.String() { + case "debug": + a.Value = slog.StringValue(filepath.Base(src.File) + "(" + filepath.Base(src.Function) + "):" + strconv.Itoa(src.Line)) + default: + a.Value = slog.StringValue(filepath.Base(src.File) + ":" + strconv.Itoa(src.Line)) + } + } else { + // If we can't cast the any from the value to + // an *slog.Source, it means the caller logged + // another attribute with a key of `caller`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to + // `logged_caller`. + a.Key = reservedKeyPrefix + key + } + case slog.LevelKey: + if lvl, ok := a.Value.Any().(slog.Level); ok { + a.Value = slog.StringValue(strings.ToLower(lvl.String())) + } else { + // If we can't cast the any from the value to + // an slog.Level, it means the caller logged + // another attribute with a key of `level`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to `logged_level`. + a.Key = reservedKeyPrefix + key + } + default: + } + return a + } +} + +func defaultReplaceAttr(_ []string, a slog.Attr) slog.Attr { + key := a.Key + switch key { + case slog.TimeKey: + if t, ok := a.Value.Any().(time.Time); ok { + a.Value = slog.TimeValue(t.UTC()) + } else { + // If we can't cast the any from the value to a + // time.Time, it means the caller logged + // another attribute with a key of `time`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to `logged_time`. + a.Key = reservedKeyPrefix + key + } + case slog.SourceKey: + if src, ok := a.Value.Any().(*slog.Source); ok { + a.Value = slog.StringValue(filepath.Base(src.File) + ":" + strconv.Itoa(src.Line)) + } else { + // If we can't cast the any from the value to + // an *slog.Source, it means the caller logged + // another attribute with a key of `source`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to + // `logged_source`. + a.Key = reservedKeyPrefix + key + } + case slog.LevelKey: + if _, ok := a.Value.Any().(slog.Level); !ok { + // If we can't cast the any from the value to + // an slog.Level, it means the caller logged + // another attribute with a key of `level`. + // Prevent duplicate keys (necessary for proper + // JSON) by renaming the key to + // `logged_level`. + a.Key = reservedKeyPrefix + key + } + default: + } + return a +} + // New returns a new slog.Logger. Each logged line will be annotated // with a timestamp. The output always goes to stderr. func New(config *Config) *slog.Logger { if config.Level == nil { - config.Level = &AllowedLevel{} - _ = config.Level.Set("info") + config.Level = NewLevel() } if config.Writer == nil { @@ -181,11 +251,11 @@ func New(config *Config) *slog.Logger { logHandlerOpts := &slog.HandlerOptions{ Level: config.Level.lvl, AddSource: true, - ReplaceAttr: defaultReplaceAttrFunc, + ReplaceAttr: defaultReplaceAttr, } if config.Style == GoKitStyle { - logHandlerOpts.ReplaceAttr = goKitStyleReplaceAttrFunc + logHandlerOpts.ReplaceAttr = newGoKitStyleReplaceAttrFunc(config.Level) } if config.Format != nil && config.Format.s == "json" { diff --git a/promslog/slog_test.go b/promslog/slog_test.go index fc824e04..ea4e176c 100644 --- a/promslog/slog_test.go +++ b/promslog/slog_test.go @@ -43,29 +43,29 @@ func TestDefaultConfig(t *testing.T) { } func TestUnmarshallLevel(t *testing.T) { - l := &AllowedLevel{} + l := NewLevel() err := yaml.Unmarshal([]byte(`debug`), l) if err != nil { t.Error(err) } - if l.s != "debug" { - t.Errorf("expected %s, got %s", "debug", l.s) + if got := l.String(); got != "debug" { + t.Errorf("expected %s, got %s", "debug", got) } } func TestUnmarshallEmptyLevel(t *testing.T) { - l := &AllowedLevel{} + l := NewLevel() err := yaml.Unmarshal([]byte(``), l) if err != nil { t.Error(err) } - if l.s != "" { - t.Errorf("expected empty level, got %s", l.s) + if got := l.String(); got != "info" { + t.Errorf("expected info (default) level, got %s", got) } } func TestUnmarshallBadLevel(t *testing.T) { - l := &AllowedLevel{} + l := NewLevel() err := yaml.Unmarshal([]byte(`debugg`), l) if err == nil { t.Error("expected error") @@ -74,8 +74,8 @@ func TestUnmarshallBadLevel(t *testing.T) { if err.Error() != expErr { t.Errorf("expected error %s, got %s", expErr, err.Error()) } - if l.s != "" { - t.Errorf("expected empty level, got %s", l.s) + if got := l.String(); got != "info" { + t.Errorf("expected info (default) level, got %s", got) } } @@ -188,3 +188,42 @@ func TestTruncateSourceFileName_GoKitStyle(t *testing.T) { t.Errorf("Expected no directory separators in caller, got: %s", output) } } + +func TestReservedKeys(t *testing.T) { + var buf bytes.Buffer + reservedKeyTestVal := "surprise! I'm a string" + + tests := map[string]struct { + logStyle LogStyle + levelKey string + sourceKey string + timeKey string + }{ + "slog_log_style": {logStyle: SlogStyle, levelKey: "level", sourceKey: "source", timeKey: "time"}, + "go-kit_log_style": {logStyle: GoKitStyle, levelKey: "level", sourceKey: "caller", timeKey: "ts"}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + buf.Reset() // Ensure buf is reset prior to tests + config := &Config{Writer: &buf, Style: tc.logStyle} + logger := New(config) + + logger.LogAttrs(context.Background(), + slog.LevelInfo, + "reserved keys test for "+name, + slog.String(tc.levelKey, reservedKeyTestVal), + slog.String(tc.sourceKey, reservedKeyTestVal), + slog.String(tc.timeKey, reservedKeyTestVal), + ) + + output := buf.String() + require.Containsf(t, output, fmt.Sprintf("%s%s=\"%s\"", reservedKeyPrefix, tc.levelKey, reservedKeyTestVal), "Expected duplicate level key to be renamed") + require.Containsf(t, output, fmt.Sprintf("%s%s=\"%s\"", reservedKeyPrefix, tc.sourceKey, reservedKeyTestVal), "Expected duplicate source key to be renamed") + require.Containsf(t, output, fmt.Sprintf("%s%s=\"%s\"", reservedKeyPrefix, tc.timeKey, reservedKeyTestVal), "Expected duplicate time key to be renamed") + + // Print logs for humans to see, if needed. + fmt.Println(buf.String()) + }) + } +}