The flattenKey function is called whenever a gauge, counter, or sample is set when using the Prometheus sink. The change most notably uses a replacer rather than a regex. The hash formatting also now uses + for concatenation rather than fmt.Sprintf for an addition perf improvement.
The Nomad team recently looked into some Nomad metrics which indicated a fair amount of CPU time spent within the Prometheus flattenKey function. While this behaviour was likely exacerbated by other cluster stresses, a small investigation found a couple of potential improvements to the function to improve its performance.
The flattenKey function is called whenever a gauge, counter, or sample is set when using the Prometheus sink. The change most notably uses a replacer rather than a regex. The hash formatting also now uses + for concatenation rather than fmt.Sprintf for an addition perf improvement.
The Nomad team recently looked into some Nomad metrics which indicated a fair amount of CPU time spent within the Prometheus
flattenKeyfunction. While this behaviour was likely exacerbated by other cluster stresses, a small investigation found a couple of potential improvements to the function to improve its performance.Benchmark Code
```go package go_metrics import ( "fmt" "regexp" "strings" "testing" "github.com/armon/go-metrics" ) func BenchmarkPrometheus_flattenKey(b *testing.B) { testCases := []struct { name string inputParts []string inputLabels []metrics.Label }{ { name: "3_parts_1_label_", inputParts: []string{"nomad", "client", "allocs"}, inputLabels: []metrics.Label{ {Name: "alloc_id", Value: "88ea10d6-bf55-2883-b263-1b0ec007d254"}, }, }, { name: "4_parts_5_labels_", inputParts: []string{"client", "host", "memory", "total"}, inputLabels: []metrics.Label{ {Name: "node_id", Value: "88ea10d6-bf55-2883-b263-1b0ec007d254"}, {Name: "datacenter", Value: "my-datacenter"}, {Name: "node_class", Value: "none"}, {Name: "node_status", Value: "ready"}, {Name: "node_scheduling_eligibility", Value: "eligible"}, }, }, { name: "forbidden_chars_", inputParts: []string{"client", ".host", ".memory", ".total"}, inputLabels: []metrics.Label{ {Name: "node_id", Value: "88ea10d6-bf55-2883-b263-1b0ec007d254"}, {Name: "datacenter", Value: "my-datacenter"}, {Name: "node_class", Value: "none"}, {Name: "node_status", Value: "ready"}, {Name: "node_scheduling_eligibility", Value: "eligible"}, }, }, } for _, tc := range testCases { b.Run(tc.name+"current", func(b *testing.B) { for i := 0; i < b.N; i++ { _, _ = flattenKey(tc.inputParts, tc.inputLabels) } }) b.Run(tc.name+"skip-regex", func(b *testing.B) { for i := 0; i < b.N; i++ { _, _ = flattenKeySkipRegex(tc.inputParts, tc.inputLabels) } }) b.Run(tc.name+"replacer", func(b *testing.B) { for i := 0; i < b.N; i++ { _, _ = flattenKeyReplacer(tc.inputParts, tc.inputLabels) } }) } } var forbiddenChars = regexp.MustCompile("[ .=\\-/]") func flattenKey(parts []string, labels []metrics.Label) (string, string) { key := strings.Join(parts, "_") key = forbiddenChars.ReplaceAllString(key, "_") hash := key for _, label := range labels { hash += fmt.Sprintf(";%s=%s", label.Name, label.Value) } return key, hash } func flattenKeySkipRegex(parts []string, labels []metrics.Label) (string, string) { key := strings.Join(parts, "_") if strings.ContainsAny(key, " .=-/") { key = forbiddenChars.ReplaceAllString(key, "_") } hash := key for _, label := range labels { hash += ";" + label.Name + "=" + label.Value } return key, hash } var forbiddenCharsReplacer = strings.NewReplacer(" ", "_", ".", "_", "=", "_", "-", "_", "/", "_") func flattenKeyReplacer(parts []string, labels []metrics.Label) (string, string) { key := strings.Join(parts, "_") key = forbiddenCharsReplacer.Replace(key) hash := key for _, label := range labels { hash += ";" + label.Name + "=" + label.Value } return key, hash } ```Benchmark Results: