search

opensearch-project / opentelemetry-demo

This repository contains the OpenSearch adaptation for the OpenTelemetry Astronomy Shop, a microservice-based distributed system intended to illustrate the implementation of OpenTelemetry in a near real-world environment.
https://opentelemetry.io/docs/demo/
Apache License 2.0
15 stars 16 forks source link

Update otel metrics index mappings #147

Open ps48 opened 3 months ago

ps48 commented 3 months ago

Bug Report

Which version of the demo you are using? (please provide either a specific commit hash

Symptom

The bug is present in the OTEL metrics schema present in the data-prepper pipeline. This schema is different from the one released in the opensearch catalog metrics schema

current mapping in the OS after running the demo is as below:

{
  "ss4o_metrics-otel-errors-2024.06.14": {
    "mappings": {
      "_meta": {
        "component": "metrics",
        "type": "metrics",
        "version": "1.0.0",
        "correlations": [
          {
            "field": "spanId",
            "foreign-field": "spanId",
            "foreign-schema": "traces"
          },
          {
            "field": "traceId",
            "foreign-field": "traceId",
            "foreign-schema": "traces"
          }
        ],
        "catalog": "observability"
      },
      "dynamic_templates": [
        {
          "exemplar_attributes_map": {
            "path_match": "exemplar.attributes.*",
            "mapping": {
              "type": "keyword"
            }
          }
        },
        {
          "instrumentation_scope_attributes_map": {
            "path_match": "instrumentationScope.attributes.*",
            "mapping": {
              "type": "keyword"
            }
          }
        }
      ],
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "aggregationTemporality": {
          "type": "keyword",
          "ignore_above": 128
        },
        "attributes": {
          "properties": {
            "data_stream": {
              "properties": {
                "dataset": {
                  "type": "keyword",
                  "ignore_above": 128
                },
                "namespace": {
                  "type": "keyword",
                  "ignore_above": 128
                },
                "type": {
                  "type": "keyword",
                  "ignore_above": 56
                }
              }
            },
            "serviceName": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "traceId": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "bucketCount": {
          "type": "long"
        },
        "bucketCountsList": {
          "type": "long"
        },
        "buckets": {
          "type": "nested",
          "properties": {
            "count": {
              "type": "long"
            },
            "max": {
              "type": "float"
            },
            "min": {
              "type": "float"
            },
            "sum": {
              "type": "double"
            }
          }
        },
        "count": {
          "type": "long"
        },
        "description": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "exemplar": {
          "properties": {
            "spanId": {
              "type": "keyword",
              "ignore_above": 256
            },
            "time": {
              "type": "date"
            },
            "traceId": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "exemplars": {
          "properties": {
            "attributes": {
              "properties": {
                "attributes": {
                  "properties": {
                    "instrumentationScope": {
                      "properties": {
                        "name": {
                          "type": "text",
                          "fields": {
                            "keyword": {
                              "type": "keyword",
                              "ignore_above": 256
                            }
                          }
                        },
                        "version": {
                          "type": "text",
                          "fields": {
                            "keyword": {
                              "type": "keyword",
                              "ignore_above": 256
                            }
                          }
                        }
                      }
                    },
                    "resource": {
                      "properties": {
                        "attributes": {
                          "properties": {
                            "docker@cli@cobra@command_path": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            },
                            "service@name": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            },
                            "telemetry@sdk@language": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            },
                            "telemetry@sdk@name": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            },
                            "telemetry@sdk@version": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            }
                          }
                        }
                      }
                    },
                    "span": {
                      "properties": {
                        "attributes": {
                          "properties": {
                            "http@method": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            },
                            "http@status_code": {
                              "type": "long"
                            },
                            "http@url": {
                              "type": "text",
                              "fields": {
                                "keyword": {
                                  "type": "keyword",
                                  "ignore_above": 256
                                }
                              }
                            }
                          }
                        }
                      }
                    },
                    "status": {
                      "properties": {
                        "code": {
                          "type": "long"
                        }
                      }
                    }
                  }
                },
                "droppedAttributesCount": {
                  "type": "long"
                },
                "droppedEventsCount": {
                  "type": "long"
                },
                "droppedLinksCount": {
                  "type": "long"
                },
                "durationInNanos": {
                  "type": "long"
                },
                "endTime": {
                  "type": "date"
                },
                "kind": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "name": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "parentSpanId": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "serviceName": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "spanId": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "startTime": {
                  "type": "date"
                },
                "traceGroup": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "traceGroupFields": {
                  "properties": {
                    "durationInNanos": {
                      "type": "long"
                    },
                    "endTime": {
                      "type": "date"
                    },
                    "statusCode": {
                      "type": "long"
                    }
                  }
                },
                "traceId": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                },
                "traceState": {
                  "type": "text",
                  "fields": {
                    "keyword": {
                      "type": "keyword",
                      "ignore_above": 256
                    }
                  }
                }
              }
            },
            "spanId": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "time": {
              "type": "date"
            },
            "traceId": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "value": {
              "type": "float"
            }
          }
        },
        "explicitBoundsCount": {
          "type": "float"
        },
        "explicitBoundsList": {
          "type": "float"
        },
        "instrumentationScope": {
          "properties": {
            "droppedAttributesCount": {
              "type": "integer"
            },
            "name": {
              "type": "keyword",
              "ignore_above": 256
            },
            "schemaUrl": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "version": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "isMonotonic": {
          "type": "boolean"
        },
        "kind": {
          "type": "keyword",
          "ignore_above": 128
        },
        "max": {
          "type": "float"
        },
        "min": {
          "type": "float"
        },
        "monotonic": {
          "type": "boolean"
        },
        "name": {
          "type": "keyword",
          "ignore_above": 256
        },
        "negativeBuckets": {
          "type": "nested",
          "properties": {
            "count": {
              "type": "long"
            },
            "max": {
              "type": "float"
            },
            "min": {
              "type": "float"
            }
          }
        },
        "negativeOffset": {
          "type": "integer"
        },
        "observedTimestamp": {
          "type": "date_nanos"
        },
        "positiveBuckets": {
          "type": "nested",
          "properties": {
            "count": {
              "type": "long"
            },
            "max": {
              "type": "float"
            },
            "min": {
              "type": "float"
            }
          }
        },
        "positiveOffset": {
          "type": "integer"
        },
        "quantileValuesCount": {
          "type": "long"
        },
        "quantiles": {
          "properties": {
            "quantile": {
              "type": "double"
            },
            "value": {
              "type": "double"
            }
          }
        },
        "scale": {
          "type": "long"
        },
        "schemaUrl": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "startTime": {
          "type": "date"
        },
        "sum": {
          "type": "float"
        },
        "time": {
          "type": "date"
        },
        "unit": {
          "type": "keyword",
          "ignore_above": 128
        },
        "value": {
          "type": "float"
        },
        "value@double": {
          "type": "double"
        },
        "value@int": {
          "type": "integer"
        },
        "zeroCount": {
          "type": "long"
        }
      }
    }
  }
}

What is the expected behavior? The main attributes of the metrics like serviceName, traceId and spanId should be keyword and not text datatype with the separate keyword mapping.

What is the actual behavior? As mentioned above the current schema has serviceName, traceId and spanId with text mappings

Reproduce

Provide the minimum required steps to result in the issue you're observing.

Run the OTEL demo as is and see the mappings of the ss4o_metrics index.

Additional Context

Add any other context about the problem here.

dblock commented 3 months ago

Can this be closed? Did #148 or #162 fix this?

[Catch All Triage, attendees 1, 2, 3, 4, 5, 6, 7]