Add e2e tests for multitenancy on Kubernetes

[frzifus]

Should look like:

Install

```yaml --- apiVersion: v1 kind: Secret metadata: name: minio-test stringData: endpoint: http://minio.minio.svc:9000 bucket: tempo access_key_id: tempo access_key_secret: supersecret type: Opaque --- apiVersion: tempo.grafana.com/v1alpha1 kind: Microservices metadata: name: foo spec: storage: secret: minio-test storageSize: 200M tenants: mode: static authentication: - tenantName: test-oidc tenantId: test-oidc oidc: secret: name: test-oidc issuerURL: https://dex.klimlive.de/dex authorization: roleBindings: - name: test-oidc roles: - read-write subjects: - kind: user name: user roles: - name: read-write permissions: - read - write resources: - logs - metrics - traces tenants: - test-oidc ```

Assert

```yaml --- apiVersion: v1 kind: ConfigMap metadata: name: tempo-foo-gateway labels: app.kubernetes.io/component: gateway app.kubernetes.io/created-by: tempo-controller app.kubernetes.io/instance: foo app.kubernetes.io/managed-by: tempo-controller app.kubernetes.io/name: tempo annotations: {} data: rbac.yaml: "roleBindings:\n- name: test-oidc\n roles:\n - read-write\n\n subjects:\n - kind: user\n name: user\n\nroles:\n- name: read-write\n permissions:\n - read\n - write\n\n resources:\n - logs\n - traces\n - metrics\n\n tenants:\n - test-oidc" tempo-gateway.rego: "package tempo\n\nimport input\nimport data.roles\nimport data.roleBindings\n\ndefault allow = false\n\nallow {\n some roleNames\n roleNames = roleBindings[matched_role_binding[_]].roles\n roles[i].name == roleNames[_]\n roles[i].resources[_] = input.resource\n roles[i].permissions[_] = input.permission\n roles[i].tenants[_] = input.tenant\n}\n\nmatched_role_binding[i] {\n roleBindings[i].subjects[_] == {\"name\": input.subject, \"kind\": \"user\"}\n}\n\nmatched_role_binding[i] {\n roleBindings[i].subjects[_] == {\"name\": input.groups[_], \"kind\": \"group\"}\n}\n" --- apiVersion: v1 kind: Secret metadata: name: tempo-foo-gateway labels: app.kubernetes.io/component: gateway app.kubernetes.io/created-by: tempo-controller app.kubernetes.io/instance: foo app.kubernetes.io/managed-by: tempo-controller app.kubernetes.io/name: tempo data: tenants.yaml: dGVuYW50czoKLSBuYW1lOiB0ZXN0LW9pZGMKICBpZDogdGVzdC1vaWRjCiAgb2lkYzoKICAgIGlzc3VlclVSTDogaHR0cHM6Ly9kZXgua2xpbWxpdmUuZGUvZGV4CiAgICAKICBvcGE6CiAgICBxdWVyeTogZGF0YS50ZW1wby5hbGxvdwogICAgcGF0aHM6CiAgICAtIC9ldGMvdGVtcG8tZ2F0ZXdheS9yYmFjLnlhbWwKICAgIC0gL2V0Yy90ZW1wby1nYXRld2F5L3RlbXBvLWdhdGV3YXkucmVnbw== --- apiVersion: apps/v1 kind: Deployment metadata: name: tempo-foo-gateway labels: app.kubernetes.io/component: gateway app.kubernetes.io/created-by: tempo-controller app.kubernetes.io/instance: foo app.kubernetes.io/managed-by: tempo-controller app.kubernetes.io/name: tempo spec: selector: matchlabels: app.kubernetes.io/component: gateway app.kubernetes.io/created-by: tempo-controller app.kubernetes.io/instance: foo app.kubernetes.io/managed-by: tempo-controller app.kubernetes.io/name: tempo template: metadata: labels: app.kubernetes.io/component: gateway app.kubernetes.io/created-by: tempo-controller app.kubernetes.io/instance: foo app.kubernetes.io/managed-by: tempo-controller app.kubernetes.io/name: tempo spec: annotations: tempo.grafana.com/config.hash: e39fb269cbe21cc0310da630a79fe34ffb35d34022d8b5c264a697133eb312b7 spec: volumes: - name: rbac volumesource: configmap: localobjectreference: name: tempo-foo-gateway - name: tenants volumesource: configmap: localobjectreference: name: tempo-foo-gateway - name: tempo-gateway volumesource: secret: secretname: tempo-foo-gateway containers: - name: tempo image: "" command: [] args: - --web.listen=0.0.0.0:8080 - --web.internal.listen=0.0.0.0:8081 - --traces.write.endpoint=tempo-foo-distributor:4317 - --traces.read.endpoint=tempo-foo-query:16686 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/observatorium/rbac.yaml - --tenants.config=/etc/observatorium/tenants.yaml - --log.level=error ports: - name: grpc-public hostport: 0 containerport: 8090 protocol: TCP hostip: "" - name: internal hostport: 0 containerport: 8081 protocol: TCP hostip: "" - name: public hostport: 0 containerport: 8080 protocol: TCP hostip: "" resources: limits: {} requests: {} volumemounts: - name: rbac readonly: true mountpath: /etc/tempo-gateway/rbac.yaml subpath: rbac.yaml - name: tenants readonly: true mountpath: /etc/tempo-gateway/tenants.yaml subpath: tenants.yaml - name: tempo-gateway readonly: true mountpath: /etc/tempo-gateway/tempo-gateway.rego subpath: lokistack-gateway.rego livenessprobe: probehandler: exec: null httpget: path: /live port: type: 0 intval: 8081 strval: "" host: "" scheme: HTTP timeoutseconds: 2 periodseconds: 30 failurethreshold: 10 readinessprobe: probehandler: httpget: path: /ready port: type: 0 intval: 8081 strval: "" host: "" scheme: HTTP initialdelayseconds: 0 timeoutseconds: 1 periodseconds: 5 successthreshold: 0 failurethreshold: 12 securitycontext: capabilities: add: [] drop: - ALL privileged: null selinuxoptions: null windowsoptions: null runasuser: null runasgroup: null runasnonroot: null readonlyrootfilesystem: true allowprivilegeescalation: false procmount: null seccompprofile: null ```

[pavolloffay]

Data reporting/querying should be added to the test https://github.com/grafana/tempo-operator/tree/main/tests/e2e/gateway

[andreasgerstmayr]

A smoketest for monolithic + gateway in static mode on OpenShift is in #816. The OpenTelemetry collector refuses to do authentication over HTTP, therefore this test is currently in the OpenShift testsuite. Once the gateway supports TLS on non-OpenShift, we can move it.