Logging of gke_container metrics returns error code 13 without any explanation

[Glennvd]

Environment details

• OS: debian jessie (docker node:8-slim)
• Node.js version: 8.9.1
• npm version: 5.5.1
• yarn version: 1.3.2
• @google-cloud/monitoring version: 0.4.1

Steps to reproduce

I've started from the go sample at https://github.com/GoogleCloudPlatform/k8s-stackdriver/tree/master/custom-metrics-stackdriver-adapter and combined that with the nodejs sample from the readme.md. This leads me to the following script.

// Imports the Google Cloud client library
const monitoring = require('@google-cloud/monitoring');
const request = require('request');

let projectId = '<my-alfanumeric-project-id>';
let container_name = '<my-container-name>';
let namespace = '<my-kubernetes-namespace-name>';
let pod_id = process.env.POD_ID;
let zone;
let instance_id;
let kube_env;
let cluster_name;
let client = new monitoring.MetricServiceClient();

request({'url': 'http://metadata/computeMetadata/v1/instance/id', headers: {'Metadata-Flavor': 'Google'}}, (error, response, body) => {
if (!error && response.statusCode == 200) {
    instance_id = body;
}
});
request({'url': 'http://metadata/computeMetadata/v1/instance/attributes/kube-env', headers: {'Metadata-Flavor': 'Google'}}, (error, response, body) => {
if (!error && response.statusCode == 200) {
    kube_env = body;
    let extracted_zone = kube_env.match(/^ZONE:\s+([a-z0-9-]+)$/im)[1];
    let extracted_cluster = kube_env.match(/^CLUSTER_NAME:\s+([a-z0-9-]+)$/im)[1];
    if(extracted_zone){
        zone = extracted_zone;
        cluster_name = extracted_cluster;
    } else {
        throw "Unable to extract zone from kube_env"
    }
}
});

setTimeout(() => {
  if(client){
    let dataPoint = {
      interval: {
        endTime: {
          seconds: parseInt(Date.now() / 1000, 10),
        },
      },
      value: {
        doubleValue: 0.1,
      },
    };

    const timeSeriesData = {
      metric: {
        type: metric,
      },
      resource: {
        type: 'gke_container',
        labels: {
          project_id: projectId,
          zone: zone,
          cluster_name: cluster_name,
          container_name: container_name,
          pod_id: pod_id,
          // not needed according to https://github.com/kawych/k8s-stackdriver/blob/docs/custom-metrics-stackdriver-adapter/testing/sd_dummy_exporter.go#L84
          instance_id: instance_id,
          namespace_id: namespace,
        },
      },
      points: [dataPoint],
    };

    const request = {
      name: client.projectPath(projectId),
      timeSeries: [timeSeriesData],
    };

    console.log(JSON.stringify(request));

    // Writes time series data
    client
      .createTimeSeries(request)
      .then(results => {
        console.debug(`Done writing time series data.`, results[0]);
      })
      .catch(err => {
        console.error('ERROR:', err);
      });
  }
}, 30000)

If i run this inside my GKE cluster, I receive the following error:

ERROR: { Error: One or more TimeSeries could not be written:
An internal error occurred: timeSeries[0]
    at /app/node_modules/grpc/src/client.js:554:15
  code: 13,
  metadata: 
   Metadata {
     _internal_repr: 
      { 'google.monitoring.v3.createtimeserieserror-bin': [Array],
        'grpc-status-details-bin': [Array] } } }

If I take the printed values, fill them in in the go example I mentioned earlier. It executes correctly. So this looks like a nodejs client bug? The only other differences are minor, like the language specific way to get the current timestamp etc.

[stephenplusplus]

@alexander-fenster do you know who we could ping about this?

[alexander-fenster]

@Glennvd in your code, the metric variable seems either omitted or undefined:

      metric: {
        type: metric, // <-- here
      },

According to the doc, the metric type should be a string. Are you sure your code sets that correctly? It could've explained the error within timeSeries[0].

[Glennvd]

These were filled in correct, this is a console.log of my payload (with some information redacted)

{
  name:  "projects/XXX"   
  timeSeries: [
   0: {
    metric: {
     type:  "custom.googleapis.com/XXX/YYY"      
    }
    points: [
     0: {
      interval: {
       endTime: {
        seconds:  1515054690         
       }
      }
      value: {
       doubleValue:  0.001        
      }
     }
    ]
    resource: {
     labels: {
      cluster_name:  "XXX"       
      container_name:  "XXX"       
      instance_id:  "163211280624099561"       
      namespace_id:  "<k8s namespace name>"       
      pod_id:  "7ed88526-eca8-11e7-b0f9-42010a8400af"       
      project_id:  "XXX"       
      zone:  "europe-west1-d"       
     }
     type:  "gke_container"      
    }
   }
  ]
 }

The only value I'm doubting is that my namespace_id could be wrong, but according to the docs that isn't even mandatory (and I get the same error when I don't include it)

[alexander-fenster]

Thanks for the payload. The error is coming from the server, and "An internal error occurred" that is returned is not really helpful. Investigating, will update the issue when I get more information.

[stephenplusplus]

Hey @alexander-fenster, any info on this? Should we connect @Glennvd with a better support channel, e.g. message board, support email, etc.?

[Glennvd]

Any update on this?

[Glennvd]

Issue has been resolved through stackdriver support. Was a bug with certain project configurations.