Failed to load gRPC binary module because it was not installed for the current system

[mfolivas]

My package.json is the following:

{
  "name": "aws-serverless-nodejs-opentelemetry",
  "version": "1.0.0",
  "description": "Reviewing and using open telemetry by using the AWS distro for [OpenTelemetry JS](https://github.com/aws-observability/aws-otel-js)",
  "main": "handler.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/mfolivas/aws-serverless-nodejs-opentelemetry.git"
  },
  "author": "",
  "license": "ISC",
  "bugs": {
    "url": "https://github.com/mfolivas/aws-serverless-nodejs-opentelemetry/issues"
  },
  "homepage": "https://github.com/mfolivas/aws-serverless-nodejs-opentelemetry#readme",
  "dependencies": {
    "@aws/otel-aws-xray-id-generator": "^0.13.1",
    "@aws/otel-aws-xray-propagator": "^0.13.0",
    "@opentelemetry/core": "^0.13.0",
    "@opentelemetry/exporter-collector-grpc": "^0.13.0",
    "@opentelemetry/metrics": "^0.13.0",
    "@opentelemetry/node": "^0.13.0",
    "opentelemetry-plugin-aws-sdk": "^0.2.1"
  }
}

My serverless.yml is the following:

service: aws-serverless-nodejs-opentelemetry

frameworkVersion: '2'

provider:
  name: aws
  runtime: nodejs12.x
  lambdaHashingVersion: 20201221

functions:
  hello:
    handler: handler.hello

Handler is trying to use the similar sample:

'use strict';
const tracer = require('./tracer')('aws-otel-integ-test');
const AWS = require('aws-sdk');
const meter = require('./metric-emitter');

module.exports.hello = async (event) => {
  console.log('First request', handleRequest('/'))
  console.log('Second request', handleRequest('/aws-sdk-call'))
  console.log('Third request', handleRequest('/outgoing-http-call'))
  return {
    statusCode: 200,
    body: JSON.stringify(
      {
        message: 'Go Serverless v1.0! Your function executed successfully!',
        input: event,
      },
      null,
      2
    ),
  };

  // Use this code if you don't use the http event with the LAMBDA-PROXY integration
  // return { message: 'Go Serverless v1.0! Your function executed successfully!', event };
};

/** A function which handles requests and send response. */
function handleRequest(url) {
  const requestStartTime = new Date().getMilliseconds();
  // start recording a time for request
  try {
    if (url === '/') {
      return 'healthcheck';
    }

    if (url === '/aws-sdk-call') {
      const s3 = new AWS.S3();
      s3.listBuckets(() => { });
      const traceID = returnTraceIdJson();
      return traceID;
    }

    if (url === '/outgoing-http-call') {
      const traceID = returnTraceIdJson();
      meter.emitsPayloadMetric(res._contentLength + mimicPayLoadSize(), '/outgoing-http-call', res.statusCode);
      meter.emitReturnTimeMetric(new Date().getMilliseconds() - requestStartTime, '/outgoing-http-call', res.statusCode);
    }
  } catch (err) {
    console.error(err);
  }
}

//returns a traceId in X-Ray JSON format
function returnTraceIdJson() {
  const traceId = tracer.getCurrentSpan().context().traceId;
  const xrayTraceId = "1-" + traceId.substring(0, 8) + "-" + traceId.substring(8);
  const traceIdJson = JSON.stringify({ "traceId": xrayTraceId });
  return traceIdJson;
}

//returns random payload size
function mimicPayLoadSize() {
  return Math.random() * 1000;
}

However, I am getting the following error:

ERROR   Uncaught Exception  {"errorType":"Error","errorMessage":"Failed to load gRPC binary module because it was not installed for the current system\nExpected directory: node-v64-linux-x64-glibc\nFound: [node-v79-darwin-x64-unknown]\nThis problem can often be fixed by running \"npm rebuild\" on the current system\nOriginal error: Cannot find module '/var/task/node_modules/grpc/src/node/extension_binary/node-v64-linux-x64-glibc/grpc_node.node'","code":"MODULE_NOT_FOUND","stack":["Error: Failed to load gRPC binary module because it was not installed for the current system","Expected directory: node-v64-linux-x64-glibc","Found: [node-v79-darwin-x64-unknown]","This problem can often be fixed by running \"npm rebuild\" on the current system","Original error: Cannot find module '/var/task/node_modules/grpc/src/node/extension_binary/node-v64-linux-x64-glibc/grpc_node.node'","    at Object.<anonymous> (/var/task/node_modules/grpc/src/grpc_extension.js:53:17)","    at Module._compile (internal/modules/cjs/loader.js:778:30)","    at Object.Module._extensions..js (internal/modules/cjs/loader.js:789:10)","    at Module.load (internal/modules/cjs/loader.js:653:32)","    at tryModuleLoad (internal/modules/cjs/loader.js:593:12)","    at Function.Module._load (internal/modules/cjs/loader.js:585:3)","    at Module.require (internal/modules/cjs/loader.js:692:17)","    at Module.Hook._require.Module.require (/var/task/node_modules/require-in-the-middle/index.js:80:39)","    at require (internal/modules/cjs/helpers.js:25:18)","    at Object.<anonymous> (/var/task/node_modules/grpc/src/client_interceptors.js:144:12)"]}

The tracer is:

/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS'" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 */

'use strict'

const { LogLevel } = require("@opentelemetry/core");
const { SimpleSpanProcessor, ConsoleSpanExporter } = require("@opentelemetry/tracing");
const { NodeTracerProvider } = require('@opentelemetry/node');
const { CollectorTraceExporter } = require('@opentelemetry/exporter-collector-grpc');

const { AWSXRayPropagator } = require('@aws/otel-aws-xray-propagator');
const { AwsXRayIdGenerator } = require('@aws/otel-aws-xray-id-generator');

const { propagation, trace } = require("@opentelemetry/api");

module.exports = (serviceName) => {
  // set global propagator
  propagation.setGlobalPropagator(new AWSXRayPropagator());

  // create a provider for activating and tracking with AWS IdGenerator
  const tracerConfig = {
    logLevel: LogLevel.ERROR,
    idGenerator: new AwsXRayIdGenerator(),
    plugins: {
      https: {
        enabled: true,
        // You may use a package name or absolute path to the file.
        path: '@opentelemetry/plugin-https',
        // https plugin options
      },
      "aws-sdk": {
        enabled: true,
        // You may use a package name or absolute path to the file.
        path: "opentelemetry-plugin-aws-sdk",
      },
    },
  };
  const tracerProvider = new NodeTracerProvider(tracerConfig);

  // add OTLP exporter
  const otlpExporter = new CollectorTraceExporter({
    serviceName: serviceName,
    url: (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT : "localhost:55680"
  });
  tracerProvider.addSpanProcessor(new SimpleSpanProcessor(otlpExporter));
  tracerProvider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter()));

  // Register the tracer
  tracerProvider.register();

  // Return an tracer instance
  return trace.getTracer("awsxray-tests");
}

[willarmiros]

Hi @mfolivas,

Thanks for raising this issue. Have you tried running npm rebuild in your environment as suggested by the error message? That should resolve the issue as a workaround for now, and we are also investigating this problem to remove that extra step altogether. I will post any updates here.

[mfolivas]

Thanks @willarmiros , I tried and it didn't work. I also noticed that the OTEL collector that I was using was pointing to the wrong port:

git clone https://github.com/aws-observability/aws-otel-collector.git ; \
    cd aws-otel-collector; \
    docker run --rm -p 4317:4317 -p 55679:55679 -p 8889:8888 \
      -e AWS_REGION=us-west-2 \
      -v "${PWD}/examples/config-test.yaml":/otel-local-config.yaml \
      --name awscollector public.ecr.aws/aws-observability/aws-otel-collector:latest \
      --config otel-local-config.yaml;

[willarmiros]

@mfolivas thanks for raising this, I've opened a PR to update the collector's docs with the right port number. I was able to reproduce the gRPC binary but npm rb did resolve it for me. Could you try starting from scratch using these instructions? If that works, you can adjust the provided sample app to your use case and continue from there. Apologies again for this inconvenience.

[mfolivas]

I'm getting the following error once I did the following: npm install && npm rb:

{"stack":"Error: 2 UNKNOWN: NoCredentialProviders: no valid providers in chain. Deprecated.\n\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors\n    at Object.exports.createStatusError (/Users/marcelo/development/open-source/aws-otel-js/sample-apps/node_modules/grpc/src/common.js:91:15)\n    at Object.onReceiveStatus (/Users/marcelo/development/open-source/aws-otel-js/sample-apps/node_modules/grpc/src/client_interceptors.js:1209:28)\n    at InterceptingListener._callNext (/Users/marcelo/development/open-source/aws-otel-js/sample-apps/node_modules/grpc/src/client_interceptors.js:568:42)\n    at InterceptingListener.onReceiveStatus (/Users/marcelo/development/open-source/aws-otel-js/sample-apps/node_modules/grpc/src/client_interceptors.js:618:8)\n    at callback (/Users/marcelo/development/open-source/aws-otel-js/sample-apps/node_modules/grpc/src/client_interceptors.js:847:24)","message":"2 UNKNOWN: NoCredentialProviders: no valid providers in chain. Deprecated.\n\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors","code":"2","metadata":"[object Object]","details":"NoCredentialProviders: no valid providers in chain. Deprecated.\n\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors","name":"Error"}

I'm running the following aws-otel-collector and getting the following error:

docker run --rm -p 4317:4317 -p 55680:55680 -p 8889:8888       -e AWS_REGION=us-west-2       -v "${PWD}/examples/config-test.yaml":/otel-local-config.yaml       --name awscollector public.ecr.aws/aws-observability/aws-otel-collector:latest       --config otel-local-config.yaml;

error:

2021-02-23T01:42:57.250Z    ERROR   exporterhelper/queued_retry.go:204  Exporting failed. Try enabling retry_on_failure config option.  {"component_kind": "exporter", "component_type": "awsxray", "component_name": "awsxray", "error": "NoCredentialProviders: no valid providers in chain. Deprecated.\n\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors"}
go.opentelemetry.io/collector/exporter/exporterhelper.(*retrySender).send
    go.opentelemetry.io/collector@v0.19.0/exporter/exporterhelper/queued_retry.go:204
go.opentelemetry.io/collector/exporter/exporterhelper.(*tracesExporterWithObservability).send
    go.opentelemetry.io/collector@v0.19.0/exporter/exporterhelper/tracehelper.go:114
go.opentelemetry.io/collector/exporter/exporterhelper.(*queuedRetrySender).send
    go.opentelemetry.io/collector@v0.19.0/exporter/exporterhelper/queued_retry.go:140
go.opentelemetry.io/collector/exporter/exporterhelper.(*traceExporter).ConsumeTraces
    go.opentelemetry.io/collector@v0.19.0/exporter/exporterhelper/tracehelper.go:68
go.opentelemetry.io/collector/receiver/otlpreceiver/trace.(*Receiver).sendToNextConsumer
    go.opentelemetry.io/collector@v0.19.0/receiver/otlpreceiver/trace/otlp.go:103
go.opentelemetry.io/collector/receiver/otlpreceiver/trace.(*Receiver).Export
    go.opentelemetry.io/collector@v0.19.0/receiver/otlpreceiver/trace/otlp.go:84
go.opentelemetry.io/collector/internal/data/protogen/collector/trace/v1._TraceService_Export_Handler
    go.opentelemetry.io/collector@v0.19.0/internal/data/protogen/collector/trace/v1/trace_service.pb.go:210
google.golang.org/grpc.(*Server).processUnaryRPC
    google.golang.org/grpc@v1.35.0/server.go:1217
google.golang.org/grpc.(*Server).handleStream
    google.golang.org/grpc@v1.35.0/server.go:1540
google.golang.org/grpc.(*Server).serveStreams.func1.2
    google.golang.org/grpc@v1.35.0/server.go:878

[willarmiros]

@mfolivas I see, this is a problem with the Collector failing to find AWS credentials. You can create a new User in your AWS account's IAM with the following policy attached: https://aws-otel.github.io/docs/setup/permissions

Then, you should be able to use the access key and secret key of that user to get the collector to send data to AWS. You can pass them in via AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables as described in the docs. If you're having further issues with the collector please open them in the collector repo, thanks!

[github-actions[bot]]

This issue is stale because it has been open 90 days with no activity. If you want to keep this issue open, please just leave a comment below and auto-close will be canceled

[github-actions[bot]]

This issue was closed because it has been marked as stale for 30 days with no activity.