Cannot find grpc_node_plugin on Ubuntu

[xnerhu]

Hello!

I have an issue where nodejs grpc rules don't work on Ubuntu on Github actions where on my WSL Ubuntu 20 they do work. It seems that grpc_node_plugin can't be found.

ERROR: /home/runner/work/project/project/backend/questions-proto/BUILD.bazel:14:21: Compiling protoc outputs for grpc_node_plugin plugin on target //backend/questions-proto:components_proto_pb failed: (Exit 1): bash failed: error executing command 
  (cd /home/runner/.cache/bazel/_bazel_runner/8ebf206443f5766e5fda802ba6a70303/sandbox/linux-sandbox/380/execroot/init && \
  exec env - \
  /bin/bash -c 'mkdir -p '\''bazel-out/k8-fastbuild/bin/backend/questions-proto/_rpg_premerge_components_proto_pb'\'' && bazel-out/host/bin/external/com_google_protobuf/protoc $@' '' '--descriptor_set_in=bazel-out/k8-fastbuild/bin/external/com_google_protobuf/empty_proto-descriptor-set.proto.bin:bazel-out/k8-fastbuild/bin/backend/questions-proto/proto-descriptor-set.proto.bin' '--plugin=protoc-gen-grpc_node_plugin=bazel-out/k8-opt-exec-2B5CBBC6/bin/external/npm/grpc-tools/bin/grpc_tools_node_protoc_plugin.sh' '--grpc_node_plugin_out=grpc_js:bazel-out/k8-fastbuild/bin/backend/questions-proto/_rpg_premerge_components_proto_pb/_plugin_fixed_grpc_node_plugin' question.proto)
Execution platform: @local_config_platform//:host

Use --sandbox_debug to see verbose messages from the sandbox
Error: spawn /home/runner/.cache/bazel/_bazel_runner/8ebf206443f5766e5fda802ba6a70303/sandbox/linux-sandbox/380/execroot/init/bazel-out/k8-opt-exec-2B5CBBC6/bin/external/npm/grpc-tools/bin/grpc_tools_node_protoc_plugin.sh.runfiles/npm/node_modules/grpc-tools/bin/grpc_node_plugin ENOENT
    at Process.ChildProcess._handle.onexit (internal/child_process.js:269:19)
    at onErrorNT (internal/child_process.js:467:16)
    at processTicksAndRejections (internal/process/task_queues.js:82:21)
--grpc_node_plugin_out: protoc-gen-grpc_node_plugin: Plugin failed with status code 1.

load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@rules_proto_grpc//js:defs.bzl", "js_grpc_node_library")

package(default_visibility = ["//visibility:public"])

proto_library(
    name = "proto",
    srcs = glob(["*.proto"]),
    strip_import_prefix = "/backend/questions-proto",
    deps = ["@com_google_protobuf//:empty_proto"],
)

js_grpc_node_library(
    name = "components_proto",
    prefix_path = "../",
    protos = [":proto"]
)

js_library(
    name = "questions-proto",
    package_name = "@backend/questions-proto",
    srcs = [
        "package.json",
    ],
    visibility = ["//visibility:public"],
    deps = [
        ":components_proto",
    ],
)

    http_archive(
        name = "com_google_protobuf",
        sha256 = "c6003e1d2e7fefa78a3039f19f383b4f3a61e81be8c19356f85b6461998ad3db",
        strip_prefix = "protobuf-3.17.3",
        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.17.3.tar.gz"],
    )

    http_archive(
        name = "rules_proto",
        sha256 = "bc12122a5ae4b517fa423ea03a8d82ea6352d5127ea48cb54bc324e8ab78493c",
        strip_prefix = "rules_proto-af6481970a34554c6942d993e194a9aed7987780",
        urls = [
            "https://github.com/bazelbuild/rules_proto/archive/af6481970a34554c6942d993e194a9aed7987780.tar.gz",
        ],
    )

    http_archive(
        name = "rules_proto_grpc",
        sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419",
        strip_prefix = "rules_proto_grpc-3.1.1",
        urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"],
    )

[aaliddell]

I think you may be missing the other WORKSPACE rules for JS: https://rules-proto-grpc.aliddell.com/en/latest/lang/js.html#id20

If you already have a “npm” rule, you need to add the required packages to the package.json at the top of that page.

[xnerhu]

I already have these rules in my WORKSPACE and also needed dependencies.

# Bazel workspace created by @bazel/create 4.0.0

# Declares that this directory is the root of a Bazel workspace.
# See https://docs.bazel.build/versions/main/build-ref.html#workspace
workspace(
    # How this workspace would be referenced with absolute labels from another workspace
    name = "init",
    # Map the @npm bazel workspace to the node_modules directory.
    # This lets Bazel use the same node_modules as other local tooling.
    managed_directories = {"@npm": ["node_modules"]},
)

load("//:tools/bazel_deps.bzl", "fetch_dependencies")

fetch_dependencies()

load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")

bazel_skylib_workspace()

load("@rules_proto_grpc//js:repositories.bzl", rules_proto_grpc_js_repos = "js_repos")

rules_proto_grpc_js_repos()

load("@build_bazel_rules_nodejs//:index.bzl", "yarn_install")

yarn_install(
    name = "npm",
    package_json = "//:package.json",
    yarn_lock = "//:yarn.lock",
    # quiet = False,
)

load(
    "@io_bazel_rules_docker//repositories:repositories.bzl",
    container_repositories = "repositories",
)

container_repositories()

load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")

container_deps()

load("@io_bazel_rules_docker//nodejs:image.bzl", nodejs_image_repos = "repositories")

nodejs_image_repos()

load("@npm//@bazel/labs:package.bzl", "npm_bazel_labs_dependencies")

npm_bazel_labs_dependencies()

load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")

rules_proto_dependencies()

rules_proto_toolchains()

load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

protobuf_deps()

# load("@rules_typescript_proto//:index.bzl", "rules_typescript_proto_dependencies")

# rules_typescript_proto_dependencies()

load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")

rules_proto_grpc_toolchains()

rules_proto_grpc_repos()

  "dependencies": {
    "@grpc/grpc-js": "^1.3.7",
    "@nestjs/common": "^8.0.6",
    "@nestjs/core": "^8.0.6",
    "@nestjs/platform-fastify": "^8.0.6",
    "express": "^4.17.1",
    "fastify": "^3.20.2",
    "google-protobuf": "^3.18.0",
    "grpc-tools": "^1.11.2",
    "grpc-web": "^1.2.1",
    "nodemon": "^2.0.12",
    "reflect-metadata": "^0.1.13",
    "rxjs": "^7.3.0",
    "socket.io": "^4.2.0",
    "socket.io-client": "^4.2.0",
    "ts-protoc-gen": "^0.15.0",
    "winston": "^3.3.3",
    "ws": "^8.2.1"
  },

[aaliddell]

Gotcha, I’ll see if I can recreate the issue with that

[xnerhu]

On normal Linux it does work, but it doesn't on Github actions.

name: CI/CD

on:
  pull_request:
  push:

jobs:
  build:
    runs-on: ubuntu-latest

      - name: Check out Git repository
        uses: actions/checkout@v2

      - name: Install Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16

      - name: Install dependencies
        run: yarn install --frozen-lockfile
        env:
          ADBLOCK: true

      - name: Build
        run: npx bazel build //backend/questions-proto:components_proto

[xnerhu]

Also here is tools/bazel_deps.bzl

# This provides the basic tools for running and packaging nodejs programs in Bazel
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

def fetch_dependencies():
    http_archive(
        name = "bazel_skylib",
        urls = [
            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
        ],
        sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
    )

    http_archive(
        name = "build_bazel_rules_nodejs",
        sha256 = "b32a4713b45095e9e1921a7fcb1adf584bc05959f3336e7351bcf77f015a2d7c",
        urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/4.1.0/rules_nodejs-4.1.0.tar.gz"],
    )

    http_archive(
        name = "io_bazel_rules_docker",
        sha256 = "1f4e59843b61981a96835dc4ac377ad4da9f8c334ebe5e0bb3f58f80c09735f4",
        strip_prefix = "rules_docker-0.19.0",
        urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.19.0/rules_docker-v0.19.0.tar.gz"],
    )

    http_archive(
        name = "com_google_protobuf",
        sha256 = "c6003e1d2e7fefa78a3039f19f383b4f3a61e81be8c19356f85b6461998ad3db",
        strip_prefix = "protobuf-3.17.3",
        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.17.3.tar.gz"],
    )

    http_archive(
        name = "rules_proto",
        sha256 = "bc12122a5ae4b517fa423ea03a8d82ea6352d5127ea48cb54bc324e8ab78493c",
        strip_prefix = "rules_proto-af6481970a34554c6942d993e194a9aed7987780",
        urls = [
            "https://github.com/bazelbuild/rules_proto/archive/af6481970a34554c6942d993e194a9aed7987780.tar.gz",
        ],
    )

    http_archive(
        name = "rules_proto_grpc",
        sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419",
        strip_prefix = "rules_proto_grpc-3.1.1",
        urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"],
    )

[xnerhu]

I have made a reproduction repo and it seems it has something to do with Bazel cache. On completely new repo it does work. I don't know why, because I have changed cache versions in my official repo and everything was built again. https://github.com/xnerhu/grpc_node_plugin-repro

[aaliddell]

I've seen it before where the node_modules directory can do odd things with the cache like this. i.e Bazel thinks it exists but it's cached just an empty directory, so trying to open files in it fails. If you test removing managed_directories here, does it still fail or can you no longer reproduce the failure?

[xnerhu]

It still fails.

[xnerhu]

I overlooked that I override node_modules folder in a different workflow step. The issue is totally on me. Thank you.

[aaliddell]

No problem, it's useful to have the issue documented in case someone has the same problem in the future 👍