Secret parameters cannot be overridden using `.secret.local` when using the Functions Emulator

[fedenusy]

[REQUIRED] Environment info

firebase-tools: 11.23.1

Platform: ubuntu

[REQUIRED] Test case

1. Initialize project
2. Uncomment helloWorld function and make it runWith any secret
3. Add functions/.secret.local containing a value for the secret
4. Run functions emulator using --project with demo- prefix
5. Observe functions emulator erroring out with a 403 because it tries to fetch the secret from a hosted Firebase project. This prevents the helloWorld function from getting loaded.

[REQUIRED] Steps to reproduce

1. mkdir test && cd test
2. firebase init > functions > any existing project > typescript
3. Change functions/src/index.ts to what's shown below.
4. Add functions/.secret.local as shown below.
5. cd functions && npm run build && firebase emulators:start --only functions --project demo-project-id

functions/src/index.ts:

import * as functions from "firebase-functions";
import { defineSecret } from "firebase-functions/params";

defineSecret('SUPA_SECRET')

export const helloWorld = functions.runWith({ secrets: ['SUPA_SECRET'] }).https.onRequest((request, response) => {
  functions.logger.info("Hello logs!", {structuredData: true});
  response.send("Hello from Firebase!");
});

functions/.secret.local:

SUPA_SECRET=shhhh

NB: I also tried placing .secret.local at the repo root, and got the same result.

[REQUIRED] Expected behavior

Emulators should pick up .secret.local.

[REQUIRED] Actual behavior

[2023-02-14T22:46:11.959Z] > command requires scopes: ["email","openid","https://www.googleapis.com/auth/cloudplatformprojects.readonly","https://www.googleapis.com/auth/firebase","https://www.googleapis.com/auth/cloud-platform"]
[2023-02-14T22:46:11.959Z] > authorizing via signed-in user (federico.nusymowicz@farmpro.ag)
i  emulators: Starting emulators: functions {"metadata":{"emulator":{"name":"hub"},"message":"Starting emulators: functions"}}
i  emulators: Detected demo project ID "demo-project-id", emulated services will use a demo configuration and attempts to access non-emulated services for this project will fail. {"metadata":{"emulator":{"name":"hub"},"message":"Detected demo project ID \"demo-project-id\", emulated services will use a demo configuration and attempts to access non-emulated services for this project will fail."}}
[2023-02-14T22:46:11.964Z] assigned listening specs for emulators {"user":{"hub":[{"address":"127.0.0.1","family":"IPv4","port":4400}],"ui":[{"address":"127.0.0.1","family":"IPv4","port":4000}],"logging":[{"address":"127.0.0.1","family":"IPv4","port":4500}]},"metadata":{"message":"assigned listening specs for emulators"}}
[2023-02-14T22:46:11.967Z] [hub] writing locator at /tmp/hub-demo-project-id.json
[2023-02-14T22:46:11.973Z] late-assigned ports for functions and eventarc emulators {"user":{"hub":[{"address":"127.0.0.1","family":"IPv4","port":4400}],"ui":[{"address":"127.0.0.1","family":"IPv4","port":4000}],"logging":[{"address":"127.0.0.1","family":"IPv4","port":4500}],"functions":[{"address":"127.0.0.1","family":"IPv4","port":5001}],"eventarc":[{"address":"127.0.0.1","family":"IPv4","port":9299}]},"metadata":{"message":"late-assigned ports for functions and eventarc emulators"}}
[2023-02-14T22:46:11.975Z] defaultcredentials: writing to file /home/fedenusy/.config/firebase/federico_nusymowicz_farmpro.ag_application_default_credentials.json
[2023-02-14T22:46:11.976Z] Setting GAC to /home/fedenusy/.config/firebase/federico_nusymowicz_farmpro.ag_application_default_credentials.json {"metadata":{"emulator":{"name":"functions"},"message":"Setting GAC to /home/fedenusy/.config/firebase/federico_nusymowicz_farmpro.ag_application_default_credentials.json"}}
[2023-02-14T22:46:11.983Z] Ignoring unsupported arg: auto_download {"metadata":{"emulator":{"name":"ui"},"message":"Ignoring unsupported arg: auto_download"}}
[2023-02-14T22:46:11.984Z] Ignoring unsupported arg: port {"metadata":{"emulator":{"name":"ui"},"message":"Ignoring unsupported arg: port"}}
[2023-02-14T22:46:11.984Z] Starting Emulator UI with command {"binary":"node","args":["/home/fedenusy/.cache/firebase/emulators/ui-v1.11.4/server/server.js"],"optionalArgs":[],"joinArgs":false} {"metadata":{"emulator":{"name":"ui"},"message":"Starting Emulator UI with command {\"binary\":\"node\",\"args\":[\"/home/fedenusy/.cache/firebase/emulators/ui-v1.11.4/server/server.js\"],\"optionalArgs\":[],\"joinArgs\":false}"}}
i  ui: Emulator UI logging to ui-debug.log {"metadata":{"emulator":{"name":"ui"},"message":"Emulator UI logging to \u001b[1mui-debug.log\u001b[22m"}}
[2023-02-14T22:46:12.068Z] Web / API server started at 127.0.0.1:4000
 {"metadata":{"emulator":{"name":"ui"},"message":"Web / API server started at 127.0.0.1:4000\n"}}
i  functions: Watching "/home/fedenusy/dev/test/functions" for Cloud Functions... {"metadata":{"emulator":{"name":"functions"},"message":"Watching \"/home/fedenusy/dev/test/functions\" for Cloud Functions..."}}
[2023-02-14T22:46:12.258Z] Validating nodejs source
[2023-02-14T22:46:13.972Z] > [functions] package.json contents: {
  "name": "functions",
  "scripts": {
    "build": "tsc",
    "build:watch": "tsc --watch",
    "serve": "npm run build && firebase emulators:start --only functions",
    "shell": "npm run build && firebase functions:shell",
    "start": "npm run shell",
    "deploy": "firebase deploy --only functions",
    "logs": "firebase functions:log"
  },
  "engines": {
    "node": "16"
  },
  "main": "lib/index.js",
  "dependencies": {
    "firebase-admin": "^11.5.0",
    "firebase-functions": "^4.2.0"
  },
  "devDependencies": {
    "typescript": "^4.9.0",
    "firebase-functions-test": "^3.0.0"
  },
  "private": true
}
[2023-02-14T22:46:13.973Z] Building nodejs source
[2023-02-14T22:46:13.973Z] Failed to find version of module node: reached end of search path /home/fedenusy/dev/test/functions/node_modules
✔  functions: Using node@16 from host.
[2023-02-14T22:46:13.975Z] Could not find functions.yaml. Must use http discovery
[2023-02-14T22:46:13.979Z] Found firebase-functions binary at '/home/fedenusy/dev/test/functions/node_modules/.bin/firebase-functions'
[2023-02-14T22:46:14.070Z] Serving at port 8189

[2023-02-14T22:46:14.210Z] Got response from /__/functions.yaml {"endpoints":{"helloWorld":{"platform":"gcfv1","availableMemoryMb":null,"timeoutSeconds":null,"minInstances":null,"maxInstances":null,"ingressSettings":null,"serviceAccountEmail":null,"vpc":null,"secretEnvironmentVariables":[{"key":"SUPA_SECRET"}],"httpsTrigger":{},"entryPoint":"helloWorld"}},"specVersion":"v1alpha1","requiredAPIs":[],"params":[{"type":"secret","name":"SUPA_SECRET"}]}
[2023-02-14T22:46:14.216Z] shutdown requested via /__/quitquitquit

[2023-02-14T22:46:14.224Z] >>> [apiv2][query] GET https://secretmanager.googleapis.com/v1/projects/demo-project-id/secrets/SUPA_SECRET [none]
[2023-02-14T22:46:14.962Z] <<< [apiv2][status] GET https://secretmanager.googleapis.com/v1/projects/demo-project-id/secrets/SUPA_SECRET 403
[2023-02-14T22:46:14.963Z] <<< [apiv2][body] GET https://secretmanager.googleapis.com/v1/projects/demo-project-id/secrets/SUPA_SECRET {"error":{"code":403,"message":"Permission denied on resource project demo-project-id.","status":"PERMISSION_DENIED","details":[{"@type":"type.googleapis.com/google.rpc.Help","links":[{"description":"Google developer console API key","url":"https://console.developers.google.com/project/demo-project-id/apiui/credential"}]},{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"CONSUMER_INVALID","domain":"googleapis.com","metadata":{"consumer":"projects/demo-project-id","service":"secretmanager.googleapis.com"}}]}}
⬢  functions: Failed to load function definition from source: FirebaseError: HTTP Error: 403, Permission denied on resource project demo-project-id. {"metadata":{"emulator":{"name":"functions"},"message":"Failed to load function definition from source: FirebaseError: HTTP Error: 403, Permission denied on resource project demo-project-id."}}

┌─────────────────────────────────────────────────────────────┐
│ ✔  All emulators ready! It is now safe to connect your app. │
│ i  View Emulator UI at http://127.0.0.1:4000/               │
└─────────────────────────────────────────────────────────────┘

┌───────────┬────────────────┬─────────────────────────────────┐
│ Emulator  │ Host:Port      │ View in Emulator UI             │
├───────────┼────────────────┼─────────────────────────────────┤
│ Functions │ 127.0.0.1:5001 │ http://127.0.0.1:4000/functions │
└───────────┴────────────────┴─────────────────────────────────┘
  Emulator Hub running at 127.0.0.1:4400
  Other reserved ports: 4500

Issues? Report them at https://github.com/firebase/firebase-tools/issues and attach the *-debug.log files.

[taeold]

Hey @fedenusy.

It looks like there is an issue with using secret parameters with the emulator.

I'd recommend making the following change to unblock your setup:

import * as functions from "firebase-functions";
import { defineSecret } from "firebase-functions/params";

--defineSecret('SUPA_SECRET')

export const helloWorld = functions.runWith({ secrets: ['SUPA_SECRET'] }).https.onRequest((request, response) => {
  functions.logger.info("Hello logs!", {structuredData: true});
  response.send("Hello from Firebase!");
});

In the meantime, I'm going to update the title of this bug to be more generic.

[sceee]

Hey @taeold just to be sure I understood the issue correctly and no longer have to search for errors in my configuration: As far as I understand it's currently not possible to run the functions emulator on an unauthenticated machine (like CI) when using defineSecret(...) anywhere in the function's code even if the secrets used in these defineSecret(...) calls are provided via a .secret.local file, right?

[abegehr]

I ran into the same issue. In-case you want to keep a syntax similar to defineSecret(), you can do something like this:

const slackWebhookUrl = { name: 'SLACK_WEBHOOK_URL', value: () => process.env.SLACK_WEBHOOK_URL as string }

export const handler = region('europe-west1')
  .runWith({ secrets: [slackWebhookUrl].map((secret) => secret.name) })
  .https.onCall(async (name, context) => {
    // …
    await sendSlackMessage(slackWebhookUrl.value(), { /* … */ })
  })

Works as expected in emulator with demo--project.

[zariweyo]

I also can't use .secret.local. Using version 12.2.1.

I have tried a lot of different ways to setup that file, (as a dotenv, json, and more), but always get a validation error when emulator are trying to read it.

functions: Failed to read local secrets file /Users/XXXXXXX/functions/.secret.local: Validation failed

Example of file:

secret_pass=1112222 secret_pass_2=344566

How should I can use that file? Now I have workarrounds, but really I need to know how to configure this file.

[mvarchdev]

Same issue here... What is the status? How can we use secrets if we cannot test them? Please solve this ASAP...

[coehne]

@zariweyo, have you tried writing the secret name in uppercase? Like SECRET_PASS=1112222? It worked for me.

[zariweyo]

Oh my god, it works!!!!. Thanks @coehne

[steveoh]

This is happening with v2 functions and the emulator with tools version 12.6.0.

import { https } from 'firebase-functions/v2'
import { defineSecret } from 'firebase-functions/params'
const secretItem = defineSecret('SECRET_ITEM')

https.onCall({secrets: [secretItem]}, () => 200);

Is the advice to not define the secret when emulating? Is there an easy way to branch that logic? It doesn't seem like it...

[satyrius]

Ran into the same problem after migrating my code to functions v2 with defineSecret. The solution is to have a variable defined in .env too.

// functions/webhook.js

const { onRequest } = require('firebase-functions/v2/https')
const { defineSecret } = require('firebase-functions/params')
const stripeKey = defineSecret('STRIPE_KEY')

exports.webhook = onRequest(
  { region: EUROPE, secrets: [stripeKey] },
  async (req, res) => {
    res.json({ received: true })
  }
)

# functions/.env

# empty value just to have this variable present for the emulator
STRIPE_KEY=

# functions/.secret.local

# actual secret value
STRIPE_KEY=sk_test_***

After that Emulator won't throw that configuration error and run the function as expected

[vajahath]

Waiting on this. Now bypassed the block with adding variables to .env

[0x80]

I also just ran into this, but happy to hear that .env works at least! Personally, I use .env.local and that works too.

[sceee]

If someone is using firebase-functions-test to test functions and encounters this, the issue is actually that firebase-functions-test does not use the emulator. Instead, there is an open issue https://github.com/firebase/firebase-functions-test/issues/196 for firebase-functions-test to support something like the old mockConfig() for the new parametrized configuration.

There is also a workaround described in this issue https://github.com/firebase/firebase-functions-test/issues/196#issuecomment-1900541854 until firebase-functions-test natively supports picking up the .env config.

[glorat]

Just a note that although it is a working workaround to populate .env, which then gets picked up by resolveParams in params.ts, this is potentially dangerous as it risks commands like firebase deploy also using .env.

The proper fix is for .secret.local to be used and for the emulator to pick this up to ignore the secrets manager check. This is being discussed at #7401

[glorat]

For reference, the code that is handling for this issue is as follows (with added comments from me)

/**
 * A param defined by the SDK may resolve to:
 * - a reference to a secret in Cloud Secret Manager, which we validate the existence of and prompt for if missing
 * - a literal value of the same type already defined in one of the .env files with key == param name
 * - the value returned by interactively prompting the user
 *   - it is an error to have params that need to be prompted if the CLI is running in non-interactive mode
 *   - the default value of the prompt comes from the SDK via param.default, which may be a literal value or a CEL expression
 *   - if the default CEL expression is not resolvable--it depends on a param whose value is not yet known--we throw an error
 *   - yes, this means that the same set of params may or may not throw depending on the order the SDK provides them to us in
 *   - after prompting, the resolved value of the param is written to the most specific .env file available
 */
export async function resolveParams(
  params: Param[],
  firebaseConfig: FirebaseConfig,
  userEnvs: Record<string, ParamValue>,
  nonInteractive?: boolean,
): Promise<Record<string, ParamValue>> {
  console.info("resolveParams");
  const paramValues: Record<string, ParamValue> = populateDefaultParams(firebaseConfig);

  // TODO(vsfan@): should we ever reject param values from .env files based on the appearance of the string?
  // Use of .env is dangerous. See https://github.com/firebase/firebase-tools/issues/5520#issuecomment-2208329205 (@glorat)
  // Better to check for .secret.local (@glorat)
  const [resolved, outstanding] = partition(params, (param) => {
    return {}.hasOwnProperty.call(userEnvs, param.name);
  });
  for (const param of resolved) {
    paramValues[param.name] = userEnvs[param.name];
  }

  // TODO: should exclude secrets found in .secret.local too

  console.info('handling secrets');
  const [needSecret, needPrompt] = partition(outstanding, (param) => param.type === "secret");
  for (const param of needSecret) {
    // this line requires both internet connection and secrets manager credentials to operate and will fail otherwise - @glorat 
    await handleSecret(param as SecretParam, firebaseConfig.projectId);
  }
...