Run netbird behind reverse proxy

[TheEdgeOfRage]

I'm looking into setting up netbird on a server where I have several other services already running, with nginx acting as a reverse proxy. I see in the documentation and docker-compose that netbird tries to expose ports 80 and 443 on the host machine and get letsencrypt certs by itself, but is there a way to have it run in http-only mode and have my host nginx handle all the letsencrypt and tls stuff?

I also see that the management service in the docker-compose file also requires access to the letsencrypt certs, but I could just mount those from my host system, right?

[mlsmaycon]

Hi @TheEdgeOfRage, Andi_bz did an awesome job summarizing the steps to run behind a Traefik proxy:

https://www.reddit.com/r/selfhosted/comments/xpju6p/comment/iu85hqy/?context=1

For Nginx, you can map the configuration and use something like:

upstream signal {
   server 192.168.3.83:310000;
}
server {
  listen 443 ssl http2;

  ssl_certificate /tmp/server.crt;  #Enter you certificate location 
  ssl_certificate_key /tmp/server.key;

  location /signalexchange {
     grpc_pass grpcs://signal;
  }
}

[TheEdgeOfRage]

Cool, I'll try that. Thank you :)

[TheEdgeOfRage]

@mlsmaycon I've been trying to get this up for a while now. I got all the server-side stuff to run and I can open the dashboard and log in through Auth0, but the CLI won't connect to the management service. I get this error when trying to connect:

$ sudo netbird up --management-url https://netbirdapi.redacted.com

WARN[2022-11-19T21:01:12+01:00]root.go:166 retrying Login to the Management service in 1.104660288s due to error rpc error: code = Unknown desc = context deadline exceeded
WARN[2022-11-19T21:01:23+01:00]root.go:166 retrying Login to the Management service in 2.160763633s due to error rpc error: code = Unknown desc = context deadline exceeded
Error: login backoff cycle failed: rpc error: code = Unknown desc = context deadline exceeded

Here's my docker-compose:

version: "3"
services:
  #UI dashboard
  dashboard:
    image: wiretrustee/dashboard:main
    restart: unless-stopped
    environment:
      - AUTH_AUDIENCE=https://redacted.eu.auth0.com/api/v2/
      - AUTH_CLIENT_ID=VooPlF3gVzYXHDyYQWM8hZMW8SJNUk2P
      - AUTH_AUTHORITY=https://redacted.eu.auth0.com/
      - USE_AUTH0=true
      - AUTH_SUPPORTED_SCOPES=openid profile email offline_access api email_verified
      - NETBIRD_MGMT_API_ENDPOINT=https://netbirdapi.redacted.com
      - NETBIRD_MGMT_GRPC_API_ENDPOINT=http://netbirdapi.redacted.com:33073
      - NGINX_SSL_PORT=443
      - AUTH_REDIRECT_URI=
      - AUTH_SILENT_REDIRECT_URI=
    ports:
      - "8029:80"
  # Signal
  signal:
    image: netbirdio/signal:latest
    restart: unless-stopped
    volumes:
      - netbird-signal:/var/lib/netbird
    ports:
      - 10000:80
  # Management
  management:
    image: netbirdio/management:latest
    restart: unless-stopped
    depends_on:
      - dashboard
    command: ["--port", "443", "--log-file", "console", "--disable-anonymous-metrics=true", "--single-account-mode-domain="]
    volumes:
      - netbird-mgmt:/var/lib/netbird
      - ./management.json:/etc/netbird/management.json
    ports:
      - 33073:33073 #API port
      - 33074:443 #API port
  # Coturn
  coturn:
    image: coturn/coturn
    restart: unless-stopped
    domainname: netbird.redacted.com
    volumes:
      - ./turnserver.conf:/etc/turnserver.conf:ro
    network_mode: host
volumes:
  netbird-mgmt:
  netbird-signal:

And here's my host nginx config for netbird:

server {
    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    server_name netbird.redacted.com;

    location / {
        proxy_pass  http://127.0.0.1:8029;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
        proxy_set_header X-Forwarded-Proto https;
    }
}

server {
    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    server_name netbirdapi.redacted.com;

    location /api {
        proxy_pass  http://127.0.0.1:33074;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
        proxy_set_header X-Forwarded-Proto https;
    }

    location /signalexchange {
        grpc_pass grpcs://127.0.0.1:10000;
    }

    location /management {
        grpc_pass grpc://127.0.0.1:33074;
        # grpc_pass grpc://127.0.0.1:33073;
        # grpc_pass grpcs://127.0.0.1:33073;
        # grpc_pass grpcs://127.0.0.1:33074;
    }
}

The commented grpc_pass directives in the management section are all the ones I've tried.

And finally, the management.json:

{
    "Stuns": [
        {
            "Proto": "udp",
            "URI": "stun:netbird.redacted.com:3478",
            "Username": "",
            "Password": null
        }
    ],
    "TURNConfig": {
        "Turns": [
            {
                "Proto": "udp",
                "URI": "turn:netbird.redacted.com:3478",
                "Username": "self",
                "Password": "redacted"
            }
        ],
        "CredentialsTTL": "12h",
        "Secret": "secret",
        "TimeBasedCredentials": false
    },
    "Signal": {
        "Proto": "http",
        "URI": "signal:80",  # tried both this and "netbird.theedgeofrage.com:443"
        "Username": "",
        "Password": null
    },
    "Datadir": "",
    "HttpConfig": {
        "Address": "0.0.0.0:33073",
        "AuthIssuer": "https://redacted.eu.auth0.com/",
        "AuthAudience": "https://redacted.eu.auth0.com/api/v2/",
        "AuthKeysLocation": "https://redacted.eu.auth0.com/.well-known/jwks.json",
        "OIDCConfigEndpoint":"https://redacted.eu.auth0.com/.well-known/openid-configuration"
    },
    "IdpManagerConfig": {
        "Manager": "none"
     },
    "DeviceAuthorizationFlow": {
        "Provider": "none",
        "ProviderConfig": {
          "Audience": "https://redacted.eu.auth0.com/api/v2/",
          "Domain": "",
          "ClientID": "",
          "TokenEndpoint": "https://redacted.eu.auth0.com/oauth/token",
          "DeviceAuthEndpoint": "https://redacted.eu.auth0.com/oauth/device/code"
         }
    }
}

Any help on this would be appreciated

[mlsmaycon]

hello @TheEdgeOfRage, I spotted a few things to adjust: on docker-compose.yml:

1. NETBIRD_MGMT_GRPC_API_ENDPOINT=http://netbirdapi.redacted.com:33073 should be NETBIRD_MGMT_GRPC_API_ENDPOINT=https://netbirdapi.redacted.com:443
2. management.ports should have only 33074:443 #API port

on nginx.conf

1. /signalexchange location is pointing to grpcs
2. also, you are pointing to localhost, but I am assuming your nginx is not running on a container, right?

on management.json

1. you can configure your signal as follows:

   "Signal": {
       "Proto": "https",
       "URI": "netbirdapi.theedgeofrage.com:443",
       "Username": "",
       "Password": null
   },

once you do that, you can run your clients with:

netbird up --management-url https://netbirdapi.redacted.com:443

[TheEdgeOfRage]

Success! A different error :smile:

Error: login failed: rpc error: code = NotFound desc = no SSO provider returned from management. If you are using hosting Netbird see documentation at https://github.com/netbirdio/netbird/tree/main/management for details

It seems to be at least connecting now though. The nginx logs show this:

 - - [20/Nov/2022:18:07:23 +0100] "POST /management.ManagementService/GetServerKey HTTP/2.0" 200 61 "-" "grpc-go/1.43.0"
 - - [20/Nov/2022:18:07:23 +0100] "POST /management.ManagementService/Login HTTP/2.0" 200 0 "-" "grpc-go/1.43.0"
 - - [20/Nov/2022:18:07:24 +0100] "POST /management.ManagementService/GetServerKey HTTP/2.0" 200 60 "-" "grpc-go/1.43.0"
 - - [20/Nov/2022:18:07:24 +0100] "POST /management.ManagementService/GetDeviceAuthorizationFlow HTTP/2.0" 200 0 "-" "grpc-go/1.43.0"

This smells like a configuration issue to me. But considering I managed to log into the dashboard through Auth0, I'm not sure what could be causing this :/ and no other issue mentions that error

[mlsmaycon]

you are missing the Domain and ClientID for the DeviceAuthorizationFlow in your management.json, if you didn't create a new application in auth0 yet, please check the guide here: https://netbird.io/docs/integrations/identity-providers/self-hosted/using-netbird-with-auth0#step-4-enable-interactive-sso-login-optional

[TheEdgeOfRage]

I have now configured my management.json like this:

{
    ...
    "DeviceAuthorizationFlow": {
        "Provider": "none",
        "ProviderConfig": {
          "Audience": "https://redacted.eu.auth0.com/api/v2/",
          "Domain": "redacted.eu.auth0.com",
          "ClientID": "redacted-client-id",
          "TokenEndpoint": "https://redacted.eu.auth0.com/oauth/token",
          "DeviceAuthEndpoint": "https://redacted.eu.auth0.com/oauth/device/code"
         }
    }
}

But I still get the same error on the client:

Error: login failed: rpc error: code = NotFound desc = no SSO provider returned from management.

[mlsmaycon]

I missed a couple of things, actually:

You've been using what it seems to be the Auth0 management API identifier as Audience, it should be the one created in step 3 of the guide. Looking at your whole configuration, you might need to update all entries of "https://redacted.eu.auth0.com/api/v2/" with the correct value.

Also, you must specify the DeviceAuthorizationFlow.Provider as "hosted" to enable it.

[TheEdgeOfRage]

Yeah, looks like I used the system API that Auth0 created by default :grimacing: I created a fresh one and used that. It works now. Thank you for the help and sorry for the bother <3

[mlsmaycon]

Thanks for checking that and for sharing your configuration files; we might use that in our docs later.

[TheEdgeOfRage]

If you find you need some other config just let me know. I'll be happy to help :)

[MohammedNoureldin]

Hey @TheEdgeOfRage,

have you managed or tried to get this working using K8s ingress? For some reason I am not able to connect. Is there any way I can debug this properly? Which configuration/annotations did you use for Nginx ingress if you have already used it?

[TheEdgeOfRage]

I have not tried to use a k8s ingress, but it shouldn't be too dissimilar to just using plain nginx. Here's my nginx config:

server {
    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    server_name netbird.example.com;

    error_log /var/log/nginx/netbird.error.log error;
    access_log /var/log/nginx/netbird.access.log;

    location / {
        proxy_pass  http://127.0.0.1:8029;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
        proxy_set_header X-Forwarded-Proto https;
    }
}

server {
    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    server_name netbirdapi.example.com;

    error_log /var/log/nginx/netbirdapi.error.log error;
    access_log /var/log/nginx/netbirdapi.access.log;

    location /api {
        proxy_pass  http://127.0.0.1:33073;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Host $server_name;
        proxy_set_header X-Forwarded-Proto https;
    }

    location /signalexchange {
        grpc_pass grpc://127.0.0.1:10000;
    }

    location /management {
        grpc_pass grpc://127.0.0.1:33073;
    }
}

I'm not sure how you would get the grpc_pass directives on a k8s ingress tho :/ Never tried that

[MohammedNoureldin]

Well, for Nginx ingress, I got it working using these annotations (though I am still unsure how important the SSL is):

    annotations:
      nginx.ingress.kubernetes.io/ssl-redirect: "true"
      nginx.ingress.kubernetes.io/backend-protocol: "GRPC"

[MohammedNoureldin]

@TheEdgeOfRage Any idea why do need to specify the --port in the args of the container and in the management.json? One of them seems to be redundant?

And also, why would we need to forward to the port 443 instead of port 80 if the netbird management will run with no TLS, because the reverse proxy should take care about forwarding the traffic from its HTTPs 443 to the containers port 80?

[TheEdgeOfRage]

I did not list the port in management.json, only as an arg.

It's set to port 443, but it's not actually serving TLS traffic, just regular plain HTTP.

[HybridRCG]

Can you please share all the config files if possible? I've bene stuck for weeks trying to get this installed. :( I have nginx as my reverse proxy. I want to use authentik . have that working as expected. But the netbird install help docs is not very clear on how to go about setting this up. Thanks , will be much appreciated!

[TheEdgeOfRage]

I have since stopped using netbird and don't have the configs anymore, sorry :/

[SinghNanak]

@HybridRCG have you found any success yet because i also want to run Netbird behind Nginx Proxy Manager with Authentik.

can you share your setup.env file. what addition did you make for npm in the file

[mathiash98]

For anyone struggling here is my working Netbird in docker with new relay server + bare metal nginx reverse proxy + Azure EntraID login:

docker-compose.yml

```yml version: "3" services: #UI dashboard dashboard: image: netbirdio/dashboard:latest restart: unless-stopped ports: - 8011:80 environment: # Endpoints - NETBIRD_MGMT_API_ENDPOINT=https://netbird.redacted.com - NETBIRD_MGMT_GRPC_API_ENDPOINT=https://netbird.redacted.com # OIDC - AUTH_AUDIENCE=CLIENT_ID - AUTH_CLIENT_ID=CLIENT_ID - AUTH_CLIENT_SECRET= - AUTH_AUTHORITY=https://login.microsoftonline.com/TENANT_ID/v2.0 - USE_AUTH0=false - AUTH_SUPPORTED_SCOPES=openid profile email offline_access User.Read api://CLIENT_ID/api - AUTH_REDIRECT_URI=/auth - AUTH_SILENT_REDIRECT_URI=/silent-auth - NETBIRD_TOKEN_SOURCE=idToken logging: driver: "json-file" options: max-size: "500m" max-file: "2" # Signal signal: image: netbirdio/signal:latest restart: unless-stopped volumes: - netbird-signal:/var/lib/netbird ports: - 10000:80 logging: driver: "json-file" options: max-size: "500m" max-file: "2" # Relay relay: image: netbirdio/relay:latest restart: unless-stopped environment: - NB_LOG_LEVEL=debug - NB_LISTEN_ADDRESS=:33080 - NB_EXPOSED_ADDRESS=rels://netbird.redacted.com:443 - NB_AUTH_SECRET=SECRET ports: - 33080:33080 logging: driver: "json-file" options: max-size: "500m" max-file: "2" # Management management: image: netbirdio/management:latest restart: unless-stopped depends_on: - dashboard volumes: - netbird-mgmt:/var/lib/netbird - ./management.json:/etc/netbird/management.json ports: - 8012:80 #API port command: [ "--port", "80", "--log-file", "console", "--log-level", "debug", "--disable-anonymous-metrics=false", "--single-account-mode-domain=netbird.redacted.com", "--dns-domain=netbird.selfhosted" ] logging: driver: "json-file" options: max-size: "500m" max-file: "2" environment: - NETBIRD_STORE_ENGINE_POSTGRES_DSN= # Coturn coturn: image: coturn/coturn:latest restart: unless-stopped #domainname: netbird.redacted.com # only needed when TLS is enabled volumes: - ./turnserver.conf:/etc/turnserver.conf:ro # - ./privkey.pem:/etc/coturn/private/privkey.pem:ro # - ./cert.pem:/etc/coturn/certs/cert.pem:ro network_mode: host command: - -c /etc/turnserver.conf logging: driver: "json-file" options: max-size: "500m" max-file: "2" volumes: netbird-mgmt: netbird-signal: ```

nginx.conf

```nginx server { # HTTP server config listen 80; server_name _; # 301 redirect to HTTPS location / { return 301 https://$host$request_uri; } } server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name netbird.redacted.com; # netbird settings ============== # This is necessary so that grpc connections do not get closed early # see https://stackoverflow.com/a/67805465 client_header_timeout 1d; client_body_timeout 1d; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Scheme $scheme; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-Host $host; grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # Proxy dashboard location / { proxy_pass http://127.0.0.1:8011; } # Proxy Relay location /relay { proxy_pass http://127.0.0.1:33080; # WebSocket support proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; # Timeout settings proxy_read_timeout 3600s; proxy_send_timeout 3600s; proxy_connect_timeout 60s; # Handle upstream errors proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; } # Proxy Signal location /signalexchange { grpc_pass grpc://127.0.0.1:10000; #grpc_ssl_verify off; grpc_read_timeout 1d; grpc_send_timeout 1d; grpc_socket_keepalive on; } # Proxy Management http endpoint location /api { proxy_pass http://127.0.0.1:8012; } # Proxy Management grpc endpoint location /management { grpc_pass grpc://127.0.0.1:8012; #grpc_ssl_verify off; grpc_read_timeout 1d; grpc_send_timeout 1d; grpc_socket_keepalive on; } # Netbird requires settings end ============== ssl_certificate /etc/letsencrypt/live/netbird.redacted.com/fullchain.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/netbird.redacted.com/privkey.pem; # managed by Certbot } ```

management.json

```json { "Stuns": [ { "Proto": "udp", "URI": "stun:netbird.redacted.com:3478", "Username": "", "Password": "" } ], "TURNConfig": { "TimeBasedCredentials": false, "CredentialsTTL": "12h0m0s", "Secret": "secret", "Turns": [ { "Proto": "udp", "URI": "turn:netbird.redacted.com:3478", "Username": "self", "Password": "PASSWORD" } ] }, "Relay": { "Addresses": [ "rels://netbird.redacted.com:443" ], "CredentialsTTL": "24h0m0s", "Secret": "SECRET" }, "Signal": { "Proto": "https", "URI": "netbird.redacted.com:443", "Username": "", "Password": "" }, "Datadir": "/var/lib/netbird/", "DataStoreEncryptionKey": "ENCRYPTIONKEY", "HttpConfig": { "LetsEncryptDomain": "", "CertFile": "", "CertKey": "", "AuthAudience": "CLIENT_ID", "AuthIssuer": "https://login.microsoftonline.com/TENANT_ID/v2.0", "AuthUserIDClaim": "oid", "AuthKeysLocation": "https://login.microsoftonline.com/TENANT_ID/discovery/v2.0/keys", "OIDCConfigEndpoint": "https://login.microsoftonline.com/TENANT_ID/v2.0/.well-known/openid-configuration", "IdpSignKeyRefreshEnabled": false, "ExtraAuthAudience": "" }, "IdpManagerConfig": { "ManagerType": "azure", "ClientConfig": { "Issuer": "https://login.microsoftonline.com/TENANT_ID/v2.0", "TokenEndpoint": "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token", "ClientID": "CLIENT_ID", "ClientSecret": "SECRET", "GrantType": "client_credentials" }, "ExtraConfig": { "GraphApiEndpoint": "https://graph.microsoft.com/v1.0", "ObjectId": "OBJECT_ID" }, "Auth0ClientCredentials": null, "AzureClientCredentials": null, "KeycloakClientCredentials": null, "ZitadelClientCredentials": null }, "DeviceAuthorizationFlow": { "Provider": "none", "ProviderConfig": { "ClientID": "", "ClientSecret": "", "Domain": "", "Audience": "CLIENT_ID", "TokenEndpoint": "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token", "DeviceAuthEndpoint": "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/devicecode", "AuthorizationEndpoint": "", "Scope": "openid", "UseIDToken": false, "RedirectURLs": null } }, "PKCEAuthorizationFlow": { "ProviderConfig": { "ClientID": "CLIENT_ID", "ClientSecret": "", "Domain": "", "Audience": "CLIENT_ID", "TokenEndpoint": "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token", "DeviceAuthEndpoint": "", "AuthorizationEndpoint": "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/authorize", "Scope": "openid profile email offline_access User.Read api://CLIENT_ID/api", "UseIDToken": true, "RedirectURLs": [ "http://localhost:53000" ] } }, "StoreConfig": { "Engine": "sqlite" }, "ReverseProxy": { "TrustedHTTPProxies": [], "TrustedHTTPProxiesCount": 0, "TrustedPeers": [ "0.0.0.0/0" ] } } ```