[ISSUE] Issue with `databricks_mws_ncc_binding` resource. cannot create mws ncc binding: unexpected error handling request: invalid character '<' looking for beginning of value

Configuration

 resource "databricks_mws_ncc_binding" "ncc_binding" {
      + id                             = (known after apply)
      + network_connectivity_config_id = "xxxx"
      + workspace_id                   = xxx
    }

Expected Behavior

Actual Behavior

Error: cannot create mws ncc binding: unexpected error handling request: invalid character '<' looking for beginning of value. This is likely a bug in the Databricks SDK for Go or the underlying REST API. Please report this issue with the following debugging information to the SDK issue tracker at https://github.com/databricks/databricks-sdk-go/issues. Request log: < HTTP/2.0 200 OK │ < Cache-Control: no-cache, no-store, must-revalidate │ < Content-Type: text/html; charset=utf-8 │ < Date: Thu, 22 Aug 2024 10:13:24 GMT │ < Server: databricks │ < Set-Cookie: enable-armeria-server-for-ui-flags=false; Max-Age=1800; Expires=Thu, 22 Aug 2024 10:43:24 GMT; Secure; HTTPOnly; SameSite=Strictenable-armeria-workspace-server-for-ui-flags=false; Max-Age=1800; Expires=Thu, 22 Aug 2024 10:43:24 GMT; Secure; HTTPOnly; SameSite=Strict │ < Strict-Transport-Security: max-age=31536000; includeSubDomains; preload │ < X-Content-Type-Options: nosniff │ < X-Request-Id: a01459dc-ae93-4ff4-8382-432b7c469949 │ < X-Ui-Svc: true │ < <!doctype html> │ < │ < │ < │ < │ < │ < │ < Databricks - Sign in │ < │ < │ < │ < │ < │ < │ < │ < │ < │ < │ < │ < │ < │ <

│ < │ < │ <

Steps to Reproduce

Terraform and provider versions

1.50.0

Is it a regression?

Debug Output

Important Factoids

Would you like to implement a fix?

We're also seeing this issue. After a bit of digging, it also seems the API itself (https://docs.databricks.com/api/azure/account/workspaces/update) is not functioning as expected.

Scenario: We have a workspace that is linked to an existing dummy NCC (...21bb). We like to change this NCC to a production NCC (...915e). We tried to do this through Terraform resulting in the issue as listed above.

When running the following CURL request:

curl  -X PATCH \
  'https://accounts.azuredatabricks.net/api/2.0/accounts/<ACCOUNT_ID>/workspaces/<WORKSPACE_ID>' \
  --header 'Accept: */*' \
  --header 'Authorization: Bearer <REDACTED_BEARER_TOKEN>' \
  --header 'Content-Type: application/json' \
  --data-raw '{
  "network_connectivity_config_id": "...915e"
}'

We're getting the following response:

200 OK

<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1">
    <meta name="theme-color" content="#000000">
    <meta name="description" content="Databricks Sign in">
    <title>Databricks - Sign in</title>
    <script>window.__DATABRICKS_CONFIG__={"isCuttingEdge":false,"publicPath":{"accounts-console":"https://databricks-ui-assets.azureedge.net/"}}</script>
    <link rel="icon"
      href="https://databricks-ui-assets.azureedge.net/favicon.ico">
    <script defer
      src="https://databricks-ui-assets.azureedge.net/static/js/8843.9334ce07.js"></script>
    <script defer
      src="https://databricks-ui-assets.azureedge.net/static/js/802.55bbb00f.js"></script>
    <script defer
      src="https://databricks-ui-assets.azureedge.net/static/js/9806.4bdefc97.js"></script>
    <script>
  function setNoCdnAndReload() {
      document.cookie = `x-databricks-cdn-inaccessible=true; path=/; max-age=86400`;
      const metric = 'cdnFallbackOccurred';
      const browserUserAgent = navigator.userAgent;
      const browserTabId = window.browserTabId;
      const performanceEntry = performance.getEntriesByType('resource').filter(e => e.initiatorType === 'script').slice(-1)[0]
      sessionStorage.setItem('databricks-cdn-fallback-telemetry-key', JSON.stringify({ tags: { browserUserAgent, browserTabId }, performanceEntry}));
      window.location.reload();
  }
</script>
    <script>
  // Set a manual timeout for dropped packets to CDN
  function loadScriptWithTimeout(src, timeout) {
     return new Promise((resolve, reject) => {
        const script = document.createElement('script');
          script.defer = true;
          script.src = src;
          script.onload = resolve;
          script.onerror = reject;
          document.head.appendChild(script);
          setTimeout(() => {
              reject(new Error('Script load timeout'));
          }, timeout);
      });
  }
  loadScriptWithTimeout('https://databricks-ui-assets.azureedge.net/static/js/login.10991e62.js', 10000).catch(setNoCdnAndReload);
</script>
    <link
      href="https://databricks-ui-assets.azureedge.net/static/css/8843.013d6ea4.css"
      rel="stylesheet">
    <link
      href="https://databricks-ui-assets.azureedge.net/static/css/9806.ec03b64f.css"
      rel="stylesheet">
    <link
      href="https://databricks-ui-assets.azureedge.net/static/css/login.5a13953a.css"
      rel="stylesheet">
  </head>
  <body>
    <noscript>
      You need to enable JavaScript to run this app.
    </noscript>
    <div id="login"></div>
    <script>const telemetryEndpoint="/telemetry-unauth?t=",uiModuleName="accountsConsoleLogin";function shouldIgnoreError(e){return!1}function generateUuidV4(){const e=window.crypto?.randomUUID?.();return e||"xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g,(e=>{const n=16*Math.random()|0;return("x"===e?n:3&n|8).toString(16)}))}function networkConnectivityTags(){const e=window.navigator.onLine,n=window.navigator.connection?.rtt??-1,t=window.navigator.connection?.downlink??-1;return{browserNavigatorOnline:e,browserConnectionEstimatedRtt:n,browserConnectionEstimatedDownlink:t,browserConnected:e&&n>0&&t>0}}function createTelemetryRequestBody(e,n={},t=null){const o=Math.round(Date.now()/1e3),r={eventId:generateUuidV4(),metric:e,tags:{...n,...networkConnectivityTags(),browserTabId:window.browserTabId,browserUserAgent:navigator.userAgent},ts:o};return t&&(r.blob=t),JSON.stringify({uploadTime:o,items:[JSON.stringify(r)]})}function recordTelemetry(e,n={},t=""){const o={method:"POST",credentials:"include",body:createTelemetryRequestBody(e,n,t),headers:{"Content-Type":"application/json","X-Databricks-Org-Id":"AC"}};fetch(telemetryEndpoint+Date.now(),o)}window.__databricks_networkConnectivityTags=networkConnectivityTags,Object.defineProperty(window,"browserTabId",{value:generateUuidV4()}),window.recordTelemetry=recordTelemetry,recordTelemetry("uiInit",{uiModule:uiModuleName,eventId:"init",eventClientSource:uiModuleName,eventType:"init"});let logCount=0;function error_handler(e,n,t,o,r){logCount++>4||shouldIgnoreError(e)||recordTelemetry("uncaughtJsException",{eventType:"jsExceptionV3",jsExceptionMessage:e,jsExceptionSource:n,jsExceptionLineno:t,jsExceptionColno:o,jsExceptionBeforeInit:!0},r&&r.stack&&r.stack.toString())}function sendBeaconOnPageExit(e){if(navigator.sendBeacon){const n=e&&e.type||"unknown",t=(Math.round(Date.now()/1e3),createTelemetryRequestBody("uiInit",{eventType:"pageExitBeforeAppInitComplete",eventName:n,eventClientSource:uiModuleName}));navigator.sendBeacon(telemetryEndpoint+Date.now(),t)}}window.onerror=error_handler,window.onunhandledrejection=function(e){error_handler(String(e.reason),null,null,null,e.reason)},window.addEventListener("beforeunload",sendBeaconOnPageExit),window.addEventListener("unload",sendBeaconOnPageExit),window.addEventListener("pagehide",sendBeaconOnPageExit),window.cleanupAfterAppInit=()=>{window.removeEventListener("beforeunload",sendBeaconOnPageExit),window.removeEventListener("unload",sendBeaconOnPageExit),window.removeEventListener("pagehide",sendBeaconOnPageExit)}</script>
  </body>
</html>

While all seems okay from the 200 OK response, after checking the workspace itself, we see the NCC is not updated to the ...915e NCC, and is still reporting the initial ...21bb dummy NCC. Furthermore, the API itself is responding with a login page (similar to the one that caused the provider to panic), so my take is that it's likely an API issue rather than a provider issue

databricks / terraform-provider-databricks