Make secure token storage the default storage mode

NEXT_CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Notable Changes
 
+* Breaking change: OAuth tokens for interactive logins (`auth_type = databricks-cli`) are now stored in the OS-native secure store by default (Keychain on macOS, Credential Manager on Windows, Secret Service on Linux) instead of `~/.databricks/token-cache.json`. After upgrading, run `databricks auth login` once per profile to re-authenticate; cached tokens from older versions are not migrated. To keep the previous file-backed storage, set `DATABRICKS_AUTH_STORAGE=plaintext` or add `auth_storage = plaintext` under `[__settings__]` in `~/.databrickscfg` (the env var takes precedence over the config setting), then re-run `databricks auth login`. On systems where the OS keyring is not reachable (e.g. Linux containers without a D-Bus session bus), the CLI transparently falls back to the file cache when reading tokens so legacy `token-cache.json` entries remain accessible without manual configuration.
+
 ### CLI
 
 * Added `databricks aitools` command group for installing Databricks skills into your coding agents (Claude Code, Cursor, Codex CLI, OpenCode, GitHub Copilot, Antigravity). Skills are fetched from [github.com/databricks/databricks-agent-skills](https://github.com/databricks/databricks-agent-skills) and either symlinked into each agent's skills directory or copied into the current project. Use `databricks aitools install` to set up, `update` to pull newer versions, `list` to see what's available, and `uninstall` to remove them.

acceptance/cmd/auth/describe/u2m-json-output/output.txt

@@ -2,7 +2,7 @@
 >>> [CLI] auth describe --profile u2m-profile --output json
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
 {
-  "mode": "plaintext",
-  "location": "~/.databricks/token-cache.json",
+  "mode": "secure",
+  "location": "OS keyring (service: databricks-cli)",
   "source": "default"
 }

acceptance/cmd/auth/describe/u2m-plaintext-config/output.txt

@@ -1,7 +1,7 @@
 
 >>> [CLI] auth describe --profile u2m-profile
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
-Unable to authenticate: error getting token: cache: token not found
+Unable to authenticate: error getting token: cache: no cached credentials; run `databricks auth login` to sign in
 Token storage: plaintext, ~/.databricks/token-cache.json (from auth_storage in [__settings__] section of [TEST_TMP_DIR]/home/.databrickscfg)
 -----
 Current configuration:

acceptance/cmd/auth/describe/u2m-plaintext-env/output.txt

@@ -1,7 +1,7 @@
 
 >>> [CLI] auth describe --profile u2m-profile
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
-Unable to authenticate: error getting token: cache: token not found
+Unable to authenticate: error getting token: cache: no cached credentials; run `databricks auth login` to sign in
 Token storage: plaintext, ~/.databricks/token-cache.json (from DATABRICKS_AUTH_STORAGE environment variable)
 -----
 Current configuration:

acceptance/cmd/auth/describe/u2m-plaintext-default/output.txt → acceptance/cmd/auth/describe/u2m-secure-default/output.txt

@@ -1,8 +1,8 @@
 
 >>> [CLI] auth describe --profile u2m-profile
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
-Unable to authenticate: error getting token: cache: token not found
-Token storage: plaintext, ~/.databricks/token-cache.json (from default)
+Unable to authenticate: error getting token: [KEYRING_LOOKUP_ERROR]
+Token storage: secure, OS keyring (service: databricks-cli) (from default)
 -----
 Current configuration:
   ✓ host: https://u2m-profile.databricks.test (from [TEST_TMP_DIR]/home/.databrickscfg config file)

acceptance/cmd/auth/describe/u2m-secure-default/test.toml

@@ -0,0 +1,11 @@
+Ignore = [
+    "home"
+]
+
+# This test runs against the real OS keyring at Lookup time (no writes).
+# macOS produces a clean miss; Linux without a usable D-Bus session bus
+# produces a backend error. Normalize both so the assertion stays on the
+# resolved storage mode, not the lookup outcome.
+[[Repls]]
+Old = 'Unable to authenticate: error getting token: .*'
+New = 'Unable to authenticate: error getting token: [KEYRING_LOOKUP_ERROR]'

acceptance/cmd/auth/logout/stale-account-id-workspace-host/output.txt

@@ -36,4 +36,4 @@ logfood (Default)  [DATABRICKS_URL]  NO
 
 === Logged out profile should no longer return a token
 >>> musterr [CLI] auth token --profile logfood
-Error: cache: databricks OAuth is not configured for this host. Try logging in again with `databricks auth login --profile logfood` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new
+Error: cache: databricks OAuth is not configured for this host. no cached credentials; run `databricks auth login` to sign in

acceptance/cmd/auth/token/default-profile/output.txt

@@ -3,7 +3,7 @@
 
 >>> errcode [CLI] auth token
 Warn: [hostmetadata] failed to fetch host metadata for https://myworkspace.test, will skip for 1m0s
-Error: cache: databricks OAuth is not configured for this host. Try logging in again with `databricks auth login --profile myprofile` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new
+Error: cache: databricks OAuth is not configured for this host. no cached credentials; run `databricks auth login` to sign in
 
 Exit code: 1
 

acceptance/cmd/auth/token/force-refresh-no-cache/output.txt

@@ -1,3 +1,3 @@
-Error: cache: databricks OAuth is not configured for this host. Try logging in again with `databricks auth login --profile test-profile` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new
+Error: cache: databricks OAuth is not configured for this host. no cached credentials; run `databricks auth login` to sign in
 
 Exit code: 1

acceptance/cmd/auth/token/output.txt

@@ -1,5 +1,5 @@
 
 >>> [CLI] auth token --host [DATABRICKS_URL]
-Error: cache: databricks OAuth is not configured for this host. Try logging in again with `databricks auth login --host [DATABRICKS_URL]` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new
+Error: cache: databricks OAuth is not configured for this host. no cached credentials; run `databricks auth login` to sign in
 
 Exit code: 1

acceptance/script.prepare

@@ -1,3 +1,9 @@
+# Force plaintext token storage so acceptance tests exercise the file-backed
+# cache rather than the OS keyring, which is not reliably reachable in CI.
+# Tests that want to exercise the secure path or the resolver default unset
+# or override this in their own script.prepare or script.
+export DATABRICKS_AUTH_STORAGE=plaintext
+
 errcode() {
     # Temporarily disable 'set -e' to prevent the script from exiting on error
     set +e

cmd/auth/auth_test.go

@@ -1,6 +1,8 @@
 package auth
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
@@ -113,7 +115,14 @@ func TestProfileHostConflictTokenViaCobra(t *testing.T) {
 // pass the conflict check (the command will fail later for other reasons, but
 // NOT with a conflict error).
 func TestProfileHostCompatibleViaCobra(t *testing.T) {
-	t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg")
+	// Copy the fixture into a temp directory so the auth login flow's writes
+	// (e.g. silent plaintext-fallback persistence on CI runners without a
+	// usable keyring) cannot dirty the checked-in fixture.
+	configPath := filepath.Join(t.TempDir(), ".databrickscfg")
+	fixture, err := os.ReadFile("./testdata/.databrickscfg")
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(configPath, fixture, 0o600))
+	t.Setenv("DATABRICKS_CONFIG_FILE", configPath)
 
 	ctx := cmdctx.GenerateExecId(t.Context())
 	cli := root.New(ctx)
@@ -125,7 +134,7 @@ func TestProfileHostCompatibleViaCobra(t *testing.T) {
 		"--host", "https://www.host1.test",
 	})
 
-	_, err := cli.ExecuteContextC(ctx)
+	_, err = cli.ExecuteContextC(ctx)
 	// The command may fail for other reasons (no browser, non-interactive, etc.)
 	// but it should NOT fail with a conflict error.
 	if err != nil {

cmd/auth/describe_test.go

@@ -240,21 +240,21 @@ func TestResolveTokenStorageInfo(t *testing.T) {
 			want:     nil,
 		},
 		{
-			name:     "databricks-cli with default plaintext",
+			name:     "databricks-cli with default secure",
 			authType: authTypeDatabricksCLI,
 			want: &tokenStorageInfo{
-				Mode:     "plaintext",
-				Location: plaintextLocation,
+				Mode:     "secure",
+				Location: secureLocation,
 				Source:   "default",
 			},
 		},
 		{
-			name:     "databricks-cli with secure from env",
+			name:     "databricks-cli with plaintext from env",
 			authType: authTypeDatabricksCLI,
-			envValue: "secure",
+			envValue: "plaintext",
 			want: &tokenStorageInfo{
-				Mode:     "secure",
-				Location: secureLocation,
+				Mode:     "plaintext",
+				Location: plaintextLocation,
 				Source:   "DATABRICKS_AUTH_STORAGE environment variable",
 			},
 		},

cmd/auth/login.go

@@ -144,15 +144,6 @@ a new profile is created.
 		ctx := cmd.Context()
 		profileName := cmd.Flag("profile").Value.String()
 
-		// Resolve the cache before the browser step so an unavailable
-		// keyring surfaces here rather than after OAuth. The probe also
-		// triggers the OS unlock prompt, which the user can answer during
-		// OAuth.
-		tokenCache, mode, err := storage.ResolveCacheForLogin(ctx, "")
-		if err != nil {
-			return err
-		}
-
 		// Cluster and Serverless are mutually exclusive.
 		if configureCluster && configureServerless {
 			return errors.New("please either configure serverless or cluster, not both")
@@ -178,6 +169,16 @@ a new profile is created.
 			}
 		}
 
+		// Resolve the cache before the browser step so an unavailable
+		// keyring surfaces here rather than after OAuth. The probe also
+		// triggers the OS unlock prompt, which the user can answer during
+		// OAuth. Run after input validation so trivially-invalid commands
+		// fail without probing.
+		tokenCache, mode, err := storage.ResolveCacheForLogin(ctx, "")
+		if err != nil {
+			return err
+		}
+
 		// When interactive and nothing was specified, show a picker that lets
 		// the user re-login to an existing profile, create a new one, or enter
 		// a host URL. With no profiles configured the picker still shows the
@@ -294,6 +295,11 @@ a new profile is created.
 		if err = persistentAuth.Challenge(); err != nil {
 			return err
 		}
+		// Lock secure mode in after a successful keyring write so a later
+		// transient keyring probe failure cannot silently demote this user
+		// to plaintext.
+		storage.PinSecureMode(ctx, mode, storage.StorageModeUnknown)
+
 		// At this point, an OAuth token has been successfully minted and stored
 		// in the CLI cache. The rest of the command focuses on:
 		// 1. Workspace selection for SPOG hosts (best-effort);
@@ -633,6 +639,7 @@ func discoveryLogin(ctx context.Context, in discoveryLoginInputs) error {
 	if err := persistentAuth.Challenge(); err != nil {
 		return discoveryErr("login via login.databricks.com failed", err)
 	}
+	storage.PinSecureMode(ctx, in.mode, storage.StorageModeUnknown)
 
 	discoveredHost := arg.GetDiscoveredHost()
 	if discoveredHost == "" {

cmd/auth/token.go

@@ -285,10 +285,22 @@ func loadToken(ctx context.Context, args loadTokenArgs) (*oauth2.Token, error) {
 			//
 			// Older SDK versions check for a particular substring to determine if
 			// the OAuth authentication type can fall through or if it is a real error.
-			// This means we need to keep this error message constant for backwards compatibility.
+			// This means we need to keep "databricks OAuth is not configured for
+			// this host" present in the error for backwards compatibility.
 			//
 			// This is captured in an acceptance test under "cmd/auth/token".
-			err = errors.New("cache: databricks OAuth is not configured for this host")
+			const compatSubstring = "databricks OAuth is not configured for this host"
+			// When storage's notFoundHintCache wrapped the ErrNotFound with
+			// an actionable hint (e.g. the post-upgrade "stored credentials
+			// from older CLI versions are no longer used; run `databricks
+			// auth login`..." copy), surface it instead of the generic
+			// "Try logging in again with ... If this fails, please report
+			// this issue" trailer. The hint is more specific and avoids
+			// users reporting expected post-upgrade behavior as a bug.
+			if hint := storage.HintForNotFound(err); hint != "" {
+				return nil, fmt.Errorf("cache: %s. %s", compatSubstring, hint)
+			}
+			err = errors.New("cache: " + compatSubstring)
 		}
 		if rewritten, rewrittenErr := auth.RewriteAuthError(ctx, args.authArguments.Host, args.authArguments.AccountID, args.profileName, err); rewritten {
 			return nil, rewrittenErr
@@ -433,6 +445,7 @@ func runInlineLogin(ctx context.Context, profiler profile.Profiler, tokenCache c
 	if err = persistentAuth.Challenge(); err != nil {
 		return "", nil, err
 	}
+	storage.PinSecureMode(ctx, mode, storage.StorageModeUnknown)
 
 	clearKeys := oauthLoginClearKeys()
 	clearKeys = append(clearKeys, databrickscfg.ExperimentalIsUnifiedHostKey)

cmd/auth/token_test.go

@@ -10,15 +10,34 @@ import (
 	"time"
 
 	"github.com/databricks/cli/libs/auth"
+	"github.com/databricks/cli/libs/auth/storage"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/databricks-sdk-go/credentials/u2m"
+	"github.com/databricks/databricks-sdk-go/credentials/u2m/cache"
 	"github.com/databricks/databricks-sdk-go/httpclient/fixtures"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/oauth2"
 )
 
+// upgradeHintTokenCache returns a notFoundHint-wrapped ErrNotFound on
+// Lookup, mirroring what storage.notFoundHintCache produces in
+// production when ~/.databricks/token-cache.json has entries and the
+// resolver picked secure mode by default. Used by TestToken_loadToken
+// to verify that auth token surfaces the upgrade-specific hint instead
+// of dropping it for the SDK-compat constant string.
+type upgradeHintTokenCache struct{}
+
+func (upgradeHintTokenCache) Store(string, *oauth2.Token) error { return nil }
+func (upgradeHintTokenCache) Lookup(string) (*oauth2.Token, error) {
+	return nil, storage.NewNotFoundHint(
+		"stored credentials from older CLI versions are no longer used; run `databricks auth login` to sign in again, or set DATABRICKS_AUTH_STORAGE=plaintext to keep using the file cache",
+	)
+}
+
+var _ cache.TokenCache = upgradeHintTokenCache{}
+
 type failOnCallTransport struct{}
 
 func (failOnCallTransport) RoundTrip(*http.Request) (*http.Response, error) {
@@ -408,6 +427,35 @@ func TestToken_loadToken(t *testing.T) {
 				"Try logging in again with `databricks auth login --host https://nonexistent.cloud.databricks.com` before retrying. " +
 				"If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new",
 		},
+		{
+			// Regression test: when notFoundHintCache wraps ErrNotFound
+			// with the upgrade copy (post-upgrade default-secure user
+			// with a populated token-cache.json), `auth token` must
+			// surface that hint instead of dropping it for the SDK-compat
+			// constant string. The combined message keeps the
+			// "OAuth is not configured for this host" substring older
+			// SDK versions look for and skips the generic "Try logging
+			// in again ... If this fails, please report this issue"
+			// trailer, which would mislead users into reporting expected
+			// post-upgrade behavior.
+			name: "ErrNotFound carrying upgrade hint surfaces it",
+			args: loadTokenArgs{
+				authArguments: &auth.AuthArguments{},
+				profileName:   "",
+				args:          []string{"nonexistent.cloud.databricks.com"},
+				tokenTimeout:  1 * time.Hour,
+				profiler:      profiler,
+				tokenCache:    upgradeHintTokenCache{},
+				persistentAuthOpts: []u2m.PersistentAuthOption{
+					u2m.WithTokenCache(upgradeHintTokenCache{}),
+					u2m.WithOAuthEndpointSupplier(&MockApiClient{}),
+				},
+			},
+			wantErr: "cache: databricks OAuth is not configured for this host. " +
+				"stored credentials from older CLI versions are no longer used; " +
+				"run `databricks auth login` to sign in again, " +
+				"or set DATABRICKS_AUTH_STORAGE=plaintext to keep using the file cache",
+		},
 		{
 			name: "errors with clear message for non-host non-profile positional arg",
 			args: loadTokenArgs{