bundle/direct/dstate: read-only feature-flag gate for direct deployment state

acceptance/bundle/state/feature_flags/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: "my job"

acceptance/bundle/state/feature_flags/output.txt

@@ -0,0 +1,22 @@
+
+=== a version-3 state recording a feature is rejected (this CLI records no features yet)
+>>> errcode [CLI] bundle plan
+Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information
+
+
+Exit code: 1
+
+=== a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2)
+>>> [CLI] bundle plan
+create jobs.my_job
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> gron.py .databricks/bundle/default/resources.json
+json.state_version = 3;

acceptance/bundle/state/feature_flags/resources.empty_features.json

@@ -0,0 +1,8 @@
+{
+ "state_version": 3,
+ "features": {},
+ "cli_version": "0.0.0-dev",
+ "lineage": "test-lineage",
+ "serial": 1,
+ "state": {}
+}

acceptance/bundle/state/feature_flags/resources.with_feature.json

@@ -0,0 +1,10 @@
+{
+ "state_version": 3,
+ "features": {
+  "future_feature": {}
+ },
+ "cli_version": "0.0.0-dev",
+ "lineage": "test-lineage",
+ "serial": 1,
+ "state": {}
+}

acceptance/bundle/state/feature_flags/script

@@ -0,0 +1,11 @@
+mkdir -p .databricks/bundle/default
+
+title "a version-3 state recording a feature is rejected (this CLI records no features yet)"
+cp resources.with_feature.json .databricks/bundle/default/resources.json
+trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features"
+
+title "a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2)"
+cp resources.empty_features.json .databricks/bundle/default/resources.json
+trace $CLI bundle plan | contains.py "Plan:"
+trace $CLI bundle deploy
+trace gron.py .databricks/bundle/default/resources.json | grep state_version

acceptance/bundle/state/feature_flags/test.toml

@@ -0,0 +1,4 @@
+Ignore = [".databricks"]
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/state/future_version/output.txt

@@ -1,3 +1,3 @@
-state version 999 is newer than supported version 2; upgrade the CLI
+state version 999 is newer than supported version 3; upgrade the CLI
 
 Exit code: 1

bundle/direct/dstate/migrate.go

@@ -3,6 +3,8 @@ package dstate
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
+	"strings"
 
 	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/libs/structs/structpath"
@@ -13,11 +15,29 @@ import (
 // migrateState runs all necessary migrations on the database.
 // It is called after loading state from disk.
 func migrateState(db *Database) error {
+	// featureStateVersion states carry a feature list this CLI does not yet write or
+	// understand (see the featureStateVersion doc comment). A featureStateVersion
+	// state with no features is equivalent to currentStateVersion, so accept it and
+	// return without running the migrations below, leaving the on-disk version at
+	// featureStateVersion rather than flipping it down. One that records any feature
+	// depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade.
+	if db.StateVersion == featureStateVersion {
+		if len(db.Features) == 0 {
+			return nil
+		}
+		features := make([]string, 0, len(db.Features))
+		for name := range db.Features {
+			features = append(features, name)
+		}
+		slices.Sort(features)
+		return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL)
+	}
+
 	if db.StateVersion == currentStateVersion {
 		return nil
 	}
-	if db.StateVersion > currentStateVersion {
-		return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion)
+	if db.StateVersion > supportedStateVersion {
+		return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, supportedStateVersion)
 	}
 
 	for version := db.StateVersion; version < currentStateVersion; version++ {

bundle/direct/dstate/state.go

@@ -21,12 +21,43 @@ import (
 )
 
 const (
+	// currentStateVersion is the schema version written for deployments that record
+	// no feature flags, and the version legacy states are migrated up to on load.
 	currentStateVersion = 2
 	initialBufferSize   = 64 * 1024
 	maxWalEntrySize     = 10 * 1024 * 1024
 	walSuffix           = ".wal"
+
+	// featureStateVersion is the schema version a future CLI will write once it
+	// records deployment state "feature flags" (see Header.Features). This CLI does
+	// not write it and records no features; it exists now only so this CLI reads
+	// such states correctly (see migrateState):
+	//   - featureStateVersion with no features  -> accept and leave the version as-is
+	//   - featureStateVersion with any feature   -> refuse, tell the user to upgrade
+	//
+	// A featureStateVersion state with no features is equivalent to
+	// currentStateVersion, but we deliberately do not flip the on-disk version down
+	// to currentStateVersion: a state written at featureStateVersion stays at
+	// featureStateVersion. This is forward-compat scaffolding so that a later release
+	// can start writing featureStateVersion + features without older CLIs (with this
+	// change) either mishandling a feature they lack or rejecting a featureless state
+	// outright. featureStateVersion is always 3.
+	featureStateVersion = 3
+
+	// supportedStateVersion is the highest schema version this CLI can read. It is
+	// normally equal to currentStateVersion — the version this CLI reads is the
+	// version it writes — and exceeds it only during a two-phase version bump like
+	// the current feature-flag scaffolding, where this CLI reads (but does not
+	// write) featureStateVersion. A state newer than this is rejected as too new.
+	supportedStateVersion = featureStateVersion
 )
 
+// featuresDocURL is the single documentation page describing deployment state
+// feature flags. It is shown when a state records a feature this CLI does not
+// support; it is a fixed link for all features. The #state-features anchor points
+// at the feature table; if it ever breaks, the user still lands on the page.
+const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features"
+
 // errStaleWAL is returned when the WAL serial is behind the expected serial.
 // The caller should delete the stale WAL and proceed normally.
 var errStaleWAL = errors.New("stale WAL")
@@ -46,6 +77,13 @@ type Header struct {
 	CLIVersion   string `json:"cli_version"`
 	Lineage      string `json:"lineage"`
 	Serial       int    `json:"serial"`
+
+	// Features maps each feature flag this state depends on to a (currently empty)
+	// value. This CLI writes no features; it only reads the field to detect a state
+	// that depends on features it lacks and refuse it (see migrateState). It is a
+	// map so a future CLI can attach per-feature data without reshaping the state.
+	// Empty/omitted for states that use no features.
+	Features map[string]struct{} `json:"features,omitempty"`
 }
 
 type Database struct {

bundle/direct/dstate/state_test.go

@@ -135,6 +135,38 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) {
 	mustFinalize(t, &recovered)
 }
 
+// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a
+// featureStateVersion state with no features is accepted as-is — the on-disk version
+// is left at featureStateVersion, not flipped down to currentStateVersion — and that
+// a featureStateVersion state recording any feature is refused. This is scaffolding
+// for the deferred version bump, special-cased to featureStateVersion only (see the
+// featureStateVersion doc comment).
+//
+// When the baseline is actually bumped to featureStateVersion, this special case must
+// go away. This test is the forcing function: it fails once featureStateVersion is
+// removed, making the author decide what the post-bump behavior should be.
+func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) {
+	// The special case applies to featureStateVersion (3) only.
+	require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case")
+	require.Equal(t, 3, featureStateVersion)
+
+	empty := &Database{Header: Header{StateVersion: featureStateVersion}}
+	require.NoError(t, migrateState(empty))
+	assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2")
+
+	// v3 that records a feature is refused: this CLI does not understand features.
+	withFeature := &Database{Header: Header{
+		StateVersion: featureStateVersion,
+		Features:     map[string]struct{}{"future_feature": {}},
+	}}
+	err := migrateState(withFeature)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "requires features this CLI does not support")
+	assert.Contains(t, err.Error(), "future_feature")
+	assert.Contains(t, err.Error(), "upgrade to the latest CLI version")
+	assert.Contains(t, err.Error(), featuresDocURL)
+}
+
 func TestDeleteState(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "state.json")