acceptance: schema-driven invariant fuzzing for bundle configs

.github/workflows/push.yml

@@ -377,6 +377,78 @@ jobs:
         run: |
           go tool -modfile=tools/task/go.mod task test-sandbox
 
+  test-fuzz:
+    needs:
+      - cleanups
+
+    # Wide rotating seed window with drift checking on: too slow for every PR, so
+    # nightly only and not part of test-result. The committed acceptance test still
+    # checks the no-panic invariant on a small fixed window per PR.
+    if: ${{ github.event_name == 'schedule' }}
+    name: "task test-fuzz"
+    runs-on:
+      group: databricks-protected-runner-group-large
+      labels: linux-ubuntu-latest-large
+
+    defaults:
+      run:
+        shell: bash
+
+    permissions:
+      id-token: write
+      contents: read
+      # Failure-reporting step comments on the PR that introduced the failing commit.
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-fuzz
+
+      - name: Run tests
+        env:
+          FUZZ_SEED_COUNT: "25"
+        run: |
+          # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window
+          # non-overlapping, so CI explores new configs every run.
+          export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT ))
+          go tool -modfile=tools/task/go.mod task test-fuzz
+
+      # Not in test-result, so surface failures by commenting on the PR that
+      # introduced the commit under test.
+      - name: Report failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          body=$(cat <<EOF
+          The nightly schema fuzz invariant job (\`task test-fuzz\`) failed on commit \`$COMMIT\`.
+
+          Run: $RUN_URL
+
+          The failing seed is printed in the job log as \`reproduce with: ...\`.
+          Reproduce locally with:
+
+          \`\`\`
+          FUZZ_SEED_START=<seed> FUZZ_SEED_COUNT=1 task test-fuzz
+          \`\`\`
+          EOF
+          )
+
+          # The commit's pulls endpoint returns the merged PR that introduced it.
+          pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty')
+          if [ -n "$pr" ]; then
+            gh pr comment "$pr" --body "$body"
+          else
+            echo "No PR found for commit $COMMIT; skipping failure comment" >&2
+          fi
+
   # This job groups the result of all the above test jobs.
   # It is a required check, so it blocks auto-merge and the merge queue.
   #

.gitignore

@@ -67,6 +67,9 @@ dist/
 # Per-module golangci-lint TMPDIR (configured in Taskfile.yml)
 /.tmp/
 
+# Local fuzz driver scratch (see .fuzztmp/run_fuzz.sh)
+.fuzztmp/
+
 # Go workspace file
 go.work
 go.work.sum

Taskfile.yml

@@ -729,6 +729,56 @@ tasks:
           --packages ./acceptance/... \
           -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox"
 
+  test-fuzz:
+    desc: Run schema fuzz invariant tests (random configs, direct engine)
+    # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't
+    # see, so always run rather than no-op a repro or shifted nightly window.
+    cmds:
+      - |
+        # Wider window than the committed run, with drift checking on; a repro can
+        # narrow it via FUZZ_SEED_START/COUNT. Slow variants can't finish 200 seeds inside
+        # the per-variant Timeout, but the fuzz script's default FUZZ_TIME_BUDGET stops each
+        # variant cleanly (as many seeds as fit, then pass) rather than being force-killed.
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}"
+        export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}"
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz"
+
+  test-fuzz-cover:
+    desc: Run the schema fuzzer under coverage and report which CLI packages it exercises
+    # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars.
+    cmds:
+      - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/
+      - mkdir -p ./acceptance/build/cover-fuzz-merged/
+      - |
+        # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run,
+        # so every fuzzed `bundle` invocation drops counter files we can aggregate. Drift
+        # off (no FUZZ_CHECK_DRIFT) so the run exercises the full deploy/plan path for as
+        # many seeds as possible instead of stopping on the first fake-server drift.
+        export CLI_GOCOVERDIR=build/cover-fuzz
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}"
+        export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}"
+        unset FUZZ_CHECK_DRIFT || true
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true
+      - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/"
+      - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt
+      - |
+        echo "== total CLI coverage exercised by the fuzz corpus =="
+        go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}'
+        echo
+        echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) =="
+        go tool covdata percent -i=acceptance/build/cover-fuzz-merged \
+          | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%%  %s\n", p, $1}' \
+          | grep -E '  (bundle|cmd/bundle)/' \
+          | sort -n
+
   # --- Integration tests ---
 
   integration:

acceptance/bin/check_schema_types.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+Assert every `type` in the bundle schema is one gen_fuzz_config.py can generate, so a new
+libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzz loop.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from gen_fuzz_config import HANDLED_TYPES
+
+
+def collect_types(node, found):
+    if isinstance(node, dict):
+        t = node.get("type")
+        if isinstance(t, str):
+            found.add(t)
+        elif isinstance(t, list):
+            found.update(x for x in t if isinstance(x, str))
+        for v in node.values():
+            collect_types(v, found)
+    elif isinstance(node, list):
+        for v in node:
+            collect_types(v, found)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--schema", required=True)
+    args = parser.parse_args()
+
+    with open(args.schema) as f:
+        schema = json.load(f)
+
+    found = set()
+    collect_types(schema, found)
+
+    unhandled = sorted(found - HANDLED_TYPES)
+    if unhandled:
+        sys.exit(f"check_schema_types: gen_fuzz_config.py cannot generate schema types {unhandled}")
+
+
+if __name__ == "__main__":
+    main()

acceptance/bin/edit_fuzz_config.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""
+Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an
+in-place update, not a recreate. Used by the `update` invariant.
+
+Some resources classify `description` as immutable (recreate on change) in the direct
+engine spec (e.g. model_serving_endpoints); editing that replans as a recreate the update
+invariant would wrongly flag, so skip it and pick a mutable field (or report none).
+
+gen_fuzz_config.py emits one scalar per line as `key: <json>`, so a regex match suffices
+(no YAML dependency for the edit itself); the immutable set is read from resources.yml.
+
+  edit_fuzz_config.py PATH            edit in place; exit 1 if no editable field
+  edit_fuzz_config.py PATH --detect   exit 0 if an editable field exists, else 1
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+# Allow an optional "- " so a comment/description that is the first key of a list-item
+# dict still matches; the captured prefix is preserved verbatim on rewrite.
+FIELD_RE = re.compile(r'^(\s*(?:- )?)(comment|description): (".*")\s*$')
+
+# A resource type header directly under `resources:` (two-space indent, as emitted by
+# gen_fuzz_config.py and the curated templates).
+TYPE_RE = re.compile(r"^  ([\w-]+):\s*$")
+
+NEW_VALUE = '"fuzz_edited_value"'
+
+# resources.yml is the source of truth for field mutability. acceptance/bin is on PATH as
+# the real dir (not a copy), so __file__ resolves two levels below the repo root.
+RESOURCES_YML = Path(__file__).resolve().parents[2] / "bundle" / "direct" / "dresources" / "resources.yml"
+
+
+def immutable_fields():
+    """Map resource type -> set of fields that recreate on change (immutable).
+
+    Hand-rolled line parser over resources.yml's fixed two-space layout, avoiding a YAML
+    dependency the harness's Python lacks.
+    """
+    result = {}
+    current_type = None
+    in_recreate = False
+    for raw in RESOURCES_YML.read_text().splitlines():
+        if not raw.strip() or raw.lstrip().startswith("#"):
+            continue
+        indent = len(raw) - len(raw.lstrip())
+        stripped = raw.strip()
+        if indent == 2 and stripped.endswith(":"):
+            current_type = stripped[:-1]
+            in_recreate = False
+        elif indent == 4 and stripped.endswith(":"):
+            in_recreate = stripped == "recreate_on_changes:"
+        elif in_recreate and current_type:
+            m = re.match(r"-\s*field:\s*(\S+)", stripped)
+            if m:
+                result.setdefault(current_type, set()).add(m.group(1))
+    return result
+
+
+def find_line(lines, immutable):
+    current_type = None
+    in_resources = False
+    for i, line in enumerate(lines):
+        stripped = line.rstrip("\n")
+        # Track the enclosing resource type so an immutable comment/description is skipped.
+        if stripped == "resources:":
+            in_resources = True
+            current_type = None
+            continue
+        if in_resources:
+            m_type = TYPE_RE.match(stripped)
+            if m_type:
+                current_type = m_type.group(1)
+            elif stripped and not stripped[0].isspace():
+                in_resources = False
+                current_type = None
+        m = FIELD_RE.match(line)
+        if m and m.group(2) not in immutable.get(current_type, ()):
+            return i, m
+    return -1, None
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path")
+    parser.add_argument("--detect", action="store_true", help="only check, don't edit")
+    args = parser.parse_args()
+
+    with open(args.path) as f:
+        lines = f.readlines()
+
+    i, m = find_line(lines, immutable_fields())
+    if m is None:
+        sys.exit(1)
+    if args.detect:
+        return
+
+    prefix, key, _ = m.groups()
+    lines[i] = f"{prefix}{key}: {NEW_VALUE}\n"
+    with open(args.path, "w") as f:
+        f.writelines(lines)
+    sys.stderr.write(f"edited {key} at line {i + 1}\n")
+
+
+if __name__ == "__main__":
+    main()

acceptance/bin/edit_fuzz_config_check.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+Contract check for edit_fuzz_config's field selection (the harness diffs stdout; a
+non-zero exit marks a violation on stderr):
+
+- A comment/description that recreates on change for its resource type (per
+  resources.yml, e.g. model_serving_endpoints.description) is never chosen, so the
+  update invariant does not assert an in-place update the backend cannot perform.
+- A mutable comment/description is still chosen, even when an immutable one appears
+  first.
+- The immutable map actually loads and reflects resources.yml.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from edit_fuzz_config import find_line, immutable_fields
+
+IMMUTABLE_ONLY = """\
+resources:
+  model_serving_endpoints:
+    foo:
+      name: "test-endpoint"
+      description: "old"
+"""
+
+IMMUTABLE_THEN_MUTABLE = """\
+resources:
+  model_serving_endpoints:
+    foo:
+      description: "immutable"
+  jobs:
+    bar:
+      description: "mutable"
+"""
+
+MUTABLE_ONLY = """\
+resources:
+  jobs:
+    bar:
+      description: "mutable"
+"""
+
+
+def choose(text, immutable):
+    i, m = find_line(text.splitlines(keepends=True), immutable)
+    return None if m is None else i
+
+
+def main():
+    immutable = immutable_fields()
+    failed = False
+
+    # Guards the loader and the classification the update invariant relies on.
+    serving_immutable = "description" in immutable.get("model_serving_endpoints", set())
+    if not serving_immutable:
+        sys.stderr.write("expected model_serving_endpoints.description to be immutable in resources.yml\n")
+        failed = True
+
+    if choose(IMMUTABLE_ONLY, immutable) is not None:
+        sys.stderr.write("picked an immutable description\n")
+        failed = True
+
+    if choose(IMMUTABLE_THEN_MUTABLE, immutable) != 6:
+        sys.stderr.write("expected to skip the immutable description and pick the mutable one\n")
+        failed = True
+
+    if choose(MUTABLE_ONLY, immutable) != 3:
+        sys.stderr.write("expected to pick the mutable description\n")
+        failed = True
+
+    print(f"model_serving_endpoints.description immutable: {serving_immutable}")
+    print(f"IMMUTABLE_ONLY: {choose(IMMUTABLE_ONLY, immutable)}")
+    print(f"IMMUTABLE_THEN_MUTABLE: {choose(IMMUTABLE_THEN_MUTABLE, immutable)}")
+    print(f"MUTABLE_ONLY: {choose(MUTABLE_ONLY, immutable)}")
+
+    if failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()