diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index eb0ea35e05b..42e98676a85 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -377,6 +377,78 @@ jobs: run: | go tool -modfile=tools/task/go.mod task test-sandbox + test-fuzz: + needs: + - cleanups + + # Wide rotating seed window with drift checking on: too slow for every PR, so + # nightly only and not part of test-result. The committed acceptance test still + # checks the no-panic invariant on a small fixed window per PR. + if: ${{ github.event_name == 'schedule' }} + name: "task test-fuzz" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + # Failure-reporting step comments on the PR that introduced the failing commit. + pull-requests: write + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fuzz + + - name: Run tests + env: + FUZZ_SEED_COUNT: "25" + run: | + # start = monotonic GITHUB_RUN_NUMBER * COUNT keeps each nightly window + # non-overlapping, so CI explores new configs every run. + export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) + go tool -modfile=tools/task/go.mod task test-fuzz + + # Not in test-result, so surface failures by commenting on the PR that + # introduced the commit under test. + - name: Report failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT: ${{ github.sha }} + run: | + body=$(cat < FUZZ_SEED_COUNT=1 task test-fuzz + \`\`\` + EOF + ) + + # The commit's pulls endpoint returns the merged PR that introduced it. + pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty') + if [ -n "$pr" ]; then + gh pr comment "$pr" --body "$body" + else + echo "No PR found for commit $COMMIT; skipping failure comment" >&2 + fi + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # diff --git a/.gitignore b/.gitignore index 543b638a537..63a15332d02 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ dist/ # Per-module golangci-lint TMPDIR (configured in Taskfile.yml) /.tmp/ +# Local fuzz driver scratch (see .fuzztmp/run_fuzz.sh) +.fuzztmp/ + # Go workspace file go.work go.work.sum diff --git a/Taskfile.yml b/Taskfile.yml index 4384de278c4..b0bac7f74bd 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -729,6 +729,56 @@ tasks: --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" + test-fuzz: + desc: Run schema fuzz invariant tests (random configs, direct engine) + # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't + # see, so always run rather than no-op a repro or shifted nightly window. + cmds: + - | + # Wider window than the committed run, with drift checking on; a repro can + # narrow it via FUZZ_SEED_START/COUNT. Slow variants can't finish 200 seeds inside + # the per-variant Timeout, but the fuzz script's default FUZZ_TIME_BUDGET stops each + # variant cleanly (as many seeds as fit, then pass) rather than being force-killed. + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" + export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz" + + test-fuzz-cover: + desc: Run the schema fuzzer under coverage and report which CLI packages it exercises + # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars. + cmds: + - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/ + - mkdir -p ./acceptance/build/cover-fuzz-merged/ + - | + # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, + # so every fuzzed `bundle` invocation drops counter files we can aggregate. Drift + # off (no FUZZ_CHECK_DRIFT) so the run exercises the full deploy/plan path for as + # many seeds as possible instead of stopping on the first fake-server drift. + export CLI_GOCOVERDIR=build/cover-fuzz + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}" + export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" + unset FUZZ_CHECK_DRIFT || true + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true + - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/" + - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt + - | + echo "== total CLI coverage exercised by the fuzz corpus ==" + go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}' + echo + echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) ==" + go tool covdata percent -i=acceptance/build/cover-fuzz-merged \ + | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%% %s\n", p, $1}' \ + | grep -E ' (bundle|cmd/bundle)/' \ + | sort -n + # --- Integration tests --- integration: diff --git a/acceptance/bin/check_schema_types.py b/acceptance/bin/check_schema_types.py new file mode 100755 index 00000000000..6f9e72c0f8d --- /dev/null +++ b/acceptance/bin/check_schema_types.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Assert every `type` in the bundle schema is one gen_fuzz_config.py can generate, so a new +libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzz loop. +""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import HANDLED_TYPES + + +def collect_types(node, found): + if isinstance(node, dict): + t = node.get("type") + if isinstance(t, str): + found.add(t) + elif isinstance(t, list): + found.update(x for x in t if isinstance(x, str)) + for v in node.values(): + collect_types(v, found) + elif isinstance(node, list): + for v in node: + collect_types(v, found) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--schema", required=True) + args = parser.parse_args() + + with open(args.schema) as f: + schema = json.load(f) + + found = set() + collect_types(schema, found) + + unhandled = sorted(found - HANDLED_TYPES) + if unhandled: + sys.exit(f"check_schema_types: gen_fuzz_config.py cannot generate schema types {unhandled}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/edit_fuzz_config.py b/acceptance/bin/edit_fuzz_config.py new file mode 100755 index 00000000000..a71cf82771f --- /dev/null +++ b/acceptance/bin/edit_fuzz_config.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Edit a `comment`/`description` scalar in a generated databricks.yml so a redeploy is an +in-place update, not a recreate. Used by the `update` invariant. + +Some resources classify `description` as immutable (recreate on change) in the direct +engine spec (e.g. model_serving_endpoints); editing that replans as a recreate the update +invariant would wrongly flag, so skip it and pick a mutable field (or report none). + +gen_fuzz_config.py emits one scalar per line as `key: `, so a regex match suffices +(no YAML dependency for the edit itself); the immutable set is read from resources.yml. + + edit_fuzz_config.py PATH edit in place; exit 1 if no editable field + edit_fuzz_config.py PATH --detect exit 0 if an editable field exists, else 1 +""" + +import argparse +import re +import sys +from pathlib import Path + +# Allow an optional "- " so a comment/description that is the first key of a list-item +# dict still matches; the captured prefix is preserved verbatim on rewrite. +FIELD_RE = re.compile(r'^(\s*(?:- )?)(comment|description): (".*")\s*$') + +# A resource type header directly under `resources:` (two-space indent, as emitted by +# gen_fuzz_config.py and the curated templates). +TYPE_RE = re.compile(r"^ ([\w-]+):\s*$") + +NEW_VALUE = '"fuzz_edited_value"' + +# resources.yml is the source of truth for field mutability. acceptance/bin is on PATH as +# the real dir (not a copy), so __file__ resolves two levels below the repo root. +RESOURCES_YML = Path(__file__).resolve().parents[2] / "bundle" / "direct" / "dresources" / "resources.yml" + + +def immutable_fields(): + """Map resource type -> set of fields that recreate on change (immutable). + + Hand-rolled line parser over resources.yml's fixed two-space layout, avoiding a YAML + dependency the harness's Python lacks. + """ + result = {} + current_type = None + in_recreate = False + for raw in RESOURCES_YML.read_text().splitlines(): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip()) + stripped = raw.strip() + if indent == 2 and stripped.endswith(":"): + current_type = stripped[:-1] + in_recreate = False + elif indent == 4 and stripped.endswith(":"): + in_recreate = stripped == "recreate_on_changes:" + elif in_recreate and current_type: + m = re.match(r"-\s*field:\s*(\S+)", stripped) + if m: + result.setdefault(current_type, set()).add(m.group(1)) + return result + + +def find_line(lines, immutable): + current_type = None + in_resources = False + for i, line in enumerate(lines): + stripped = line.rstrip("\n") + # Track the enclosing resource type so an immutable comment/description is skipped. + if stripped == "resources:": + in_resources = True + current_type = None + continue + if in_resources: + m_type = TYPE_RE.match(stripped) + if m_type: + current_type = m_type.group(1) + elif stripped and not stripped[0].isspace(): + in_resources = False + current_type = None + m = FIELD_RE.match(line) + if m and m.group(2) not in immutable.get(current_type, ()): + return i, m + return -1, None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("path") + parser.add_argument("--detect", action="store_true", help="only check, don't edit") + args = parser.parse_args() + + with open(args.path) as f: + lines = f.readlines() + + i, m = find_line(lines, immutable_fields()) + if m is None: + sys.exit(1) + if args.detect: + return + + prefix, key, _ = m.groups() + lines[i] = f"{prefix}{key}: {NEW_VALUE}\n" + with open(args.path, "w") as f: + f.writelines(lines) + sys.stderr.write(f"edited {key} at line {i + 1}\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/edit_fuzz_config_check.py b/acceptance/bin/edit_fuzz_config_check.py new file mode 100755 index 00000000000..b8160dbe627 --- /dev/null +++ b/acceptance/bin/edit_fuzz_config_check.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Contract check for edit_fuzz_config's field selection (the harness diffs stdout; a +non-zero exit marks a violation on stderr): + +- A comment/description that recreates on change for its resource type (per + resources.yml, e.g. model_serving_endpoints.description) is never chosen, so the + update invariant does not assert an in-place update the backend cannot perform. +- A mutable comment/description is still chosen, even when an immutable one appears + first. +- The immutable map actually loads and reflects resources.yml. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from edit_fuzz_config import find_line, immutable_fields + +IMMUTABLE_ONLY = """\ +resources: + model_serving_endpoints: + foo: + name: "test-endpoint" + description: "old" +""" + +IMMUTABLE_THEN_MUTABLE = """\ +resources: + model_serving_endpoints: + foo: + description: "immutable" + jobs: + bar: + description: "mutable" +""" + +MUTABLE_ONLY = """\ +resources: + jobs: + bar: + description: "mutable" +""" + + +def choose(text, immutable): + i, m = find_line(text.splitlines(keepends=True), immutable) + return None if m is None else i + + +def main(): + immutable = immutable_fields() + failed = False + + # Guards the loader and the classification the update invariant relies on. + serving_immutable = "description" in immutable.get("model_serving_endpoints", set()) + if not serving_immutable: + sys.stderr.write("expected model_serving_endpoints.description to be immutable in resources.yml\n") + failed = True + + if choose(IMMUTABLE_ONLY, immutable) is not None: + sys.stderr.write("picked an immutable description\n") + failed = True + + if choose(IMMUTABLE_THEN_MUTABLE, immutable) != 6: + sys.stderr.write("expected to skip the immutable description and pick the mutable one\n") + failed = True + + if choose(MUTABLE_ONLY, immutable) != 3: + sys.stderr.write("expected to pick the mutable description\n") + failed = True + + print(f"model_serving_endpoints.description immutable: {serving_immutable}") + print(f"IMMUTABLE_ONLY: {choose(IMMUTABLE_ONLY, immutable)}") + print(f"IMMUTABLE_THEN_MUTABLE: {choose(IMMUTABLE_THEN_MUTABLE, immutable)}") + print(f"MUTABLE_ONLY: {choose(MUTABLE_ONLY, immutable)}") + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/fuzz_gen_config.py b/acceptance/bin/fuzz_gen_config.py new file mode 100755 index 00000000000..a228bb0c82a --- /dev/null +++ b/acceptance/bin/fuzz_gen_config.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from +FUZZ_MODE so the invariant target scripts don't each duplicate the branch: + + generate (default) - build a config from scratch by walking `bundle schema` + (gen_fuzz_config.py). + mutate - start from a curated invariant config and perturb it + (mutate_fuzz_config.py). + +Reads its inputs from the environment the invariant scripts already export: FUZZ_SEED, +FUZZ_SCHEMA, UNIQUE_NAME, FUZZ_RESOURCES, FUZZ_RESOURCE_COUNT, TESTDIR. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from gen_fuzz_config import gen_config, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +# Curated single-resource configs that deploy standalone against the fake server (only +# $UNIQUE_NAME, no init script). All are also in the invariant INPUT_CONFIG matrix, so +# they stay deploy-verified. The seed selects one; mutate_fuzz_config perturbs it. +MUTATE_BASES = [ + "catalog", + "external_location", + "job", + "model", + "model_serving_endpoint", + "pipeline", + "registered_model", + "schema", + "secret_scope", + "sql_warehouse", + "volume", +] + + +def generate(seed): + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + allowed = {r.strip() for r in os.environ.get("FUZZ_RESOURCES", "").split(",") if r.strip()} + unique = f"{os.environ['UNIQUE_NAME']}-{seed}" + count = int(os.environ.get("FUZZ_RESOURCE_COUNT", "1")) + return to_yaml(gen_config(schema, seed, unique, allowed, count)) + + +def mutate_base(seed): + name = MUTATE_BASES[seed % len(MUTATE_BASES)] + path = os.path.join(os.environ["TESTDIR"], "..", "configs", name + ".yml.tmpl") + with open(path) as f: + rendered = substitute_variables(f.read()) + config = load_yaml(rendered) + return to_yaml(mutate(config, seed)) + + +def main(): + seed = int(os.environ["FUZZ_SEED"]) + mode = os.environ.get("FUZZ_MODE", "generate") + if mode == "generate": + sys.stdout.write(generate(seed)) + elif mode == "mutate": + sys.stdout.write(mutate_base(seed)) + else: + sys.exit(f"fuzz_gen_config: unknown FUZZ_MODE {mode!r}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py new file mode 100755 index 00000000000..9aca9166cce --- /dev/null +++ b/acceptance/bin/gen_fuzz_config.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +""" +Generate a random bundle config from the bundle JSON schema. + +Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf +branches) and emits one or more random resources as databricks.yml, seeded by --seed. +With --resource-count > 1 it also links resources with ${resources.*} references (each +resource referencing an earlier one) so the interpolation and deploy-ordering machinery is +exercised. Free-form scalars are occasionally replaced with dangerous / near-range-end +values (DANGEROUS_STRINGS, DANGEROUS_INTS) to probe the CLI's input handling. Feeds the +invariant tests; the harness filters out configs the CLI rejects, so output may be +structurally-random but sometimes invalid. +""" + +import argparse +import json +import os +import random +import re +import sys + +# The schema is recursive (e.g. task -> for_each_task -> task); cap the walk. +MAX_DEPTH = 6 + +# The ${...} interpolation branch the schema wraps every field in (see +# bundle/internal/schema/main.go addInterpolationPatterns); we emit concrete values. +INTERPOLATION_MARKER = "\\$\\{" + +# Types the generator can produce; keep in sync with libs/jsonschema.Type. +SCALAR_TYPES = {"boolean", "integer", "number", "string"} +HANDLED_TYPES = SCALAR_TYPES | {"object", "array"} + +# Cross-resource references must resolve to objects that exist on every +# workspace (the fake test server and real UC alike). "main"/"default" are the +# standard seeded catalog/schema; these mirror acceptance/bundle/invariant/configs. +# Without pinning, the generator emits random names that the fake server accepts +# but real UC rejects (e.g. CATALOG_DOES_NOT_EXIST), so the config is dropped at +# deploy and never exercises the invariant. +DEFAULT_CATALOG = "main" +DEFAULT_SCHEMA = "default" + +# "account users" is a group present on every workspace, plus one privilege UC +# accepts for each grant-bearing securable type (from the curated configs). Real +# UC rejects an unknown principal or a privilege that doesn't apply to the +# securable, so a random grant would deploy on the fake server yet fail on cloud. +DEFAULT_PRINCIPAL = "account users" +GRANT_PRIVILEGE = { + "catalogs": "USE_CATALOG", + "schemas": "USE_SCHEMA", + "volumes": "READ_VOLUME", + "registered_models": "EXECUTE", + "external_locations": "READ_FILES", + "vector_search_indexes": "SELECT", +} + +# Permissions blocks cannot be variable references; each entry needs a concrete +# principal and a level valid for the resource type (see invariant configs). +DEFAULT_PERMISSION_GROUP = "users" +PERMISSION_LEVEL = { + "alerts": "CAN_MANAGE", + "apps": "CAN_USE", + "clusters": "CAN_ATTACH_TO", + "dashboards": "CAN_READ", + "database_instances": "CAN_USE", + "experiments": "CAN_READ", + "genie_spaces": "CAN_READ", + "jobs": "CAN_VIEW", + "model_serving_endpoints": "CAN_VIEW", + "models": "CAN_READ", + "pipelines": "CAN_VIEW", + "postgres_projects": "CAN_USE", + "secret_scopes": "READ", + "sql_warehouses": "CAN_VIEW", + "vector_search_endpoints": "CAN_USE", +} + +# Fields the bundle schema still lists but the user never sets (backend output / +# computed). Emitting them causes false drift after terraform→direct migrate. +# Keep in sync with bundle/direct/dresources/resources.yml output_only and +# backend_defaults where the field is not user-writable. +SKIP_PROPERTY_NAMES = frozenset( + { + "browse_only", + "created_at", + "created_by", + "creator_name", + # An etag is a read value the backend assigns; the CLI rejects one set in + # bundle config (e.g. "genie space ... has an etag set. Etags must not be set"). + "etag", + "full_name", + "metastore_id", + "owner", + "storage_location", + "updated_at", + "updated_by", + } +) + +# Resource types whose schema omits required[] (or whose required[] can't be honored +# in bundle YAML) but which need these fields to deploy. See RESOURCE_FIELD_ALLOWLIST +# and the *_BY_RESOURCE tables below for the values these fields take. +RESOURCE_REQUIRED_FIELDS = { + "registered_models": frozenset({"catalog_name", "name", "schema_name"}), + "dashboards": frozenset({"display_name", "file_path", "warehouse_id"}), + "alerts": frozenset({"display_name", "file_path", "warehouse_id"}), + "apps": frozenset({"name", "source_code_path"}), + "genie_spaces": frozenset({"serialized_space", "title", "warehouse_id"}), +} + +# Fields to drop for a specific resource type because they conflict with the field +# set we do emit. Dashboards and Genie spaces take their body from file_path XOR an +# inline serialized_* field, so emitting both is rejected ("both ... are set"). +RESOURCE_SKIP_FIELDS = { + "dashboards": frozenset({"serialized_dashboard"}), + "genie_spaces": frozenset({"file_path"}), + "apps": frozenset({"git_repository", "git_source"}), +} + +# Resource types where only a fixed field set is allowed in bundle YAML. Alerts read +# their spec from the .dbalert.json referenced by file_path; the CLI rejects any other +# field (see bundle/config/mutator/load_dbalert_files.go allowedInYAML). +RESOURCE_FIELD_ALLOWLIST = { + "alerts": frozenset({"display_name", "file_path", "lifecycle", "permissions", "warehouse_id"}), +} + +# file_path points at a serialized-body fixture copied into every seed dir from +# acceptance/bundle/invariant/data (see fuzz/script). The extension selects the parser. +FILE_PATH_BY_RESOURCE = { + "dashboards": "./dashboard.lvdash.json", + "alerts": "./alert.dbalert.json", +} + +# A local directory holding app source, also copied in from data/. +APP_SOURCE_CODE_PATH = "./app" + +# An absolute workspace path is treated as already-remote, skipping the local-notebook +# existence/extension check a bare token would fail. +NOTEBOOK_PATH = "/Shared/notebook" + +# Fields declared as string in the schema but parsed as google.protobuf.Duration at +# config load (e.g. suspend_timeout_duration, ttl); a bare token fails to parse. +DURATION_VALUE = "3600s" + +# Dangerous / near-range-end probes injected into free-form scalars: empty and +# whitespace-only strings, an over-long string, embedded newlines/tabs, non-ASCII, quotes, +# a dangling ${...} reference, a path-traversal string, and int32/int64 boundaries. The CLI +# must reject or round-trip these without panicking; mutate_fuzz_config.py reuses both lists. +DANGEROUS_STRINGS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + "quote\"and'apostrophe", + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", +] +DANGEROUS_INTS = [ + 2**31, + -(2**31), + 2**63 - 1, + -1, +] + +# Only inject a dangerous value some of the time: a fuzzed field mostly keeps a plausible +# value so the config still deploys and exercises the invariant, not just the reject path. +DANGEROUS_PROB = 0.15 + + +class Generator: + def __init__(self, schema, rng, unique): + self.root = schema + self.rng = rng + self.unique = unique + # Set to the top-level resource type before generating its element, so + # grants can pick a privilege valid for that securable. + self.rtype = None + + def resolve(self, schema): + # Follow $ref chains, e.g. "#/$defs/github.com/.../resources.Job", nested + # under $defs by "/"-separated path segments. + while isinstance(schema, dict) and "$ref" in schema: + cur = self.root["$defs"] + for part in schema["$ref"].split("/")[2:]: + cur = cur[part] + schema = cur + return schema + + def is_interpolation(self, branch): + return branch.get("type") == "string" and INTERPOLATION_MARKER in branch.get("pattern", "") + + def choose_branch(self, branches): + # Prefer concrete branches over the ${...} alternatives. + concrete = [b for b in branches if not self.is_interpolation(b)] + return self.rng.choice(concrete or branches) + + def field_behaviors(self, schema): + if not isinstance(schema, dict): + return [] + resolved = self.resolve(schema) + behaviors = list(schema.get("x-databricks-field-behaviors", [])) + if resolved is not schema: + behaviors.extend(resolved.get("x-databricks-field-behaviors", [])) + return behaviors + + def should_skip_property(self, prop_name, prop_schema): + if prop_name in SKIP_PROPERTY_NAMES: + return True + if prop_name in RESOURCE_SKIP_FIELDS.get(self.rtype, ()): + return True + resolved = self.resolve(prop_schema) + if "OUTPUT_ONLY" in self.field_behaviors(prop_schema): + return True + if resolved.get("readOnly"): + return True + return False + + def gen(self, schema, depth, name=""): + # A Genie space body is a free-form interface{}; the backend rejects unknown + # keys, so emit the minimal accepted body instead of a random object. + if name == "serialized_space": + return {"version": 1} + + schema = self.resolve(schema) + if not isinstance(schema, dict) or not schema: + return self.gen_scalar({"type": "string"}, name) + + if name == "grants": + return self.gen_grants() + if name == "permissions": + return self.gen_permissions() + + if "const" in schema: + return schema["const"] + if schema.get("enum"): + return self.rng.choice(schema["enum"]) + + for key in ("oneOf", "anyOf"): + if schema.get(key): + return self.gen(self.choose_branch(schema[key]), depth, name) + + t = schema.get("type") + if t == "object" or "properties" in schema or self.is_map(schema): + return self.gen_object(schema, depth) + if t == "array": + return self.gen_array(schema, depth, name) + return self.gen_scalar(schema, name) + + def is_map(self, schema): + return isinstance(schema.get("additionalProperties"), dict) and not schema.get("properties") + + def gen_object(self, schema, depth): + props = schema.get("properties", {}) + required = set(schema.get("required", [])) + allowlist = None + if depth == 0 and self.rtype: + required |= RESOURCE_REQUIRED_FIELDS.get(self.rtype, set()) + allowlist = RESOURCE_FIELD_ALLOWLIST.get(self.rtype) + result = {} + + for prop_name, prop_schema in props.items(): + # A restricted resource (e.g. alerts) rejects any field outside its + # allow-list, even a schema-required one supplied via the file instead. + if allowlist is not None and prop_name not in allowlist: + continue + if self.should_skip_property(prop_name, prop_schema): + continue + # Always emit required fields; emit optional ones less often as we go + # deeper to keep configs from exploding. + keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) + if not keep: + continue + value = self.gen(prop_schema, depth + 1, prop_name) + if value is not None: + result[prop_name] = value + + # Map type (additionalProperties, no fixed properties): synthesize a few + # random keys, e.g. resources. or string maps like tags. + if self.is_map(schema): + for _ in range(self.rng.randint(1, 2)): + key = self.token() + result[key] = self.gen(schema["additionalProperties"], depth + 1, key) + + return result + + def gen_array(self, schema, depth, name): + items = schema.get("items") + if not items or depth >= MAX_DEPTH: + return [] + return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] + + def gen_grants(self): + # One known-good grant for the current securable. Skip grants for a type + # we have no valid privilege for, rather than emit one real UC rejects. + privilege = GRANT_PRIVILEGE.get(self.rtype) + if privilege is None: + return [] + return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] + + def gen_permissions(self): + # One known-good permission for the current resource. Skip types we have + # no valid level for, rather than emit a random principal or ${...} ref. + level = PERMISSION_LEVEL.get(self.rtype) + if level is None: + return [] + return [{"level": level, "group_name": DEFAULT_PERMISSION_GROUP}] + + def gen_scalar(self, schema, name): + t = schema.get("type") + if t == "boolean": + # destroy_recreate invariant requires destroy to succeed. + if name == "prevent_destroy": + return False + return self.rng.choice([True, False]) + if t == "integer": + # The field is in hours, but UC validates it as a window of 0 or 7-30 + # days; only 0 or 168-720 (hours) are accepted. + if name == "custom_max_retention_hours": + return self.rng.choice([0, self.rng.randint(168, 720)]) + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_INTS) + return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) + if t == "number": + return round(self.rng.uniform(0, 1000), 2) + # Fail loud on an unknown type; a missing type is "any" and falls through to string. + if t is not None and t not in SCALAR_TYPES: + sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") + # string (default) + # Pin cross-resource references and typed-string fields to values the backend + # accepts; a random token fails format/existence validation and drops the config. + if name == "catalog_name": + return DEFAULT_CATALOG + if name == "schema_name": + return DEFAULT_SCHEMA + if name == "warehouse_id": + return os.environ.get("TEST_DEFAULT_WAREHOUSE_ID", "") + if name == "notebook_path": + return NOTEBOOK_PATH + if name == "source_code_path": + return APP_SOURCE_CODE_PATH + if name == "file_path": + return FILE_PATH_BY_RESOURCE.get(self.rtype, self.token()) + if name.endswith("_duration") or name == "ttl": + return DURATION_VALUE + if name == "name" and self.rtype == "vector_search_indexes": + # UC requires the full three-level catalog.schema.table name, and each + # part accepts only alphanumerics and underscores. + table = re.sub(r"[^0-9a-zA-Z_]", "_", f"fuzz_index_{self.unique}") + return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" + if name in ("name", "display_name"): + return f"fuzz-{name}-{self.unique}" + # A free-form string with no pinned meaning (e.g. description, comment, tag value): + # probe dangerous / near-range-end input here, where a rejected or normalized value + # doesn't just fail the field-format check a pinned field above guards against. + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_STRINGS) + return self.token() + + def token(self): + return "fuzz_" + "".join(self.rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def resource_types(schema, gen): + # resources is oneOf[{ object with one property per resource type }]. + resources = gen.resolve(schema["properties"]["resources"]) + obj = next(b for b in resources["oneOf"] if b.get("type") == "object") + return obj["properties"] + + +def gen_resource(schema, gen, types, candidates, seed, unique, index): + rtype = gen.rng.choice(sorted(candidates)) + + # Each resource type is a map ref; the element schema is the object branch's + # additionalProperties. + map_schema = gen.resolve(types[rtype]) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + element = obj["additionalProperties"] + + # The first resource keeps the bare key/name so a seed produces the same first + # resource regardless of --resource-count; later resources are index-suffixed to + # stay unique within the config. + if index == 0: + key = f"fuzz_{rtype}_{seed}" + gen.unique = unique + else: + key = f"fuzz_{rtype}_{seed}_{index}" + gen.unique = f"{unique}-{index}" + gen.rtype = rtype + instance = gen.gen(element, 0) + return rtype, key, instance, gen.resolve(element) + + +def object_properties(gen, schema): + # The resource element is oneOf[object, ${...} string]; return the object + # branch's properties, matching the branch gen() picks to build the instance. + schema = gen.resolve(schema) + if "properties" in schema: + return schema["properties"] + for key in ("oneOf", "anyOf"): + for branch in schema.get(key, []): + resolved = gen.resolve(branch) + if "properties" in resolved: + return resolved["properties"] + return {} + + +def cross_ref_field(gen, element): + # A free-text scalar safe to overwrite with a reference; both names cover most + # resource types (jobs use "description", UC resources use "comment"). + props = object_properties(gen, element) + for field in ("description", "comment"): + if field in props: + return field + return None + + +def target_ref_field(instance): + # Reference the target's identity field: a string, so the type stays compatible + # with the description/comment field it lands in, and an input (not an output like + # ".id") so it resolves for every resource type and converges without drift. + # Output-field references are covered by the curated cross-ref configs. + for field in ("name", "display_name"): + if isinstance(instance.get(field), str): + return field + return None + + +def inject_cross_ref(gen, records): + # Link resources so deploy has to order them and resolve the references. A + # record may only reference an earlier one, so the reference graph stays + # acyclic: deploy must topologically order resources, and a cycle can't be + # ordered (the config would be rejected instead of exercising the invariant). + if len(records) < 2: + return + for i, source in enumerate(records): + if not source["ref_field"]: + continue + targets = [t for t in records[:i] if target_ref_field(t["instance"])] + if not targets: + continue + target = gen.rng.choice(targets) + field = target_ref_field(target["instance"]) + source["instance"][source["ref_field"]] = f"${{resources.{target['rtype']}.{target['key']}.{field}}}" + + +def gen_config(schema, seed, unique, allowed, resource_count=1): + if resource_count < 1: + sys.exit(f"gen_fuzz_config: --resource-count must be >= 1, got {resource_count}") + + gen = Generator(schema, random.Random(seed), unique) + + types = resource_types(schema, gen) + candidates = [t for t in types if not allowed or t in allowed] + if not candidates: + sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") + + records = [] + for index in range(resource_count): + rtype, key, instance, element = gen_resource(schema, gen, types, candidates, seed, unique, index) + records.append({"rtype": rtype, "key": key, "instance": instance, "ref_field": cross_ref_field(gen, element)}) + + inject_cross_ref(gen, records) + + resources = {} + for record in records: + resources.setdefault(record["rtype"], {})[record["key"]] = record["instance"] + + return { + "bundle": {"name": f"fuzz-{unique}"}, + "resources": resources, + } + + +def to_yaml(obj, indent=0, list_item=False): + pad = " " * indent + if isinstance(obj, dict): + if not obj: + return f"{pad}{{}}\n" if not list_item else f"{pad}- {{}}\n" + out = "" + first = True + for k, v in obj.items(): + prefix = pad + "- " if list_item and first else (pad + " " if list_item else pad) + child_indent = indent + 2 if list_item else indent + 1 + if isinstance(v, (dict, list)) and v: + out += f"{prefix}{k}:\n" + to_yaml(v, child_indent) + else: + out += f"{prefix}{k}: {dump_scalar(v)}\n" + first = False + return out + if isinstance(obj, list): + if not obj: + return f"{pad}[]\n" + out = "" + for item in obj: + if isinstance(item, (dict, list)): + out += to_yaml(item, indent, list_item=True) + else: + out += f"{pad}- {dump_scalar(item)}\n" + return out + return f"{pad}{dump_scalar(obj)}\n" + + +def dump_scalar(v): + # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default would escape an + # astral char (e.g. the 🚀 probe) into a UTF-16 surrogate pair (\ud83d\ude80), which + # YAML's parser rejects as an "invalid Unicode character escape code" -- so the config + # dies at parse time and never reaches bundle logic. A literal UTF-8 scalar is valid + # YAML and exercises the CLI's actual unicode handling instead. Control chars (\n, \t) + # are still escaped by json.dumps regardless, which YAML accepts. + return json.dumps(v, ensure_ascii=False) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--schema", required=True, help="Path to bundle JSON schema") + parser.add_argument("--seed", type=int, required=True, help="RNG seed (for reproducibility)") + parser.add_argument("--unique", default="local", help="Unique suffix for resource names") + parser.add_argument( + "--resources", + default="", + help="Comma-separated allow-list of resource types (default: all)", + ) + parser.add_argument( + "--resource-count", + type=int, + default=1, + help="Number of resources to emit (default: 1)", + ) + args = parser.parse_args() + + with open(args.schema) as f: + schema = json.load(f) + + allowed = {r.strip() for r in args.resources.split(",") if r.strip()} + config = gen_config(schema, args.seed, args.unique, allowed, args.resource_count) + sys.stdout.write(to_yaml(config)) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py new file mode 100755 index 00000000000..8649f4c90b5 --- /dev/null +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as +`key: `. edit_fuzz_config.py relies on this to edit a field by regex, not a YAML +parser. Prints each case's YAML (diffed by the harness) and exits non-zero on a violation. +""" + +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from edit_fuzz_config import FIELD_RE +from gen_fuzz_config import DANGEROUS_STRINGS, SKIP_PROPERTY_NAMES, gen_config, to_yaml + +# Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, empty containers. +CASES = [ + {"comment": "value: with a colon", "description": 'quote " and : colon'}, + {"resources": {"jobs": {"j": {"name": "n", "tags": {"team": "jobs"}}}}}, + {"tasks": [{"description": "d", "timeout_seconds": 3600}, {"comment": "c"}]}, + {"nums": [0, 1, 2], "flag": True, "ratio": 1.5, "empty_map": {}, "empty_list": []}, +] + +HEADER = re.compile(r"[\w.\-]+:$") # non-empty container: `key:` +SCALAR = re.compile(r"[\w.\-]+: (.+)$") # `key: ` + + +def check_line(line): + rest = line.lstrip(" ") + rest = rest.removeprefix("- ") + if HEADER.fullmatch(rest): + return # container header; value is on following lines + m = SCALAR.fullmatch(rest) + if m: + json.loads(m.group(1)) # value must be single-line JSON + return + json.loads(rest) # bare list scalar: `- ` + + +def main(): + failed = False + for case in CASES: + text = to_yaml(case) + sys.stdout.write(text) + for line in text.splitlines(): + if not line.strip(): + continue + try: + check_line(line) + except ValueError: + sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") + failed = True + + # edit_fuzz_config's FIELD_RE must still match when the value contains a colon (CASES[0]). + if not any(FIELD_RE.match(line) for line in to_yaml(CASES[0]).splitlines()): + sys.stderr.write("FIELD_RE did not match a comment/description line\n") + failed = True + + # Generate mode now emits DANGEROUS_STRINGS into free-form fields; each must still + # serialize to a single `key: ` line so edit_fuzz_config can rewrite it in place. + for i, val in enumerate(DANGEROUS_STRINGS): + line = to_yaml({"description": val}).rstrip("\n") + if "\n" in line or not FIELD_RE.match(line): + sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line comment/description contract: {line!r}\n") + failed = True + + # Multi-resource configs merge types under resources., and one resource + # references another's name so the interpolation/ordering path is exercised. + def resource_type(field): + return { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": {"name": {"type": "string"}, field: {"type": "string"}}, + "required": ["name"], + }, + } + ] + } + + multi = gen_config( + { + "$defs": {}, + "properties": { + "resources": { + "oneOf": [ + { + "type": "object", + "properties": { + "jobs": resource_type("description"), + "volumes": resource_type("comment"), + }, + } + ] + } + }, + }, + seed=42, + unique="check", + allowed=set(), + resource_count=2, + ) + if len(multi["resources"]) < 1 or sum(len(v) for v in multi["resources"].values()) != 2: + sys.stderr.write("gen_config did not emit two resources\n") + failed = True + + values = [v for insts in multi["resources"].values() for inst in insts.values() for v in inst.values()] + if not any(isinstance(v, str) and v.startswith("${resources.") for v in values): + sys.stderr.write("gen_config did not inject a cross-resource reference\n") + failed = True + + schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../bundle/schema/jsonschema.json") + with open(schema_path) as f: + schema = json.load(f) + seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}, resource_count=1) + rm = seed24["resources"]["registered_models"]["fuzz_registered_models_24"] + if SKIP_PROPERTY_NAMES & set(rm): + sys.stderr.write( + f"seed 24 registered_models emitted output-only fields: {sorted(SKIP_PROPERTY_NAMES & set(rm))}\n" + ) + failed = True + for field in ("name", "catalog_name", "schema_name"): + if field not in rm: + sys.stderr.write(f"seed 24 registered_models missing {field}\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py new file mode 100755 index 00000000000..cd9c9336801 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Mutate a known-good bundle config by deleting and perturbing random fields. + +Complements gen_fuzz_config.py (generate-from-scratch via schema walk): instead of +building a config from the schema, this starts from a curated invariant config that +already deploys and applies a few seeded mutations (delete a field, replace a scalar +with a fuzz token, a boundary/dangerous value, or an empty container). It exercises the +CLI's handling of perturbed-but-realistic input, and reaches a much higher deploy rate +than the schema walk, since the base already resolves. + +Reads the base databricks.yml (already envsubst-rendered) from stdin, writes the mutated +config to stdout. --seed makes the mutation reproducible. + +The invariant harness only asserts no-panic on fuzzed configs (SKIP_DRIFT_CHECK), so a +mutation that makes the config invalid is fine: the CLI must reject it cleanly, not crash. +""" + +import argparse +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, to_yaml + +# Same near-range-end and dangerous-character probes the schema-walk generator injects into +# free-form scalars; here we drop them onto any field (see mutate_once). +DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS + + +def tokenize(text): + # (indent, content) per non-blank, non-comment line. Only full-line comments are + # stripped; the curated bases don't use trailing "#" in values. + out = [] + for raw in text.splitlines(): + stripped = raw.lstrip(" ") + if not stripped or stripped.startswith("#"): + continue + out.append((len(raw) - len(stripped), stripped.rstrip())) + return out + + +def scalar(text): + if text in ("", "null", "~"): + return None + if text == "true": + return True + if text == "false": + return False + try: + return int(text) + except ValueError: + pass + try: + return float(text) + except ValueError: + pass + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def parse_block(tokens, i, indent): + if i >= len(tokens): + return {}, i + first = tokens[i][1] + if first.startswith("- ") or first == "-": + return parse_seq(tokens, i, indent) + if ": " in first or first.endswith(":"): + return parse_map(tokens, i, indent) + # Bare scalar: the whole block is a single value (e.g. a list scalar item). + return scalar(first), i + 1 + + +def parse_map(tokens, i, indent): + result = {} + while i < len(tokens) and tokens[i][0] == indent: + content = tokens[i][1] + if content.startswith("- "): + break + if ": " in content: + key, _, rest = content.partition(": ") + result[key.strip()] = scalar(rest) + i += 1 + elif content.endswith(":"): + key = content[:-1].strip() + i += 1 + if i < len(tokens) and tokens[i][0] > indent: + value, i = parse_block(tokens, i, tokens[i][0]) + else: + value = None + result[key] = value + else: + break + return result, i + + +def parse_seq(tokens, i, indent): + result = [] + while i < len(tokens) and tokens[i][0] == indent and (tokens[i][1].startswith("- ") or tokens[i][1] == "-"): + after = tokens[i][1][2:] if tokens[i][1].startswith("- ") else "" + child_indent = indent + 2 + # The item is its own block: the inline remainder (re-indented to child_indent) + # plus any deeper continuation lines that belong to it. + item = [] + if after: + item.append((child_indent, after)) + i += 1 + while i < len(tokens) and tokens[i][0] >= child_indent: + item.append(tokens[i]) + i += 1 + result.append(parse_block(item, 0, child_indent)[0] if item else None) + return result, i + + +def load_yaml(text): + tokens = tokenize(text) + value, _ = parse_block(tokens, 0, 0) + return value + + +def collect(node, out): + # (container, key) for every child, so a mutation can delete or replace it in place. + if isinstance(node, dict): + for k, v in node.items(): + out.append((node, k)) + collect(v, out) + elif isinstance(node, list): + for idx, v in enumerate(node): + out.append((node, idx)) + collect(v, out) + + +def token(rng): + return "fuzz_" + "".join(rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def mutate_once(rng, roots): + refs = [] + for root in roots: + collect(root, refs) + if not refs: + return + container, key = rng.choice(refs) + op = rng.choice(["delete", "scalar", "dangerous", "empty"]) + if op == "delete": + del container[key] + elif op == "scalar": + container[key] = token(rng) + elif op == "dangerous": + container[key] = rng.choice(DANGEROUS) + else: + container[key] = rng.choice([{}, [], None]) + + +def mutate(config, seed): + rng = random.Random(seed) + + # Mutate only inside resource instances: keep bundle/name and the + # resources.. skeleton so there is always something to deploy, while + # every field of the instance (including required ones) is fair game. + roots = [] + for instances in config.get("resources", {}).values(): + if isinstance(instances, dict): + roots.extend(v for v in instances.values() if isinstance(v, (dict, list))) + + for _ in range(rng.randint(1, 3)): + mutate_once(rng, roots) + + return config + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--seed", type=int, required=True, help="RNG seed (for reproducibility)") + args = parser.parse_args() + + config = load_yaml(sys.stdin.read()) + if not isinstance(config, dict): + sys.exit("mutate_fuzz_config: base config did not parse to a mapping") + + sys.stdout.write(to_yaml(mutate(config, args.seed))) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py new file mode 100755 index 00000000000..6df04885dc6 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Contract check for mutate_fuzz_config's minimal YAML loader and mutation engine +(the harness diffs stdout; a non-zero exit marks a violation on stderr): + +- The loader round-trips every curated base config: load -> to_yaml -> load is a + fixed point, so a base template the loader can't represent is caught here rather + than as a confusing fuzz failure. +- Mutation is deterministic for a fixed seed (reproducible repros). + +It also prints a few mutated configs so an accidental change to the algorithm shows up +as an output diff. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from fuzz_gen_config import MUTATE_BASES +from mutate_fuzz_config import load_yaml, mutate, to_yaml + +CONFIGS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") + + +def render(name): + with open(os.path.join(CONFIGS, name + ".yml.tmpl")) as f: + return substitute_variables(f.read()) + + +def main(): + # Fixed so the printed configs are stable regardless of the harness's unique name. + os.environ["UNIQUE_NAME"] = "check" + failed = False + + for name in MUTATE_BASES: + text = render(name) + parsed = load_yaml(text) + if not isinstance(parsed, dict) or "resources" not in parsed: + sys.stderr.write(f"{name}: base did not parse to a config with resources\n") + failed = True + continue + # load -> emit -> load must be a fixed point. + if load_yaml(to_yaml(parsed)) != parsed: + sys.stderr.write(f"{name}: loader is not a round-trip fixed point\n") + failed = True + + # Mutation must be reproducible for a fixed seed. + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("volume")), seed)) + b = to_yaml(mutate(load_yaml(render("volume")), seed)) + if a != b: + sys.stderr.write(f"seed {seed}: mutation is not deterministic\n") + failed = True + + for seed in range(3): + sys.stdout.write(f"=== volume seed={seed} ===\n") + sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index 3ee8d65bc90..da100803191 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -31,3 +31,16 @@ def run(cmd): if result.returncode != 0: raise RunError(f"{cmd} failed with code {result.returncode}") return result + + +def load_plan(path): + # Empty or invalid output means `bundle plan` failed; exit cleanly (no traceback) + # so the fuzzer treats it as a rejected config, not a bug. Returns (data, raw). + with open(path) as fobj: + raw = fobj.read() + if not raw.strip(): + sys.exit(f"{path}: empty plan output (bundle plan failed)") + try: + return json.loads(raw), raw + except json.JSONDecodeError as e: + sys.exit(f"{path}: invalid plan JSON: {e}\n{raw}") diff --git a/acceptance/bin/verify_no_drift.py b/acceptance/bin/verify_no_drift.py index 9b272c1ce79..3afb3e3dd9a 100755 --- a/acceptance/bin/verify_no_drift.py +++ b/acceptance/bin/verify_no_drift.py @@ -3,26 +3,23 @@ Check that all actions in plan are "skip". """ -import json +import os import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from util import load_plan + def check_plan(path): - with open(path) as fobj: - raw = fobj.read() + data, raw = load_plan(path) changes_detected = 0 - - try: - data = json.loads(raw) - for key, value in data["plan"].items(): - action = value.get("action") - if action != "skip": - print(f"Unexpected {action=} for {key}") - changes_detected += 1 - except Exception: - print(raw, flush=True) - raise + for key, value in data["plan"].items(): + action = value.get("action") + if action != "skip": + print(f"Unexpected {action=} for {key}") + changes_detected += 1 if changes_detected: print(raw, flush=True) diff --git a/acceptance/bin/verify_plan_action.py b/acceptance/bin/verify_plan_action.py new file mode 100755 index 00000000000..d08c0113875 --- /dev/null +++ b/acceptance/bin/verify_plan_action.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +Check that a `bundle plan -o json` shows an expected action, for invariants beyond +no-drift. + + verify_plan_action.py PATH update every changed resource is an in-place update + (not a recreate) and at least one changed + verify_plan_action.py PATH create every resource is a create (e.g. a re-plan + after destroy must recreate everything) + +Action vocabulary mirrors bundle/deployplan/action.go. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from util import load_plan + +# update_id/resize keep the resource (no recreate), so they count as in-place updates. +ALLOWED = { + "update": {"update", "update_id", "resize"}, + "create": {"create"}, +} +# After a destroy, a "skip" means the resource survived (orphaned state), so skip is +# only tolerated for the update check, where unrelated siblings may be unchanged. +SKIP_OK = {"update": True, "create": False} + + +def main(): + path, expected = sys.argv[1], sys.argv[2] + allowed = ALLOWED[expected] + skip_ok = SKIP_OK[expected] + + data, raw = load_plan(path) + + matched = 0 + bad = 0 + for key, value in data["plan"].items(): + action = value.get("action") + if action == "skip" and skip_ok: + continue + if action in allowed: + matched += 1 + else: + print(f"Unexpected {action=} for {key} (expected {expected})") + bad += 1 + + if matched == 0: + print(f"plan shows no {expected} action; expected at least one") + bad += 1 + + if bad: + print(raw, flush=True) + sys.exit(10) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 184d3f541c4..3f653324498 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -4,3 +4,27 @@ no_drift test checks that there are no actions planned after successful deploy. test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. + +The fuzz/ test generates random configs from the live `databricks bundle schema` +(see fuzz/script) and runs each one through a real invariant test script. The target is +selected by `FUZZ_TARGET` (matrixed in fuzz/test.toml); each target is also a curated +invariant test that runs over the `INPUT_CONFIG` matrix. `FUZZ_RESOURCE_COUNT` (also +matrixed in fuzz/test.toml) controls how many resources each generated config contains; +with more than one, the generator links them with `${resources.*}` references (each +resource referencing an earlier one, so the graph stays acyclic) so the interpolation and +deploy-ordering paths are exercised. Free-form scalars are occasionally replaced with +dangerous / near-range-end values (empty, whitespace, over-long, control characters, +int32/int64 boundaries) to probe the CLI's input handling. + +- `no_drift` -- deploy, then no drift +- `migrate` -- Terraform deploy, migrate to direct, then no drift +- `redeploy` -- deploy twice; the second deploy must be a no-op +- `canonical` -- `validate -o json` must be byte-identical across two runs +- `update` -- edit a comment/description; the redeploy must update in place (not recreate) +- `destroy_recreate` -- deploy then destroy; a re-plan must recreate everything + +Since the schema comes from the CLI under test, an unrelated struct change can shift a +seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), +not flakiness; reproduce with +`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift task test-fuzz`. +For a multi-resource repro, add `FUZZ_RESOURCE_COUNT=2`. diff --git a/acceptance/bundle/invariant/canonical/out.test.toml b/acceptance/bundle/invariant/canonical/out.test.toml new file mode 100644 index 00000000000..8901423c85f --- /dev/null +++ b/acceptance/bundle/invariant/canonical/out.test.toml @@ -0,0 +1,58 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/canonical/output.txt b/acceptance/bundle/invariant/canonical/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/canonical/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/canonical/script b/acceptance/bundle/invariant/canonical/script new file mode 100644 index 00000000000..fe8acc5eb42 --- /dev/null +++ b/acceptance/bundle/invariant/canonical/script @@ -0,0 +1,43 @@ +# Invariant to test: `bundle validate -o json` is deterministic -- two runs must be +# byte-identical. Catches unstable map ordering / serialization in config loading. +# No deploy, so no cleanup or cloud state. + +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +$CLI bundle validate -o json > validate1.json 2>LOG.validate1.err +validate_rc=$? +cat LOG.validate1.err | contains.py '!panic:' '!internal error' > /dev/null + +# A config that fails to validate is an invalid fuzz config, not a bug, so stop before +# the marker (curated tests already aborted above under `bash -e`). +if [ "$validate_rc" -ne 0 ]; then + exit "$validate_rc" +fi + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +$CLI bundle validate -o json > validate2.json 2>LOG.validate2.err +cat LOG.validate2.err | contains.py '!panic:' '!internal error' > /dev/null + +# Determinism is cloud-independent and cheap, so it always runs (no SKIP_DRIFT_CHECK +# gate): identical input must yield identical output. A diff is a real bug. +diff validate1.json validate2.json > LOG.validate.diff diff --git a/acceptance/bundle/invariant/destroy_recreate/out.test.toml b/acceptance/bundle/invariant/destroy_recreate/out.test.toml new file mode 100644 index 00000000000..8901423c85f --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/out.test.toml @@ -0,0 +1,58 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/destroy_recreate/output.txt b/acceptance/bundle/invariant/destroy_recreate/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/destroy_recreate/script b/acceptance/bundle/invariant/destroy_recreate/script new file mode 100644 index 00000000000..7e615904f5f --- /dev/null +++ b/acceptance/bundle/invariant/destroy_recreate/script @@ -0,0 +1,85 @@ +# Invariant to test: after deploy then destroy, a re-plan wants to CREATE everything +# again -- proving destroy cleared all tracked state. A resource destroy forgets shows +# up as "skip" (still present), a bug. +# Additional checks: no internal errors / panics in validate/plan/deploy/destroy + +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + # This test destroys to LOG.destroy itself, so the trap logs elsewhere to keep + # the body's destroy output (and any panic) intact for the post-run scan. + trace $CLI bundle destroy --auto-approve &> LOG.destroy_cleanup + cat LOG.destroy_cleanup | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# Destroy unconditionally so any panic lands in LOG.destroy for the post-scan; +# completeness (re-plan recreates everything) is gated below. +trace $CLI bundle destroy --auto-approve &> LOG.destroy +destroy_rc=$? +cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + +# A clean destroy leaves nothing, so stop the cleanup trap from destroying again +# (which would hit unstubbed URLs on the fake server). +if [ "$destroy_rc" -eq 0 ]; then + deployed="" +fi + +# A fuzzed config can deploy yet legitimately leave state a re-plan reads differently, +# so the fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check +# that a re-plan recreates everything. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + if [ "$destroy_rc" -ne 0 ]; then + exit "$destroy_rc" + fi + + $CLI bundle plan -o json > LOG.recreate_plan.json 2>LOG.recreate_plan.err + cat LOG.recreate_plan.err | contains.py '!panic:' '!internal error' > /dev/null + verify_plan_action.py LOG.recreate_plan.json create +fi diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml new file mode 100644 index 00000000000..868a98f0bb8 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -0,0 +1,17 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] +EnvMatrix.FUZZ_TARGET = [ + "no_drift", + "migrate", + "redeploy", + "canonical", + "update", + "destroy_recreate" +] +EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/output.txt b/acceptance/bundle/invariant/fuzz/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script new file mode 100644 index 00000000000..86cb6bfea05 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/script @@ -0,0 +1,147 @@ +# Invariant fuzzing: generate a random config per seed and run the real invariant test +# script (no_drift/script or migrate/script). Rejection vs bug: those scripts print +# INPUT_CONFIG_OK once a config deploys, so a non-zero result before the marker is a +# rejection; a panic anywhere, or a failure after it, is a bug. +# +# Drift checking is opt-in (FUZZ_CHECK_DRIFT): a random config can deploy yet legitimately +# differ from the fake server, so the committed run asserts only no-panic. + +START="${FUZZ_SEED_START:-0}" +COUNT="${FUZZ_SEED_COUNT:-5}" + +# Per-seed cap: a seed past this generous budget is stuck, not slow, so SIGQUIT (Go dumps +# goroutines for triage) then SIGKILL and flag a hang. Needs GNU timeout; else uncapped. +SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" + +# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a +# slow-but-progressing variant (the mutate/deploy combos never finish a large seed window) +# is not force-killed at the per-script Timeout and read as a failure. Defaults on rather +# than opt-in so a direct `go test` run truncates cleanly too, not just `task test-fuzz`. +# 900s leaves margin under the 20m Timeout in test.toml for the last-started seed (capped at +# SEED_TIMEOUT) plus teardown; keep it comfortably below Timeout - SEED_TIMEOUT. Set +# FUZZ_TIME_BUDGET=0 to disable the cap (e.g. a completionist run with a raised Timeout). +BUDGET="${FUZZ_TIME_BUDGET:-900}" + +if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then + export SKIP_DRIFT_CHECK=1 +fi + +# no_drift/script reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. +export READPLAN="" + +# Emit the schema from the CLI under test so the generator always matches it. +$CLI bundle schema > schema.json 2>LOG.schema.err +cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null + +# Fail loud on a schema type the generator can't produce (the loop below would hide it). +check_schema_types.py --schema schema.json + +# One seed's worth of work, factored out so it can run either directly or under `timeout`. +seed_body() { + cd "$1" + # The generator points file_path/source_code_path fields at these fixtures + # (see gen_fuzz_config.py *_BY_RESOURCE), so make them resolvable from the + # bundle root, matching how the curated invariant scripts stage data/. + cp -r "$TESTDIR/../data/." . + export FUZZ_SEED="$2" + export FUZZ_SCHEMA="../schema.json" + source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" +} + +# timeout spawns a fresh bash without our shell functions, so export them (and seed_body). +export -f $(compgen -A function) + +# Run one seed, capped at SEED_TIMEOUT when `timeout` is available. +run_seed() { + local dir="$1" seed="$2" + # SEED_TIMEOUT=0 or no timeout binary: run uncapped (matches the hang reproduce hint). + if [ "$SEED_TIMEOUT" != "0" ] && command -v timeout > /dev/null 2>&1; then + timeout --signal=QUIT --kill-after=10s "$SEED_TIMEOUT" \ + bash -euo pipefail -c 'seed_body "$@"' _ "$dir" "$seed" > "$dir/LOG.check" 2>&1 + else + ( seed_body "$dir" "$seed" ) > "$dir/LOG.check" 2>&1 + fi +} + +# One machine-readable line per seed so a run is tallyable without grepping logs. A file, +# not stdout, so the committed run's empty-output assertion holds. +record() { + echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate} count=${FUZZ_RESOURCE_COUNT:-1}" >> LOG.summary +} + +for ((offset = 0; offset < COUNT; offset++)); do + # Stop before the per-test timeout kills us mid-seed; a clean stop, not a failure, so + # log to a file rather than the compared stdout/stderr. BUDGET=0 disables the cap. + if [ "$BUDGET" != "0" ] && [ "$SECONDS" -ge "$BUDGET" ]; then + echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget + break + fi + + seed=$((START + offset)) + dir="seed-$seed" + mkdir -p "$dir" + + set +e + run_seed "$dir" "$seed" + rc=$? + set -e + + if [ "$rc" -eq 0 ]; then + record deployed "$seed" + # Optional generator-independent regression corpus: persist configs that actually + # deployed, so runs accumulate known-good inputs that stay valid (and replayable) + # even after the generator changes. Unset by default, so the committed run and its + # empty-output assertion are unaffected. + if [ -n "${FUZZ_CORPUS_DIR:-}" ]; then + mkdir -p "$FUZZ_CORPUS_DIR" + cp "$dir/databricks.yml" "$FUZZ_CORPUS_DIR/${FUZZ_MODE:-generate}-${FUZZ_TARGET:-no_drift}-count${FUZZ_RESOURCE_COUNT:-1}-seed${seed}.yml" + fi + continue + fi + + # timeout exits 124 (137 if the SIGKILL backstop fired): the seed hung. Report a hang, + # distinct from a drift bug; any goroutine dump is preserved in the seed's LOG.*. + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + record hang "$seed" + echo "fuzz: seed $seed hung (>${SEED_TIMEOUT}s), reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_SEED_TIMEOUT=0 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 + exit 1 + fi + + bug="" + + # A panic anywhere is a bug even if the CLI then rejects the config. + if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic:' '!internal error' > /dev/null; then + bug=1 + fi + + # Failing after INPUT_CONFIG_OK means the config deployed but drifted (or destroy + # failed); failing before it with no panic just means the config was rejected. + if grep -q INPUT_CONFIG_OK "$dir/LOG.check"; then + bug=1 + fi + + if [ -n "$bug" ]; then + record bug "$seed" + echo "fuzz: invariant failed, reproduce with: FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} FUZZ_RESOURCE_COUNT=${FUZZ_RESOURCE_COUNT:-1} task test-fuzz" >&2 + exit 1 + fi + + # A 501 "No stub found" is a testserver coverage gap (see IgnoreUnhandledRequests); + # anything else before INPUT_CONFIG_OK is a genuine config rejection. + if grep -qs "No stub found for pattern" "$dir"/LOG.*; then + record gap "$seed" + else + record rejected "$seed" + fi +done + +# Per-variant tally for at-a-glance triage. Reached only on a clean run; a bug/hang exits +# above with the per-seed lines already recorded. +if [ -f LOG.summary ]; then + # Snapshot the counts before appending the header, else awk would also count it. + totals=$(awk '{print $1}' LOG.summary | sort | uniq -c) + { + echo "--- totals ---" + echo "$totals" + } >> LOG.summary +fi diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml new file mode 100644 index 00000000000..afd3a0bcc5c --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -0,0 +1,25 @@ +# Schema fuzzing (see script). Unlike the curated invariant tests, the fuzzer +# generates its own configs, so drop the inherited INPUT_CONFIG matrix. +EnvMatrix.INPUT_CONFIG = [] + +# Raise the inherited 10m cap so the nightly FUZZ_TIME_BUDGET (script) has room, plus the +# final seed's tail. The committed run (5 seeds, no drift) finishes in seconds regardless. +Timeout = '20m' + +# The fuzzer can emit resource types the testserver doesn't model; a missing handler is a +# coverage gap, so return 501 (config rejected) instead of failing the whole run. +IgnoreUnhandledRequests = true + +# Run the real invariant test script for each target. migrate ignores +# DATABRICKS_BUNDLE_ENGINE and starts from a Terraform deployment. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "redeploy", "canonical", "update", "destroy_recreate"] +EnvMatrix.FUZZ_RESOURCE_COUNT = ["1", "2", "3"] + +# generate = build a config from the schema (gen_fuzz_config.py); mutate = perturb a +# curated invariant config (mutate_fuzz_config.py). Dispatched by fuzz_gen_config.py. +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] + +# mutate starts from a single-resource base and ignores FUZZ_RESOURCE_COUNT, so only run +# it once rather than duplicating the same output across the count matrix. +EnvMatrixExclude.mutate_count2 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=2"] +EnvMatrixExclude.mutate_count3 = ["FUZZ_MODE=mutate", "FUZZ_RESOURCE_COUNT=3"] diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 78f45faa7da..a4357474be3 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -3,25 +3,36 @@ unset DATABRICKS_BUNDLE_ENGINE -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init -fi + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml -cp databricks.yml LOG.config + cp databricks.yml LOG.config +fi cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + trace $CLI bundle destroy --auto-approve &> LOG.destroy cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" if [ -f "$CLEANUP_SCRIPT" ]; then source "$CLEANUP_SCRIPT" &> LOG.cleanup fi @@ -30,7 +41,12 @@ cleanup() { trap cleanup EXIT trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy +deploy_rc=$? cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 echo INPUT_CONFIG_OK @@ -40,7 +56,7 @@ MIGRATE_ARGS="" # (order-sensitive), terraform plan reports positional drift when the bundle config # specifies depends_on in a different order than the provider's sorted state. # This is a false positive -- the logical dependencies are identical. -if [[ "$INPUT_CONFIG" == "job_with_depends_on.yml.tmpl" ]]; then +if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then MIGRATE_ARGS="--noplancheck" fi @@ -48,6 +64,10 @@ trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null -$CLI bundle plan -o json > plan.json 2>plan.json.err -cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null -verify_no_drift.py plan.json +# A fuzzed config can migrate yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to assert only no-panic; curated configs check drift. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + $CLI bundle plan -o json > plan.json 2>plan.json.err + cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py plan.json +fi diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 95ecd7cbfd7..31cf4bc6fbb 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -1,30 +1,41 @@ # Invariant to test: no drift after deploy # Additional checks: no internal errors / panics in validate/plan/deploy -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init -fi + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml -cp databricks.yml LOG.config + cp databricks.yml LOG.config +fi # We redirect output rather than record it because some configs that are being tested may produce warnings trace $CLI bundle validate &> LOG.validate -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" if [ -f "$CLEANUP_SCRIPT" ]; then source "$CLEANUP_SCRIPT" &> LOG.cleanup fi @@ -33,20 +44,45 @@ cleanup() { trap cleanup EXIT $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null +cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 # Special message to fuzzer that generated config was fine. # Any failures after this point will be considered as "bug detected" by fuzzer. echo INPUT_CONFIG_OK -# Check both text and JSON plan for no changes -# Note, expect that there maybe more than one resource unchanged -$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err -cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null -verify_no_drift.py LOG.planjson +# A fuzzed config can deploy yet legitimately differ from the fake server, so the +# fuzzer sets SKIP_DRIFT_CHECK to swap the exact no-drift check for the weaker but +# fidelity-independent plan-determinism oracle; curated configs check full drift. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # Check both text and JSON plan for no changes + # Note, expect that there maybe more than one resource unchanged + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson -$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan -cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +else + # Fuzz mode: the exact no-drift check false-positives because the fake server does + # not round-trip every field, and "no destructive re-plan" is no better -- for an + # unchanged config any recreate is just a local-vs-remote representation mismatch on + # an immutable field (fake-server fidelity), not real churn. So assert the one + # invariant that is independent of server fidelity: planning is deterministic. Two + # consecutive plans of the same deployed state must be byte-identical; a diff means + # nondeterministic planning/serialization (unstable map order, per-run randomness), a + # real bug. `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a + # bug -- both plans are then empty and the diff trivially passes. + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff +fi diff --git a/acceptance/bundle/invariant/redeploy/out.test.toml b/acceptance/bundle/invariant/redeploy/out.test.toml new file mode 100644 index 00000000000..8901423c85f --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/out.test.toml @@ -0,0 +1,58 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/redeploy/output.txt b/acceptance/bundle/invariant/redeploy/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/redeploy/script b/acceptance/bundle/invariant/redeploy/script new file mode 100644 index 00000000000..af5bddd50e1 --- /dev/null +++ b/acceptance/bundle/invariant/redeploy/script @@ -0,0 +1,97 @@ +# Invariant to test: a second deploy of an unchanged config is a clean no-op +# Additional checks: no internal errors / panics in validate/plan/deploy +# +# Catches create handlers that don't round-trip their inputs (or mutators that +# re-derive a field), which surface as a redeploy wanting to change or recreate. + +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# A fuzzed config can deploy yet legitimately fail to redeploy or differ, so the fuzzer +# sets SKIP_DRIFT_CHECK to swap the exact no-op check for the weaker but +# fidelity-independent plan-determinism oracle; curated configs check the no-op. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + trace $CLI bundle deploy &> LOG.redeploy + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + # Check both text and JSON plan for no changes + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson + + $CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null +else + # Fuzz mode: the exact no-op check false-positives because the fake server does not + # round-trip every field, so a redeploy can legitimately want a destructive recreate + # (e.g. an immutable field the fake server reformats) and then fail non-interactively. + # That is a fidelity gap, not a bug, so tolerate a non-zero redeploy; a panic in it is + # still caught below. On a clean redeploy, assert the one fidelity-independent + # invariant: planning is deterministic -- two consecutive plans of the redeployed state + # must be byte-identical. A diff means nondeterministic planning/serialization + # (unstable map order, per-run randomness), a real bug. + redeploy_rc=0 + trace $CLI bundle deploy &> LOG.redeploy || redeploy_rc=$? + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + if [ "$redeploy_rc" -eq 0 ]; then + # `|| true`: a plan that fails on an unstubbed read is a coverage gap, not a bug -- + # both plans are then empty and the diff trivially passes. + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err || true + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err || true + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + fi +fi diff --git a/acceptance/bundle/invariant/update/out.test.toml b/acceptance/bundle/invariant/update/out.test.toml new file mode 100644 index 00000000000..8901423c85f --- /dev/null +++ b/acceptance/bundle/invariant/update/out.test.toml @@ -0,0 +1,58 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/update/output.txt b/acceptance/bundle/invariant/update/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/update/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/update/script b/acceptance/bundle/invariant/update/script new file mode 100644 index 00000000000..7a91c38cef9 --- /dev/null +++ b/acceptance/bundle/invariant/update/script @@ -0,0 +1,84 @@ +# Invariant to test: editing a comment/description redeploys as an in-place update, not +# a recreate, and converges. Exercises the update (PATCH) path create-only deploys never +# touch. +# Additional checks: no internal errors / panics in validate/plan/deploy + +if [ -n "${FUZZ_SEED:-}" ]; then + fuzz_gen_config.py > databricks.yml 2>LOG.gen.err + cat LOG.gen.err | contains.py '!Traceback' > /dev/null + cp databricks.yml LOG.config +else + # Copy data files to test directory + cp -r "$TESTDIR/../data/." . &> LOG.cp + + # Run init script if present + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + + cp databricks.yml LOG.config +fi + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + +cleanup() { + # A rejected fuzz config deployed nothing; destroying nothing hits unstubbed URLs. + if [ -z "${deployed:-}" ]; then + return + fi + + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +deploy_rc=$? +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null +if [ "$deploy_rc" -ne 0 ]; then + exit "$deploy_rc" +fi +deployed=1 + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +# A fuzzed config can deploy yet legitimately differ on update, so the fuzzer sets +# SKIP_DRIFT_CHECK to assert only no-panic; curated configs check the update path. +if [ -z "${SKIP_DRIFT_CHECK:-}" ]; then + # Only configs with an editable comment/description exercise the update path; + # others just verify the deploy above (edit_fuzz_config.py --detect exits 1). + if edit_fuzz_config.py databricks.yml --detect 2>LOG.detect.err; then + # Change the comment/description; the re-plan must show an in-place update. + edit_fuzz_config.py databricks.yml 2>LOG.edit.err + cat LOG.edit.err | contains.py '!Traceback' > /dev/null + + $CLI bundle plan -o json > LOG.update_plan.json 2>LOG.update_plan.err + cat LOG.update_plan.err | contains.py '!panic:' '!internal error' > /dev/null + + trace $CLI bundle deploy &> LOG.redeploy + cat LOG.redeploy | contains.py '!panic:' '!internal error' > /dev/null + + # The edit must update in place, not recreate. + verify_plan_action.py LOG.update_plan.json update + + # And the applied update must converge: a re-plan shows no further changes. + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson + fi +fi diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index b2d1a9f578f..545445da041 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -89,6 +89,11 @@ type TestConfig struct { // out.requests.txt RecordRequests *bool + // Return 501 for a request with no handler instead of failing the test. The fuzzer + // emits resource types the testserver may not model; a missing handler is a gap, not + // a bug. + IgnoreUnhandledRequests *bool + // List of request headers to include when recording requests. IncludeRequestHeaders []string diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 3b1be7a79b3..5c1c6591631 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -151,7 +151,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o // Default case. Start a dedicated local server for the test with the server stubs configured // as overrides. - host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir) + host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir, isTruePtr(config.IgnoreUnhandledRequests)) cfg := &sdkconfig.Config{ Host: host, Token: token, @@ -204,8 +204,10 @@ func startLocalServer(t *testing.T, logRequests bool, includeHeaders []string, outputDir string, + ignoreUnhandledRequests bool, ) string { s := testserver.New(t) + s.IgnoreUnhandledRequests = ignoreUnhandledRequests // Record API requests in out.requests.txt if RecordRequests is true // in test.toml diff --git a/acceptance/selftest/edit_fuzz_config/out.test.toml b/acceptance/selftest/edit_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/edit_fuzz_config/output.txt b/acceptance/selftest/edit_fuzz_config/output.txt new file mode 100644 index 00000000000..4037d0995cf --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/output.txt @@ -0,0 +1,4 @@ +model_serving_endpoints.description immutable: True +IMMUTABLE_ONLY: None +IMMUTABLE_THEN_MUTABLE: 6 +MUTABLE_ONLY: 3 diff --git a/acceptance/selftest/edit_fuzz_config/script b/acceptance/selftest/edit_fuzz_config/script new file mode 100644 index 00000000000..78dfbef7825 --- /dev/null +++ b/acceptance/selftest/edit_fuzz_config/script @@ -0,0 +1 @@ +edit_fuzz_config_check.py diff --git a/acceptance/selftest/gen_fuzz_config/out.test.toml b/acceptance/selftest/gen_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/gen_fuzz_config/output.txt b/acceptance/selftest/gen_fuzz_config/output.txt new file mode 100644 index 00000000000..75c5a658de4 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/output.txt @@ -0,0 +1,20 @@ +comment: "value: with a colon" +description: "quote \" and : colon" +resources: + jobs: + j: + name: "n" + tags: + team: "jobs" +tasks: + - description: "d" + timeout_seconds: 3600 + - comment: "c" +nums: + - 0 + - 1 + - 2 +flag: true +ratio: 1.5 +empty_map: {} +empty_list: [] diff --git a/acceptance/selftest/gen_fuzz_config/script b/acceptance/selftest/gen_fuzz_config/script new file mode 100644 index 00000000000..2737c67674d --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/script @@ -0,0 +1 @@ +gen_fuzz_config_check.py diff --git a/acceptance/selftest/mutate_fuzz_config/out.test.toml b/acceptance/selftest/mutate_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/mutate_fuzz_config/output.txt b/acceptance/selftest/mutate_fuzz_config/output.txt new file mode 100644 index 00000000000..4b7289cfa51 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/output.txt @@ -0,0 +1,36 @@ +=== volume seed=0 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: "main" + schema_name: [] + grants: + - principal: "account users" +=== volume seed=1 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: " " + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" +=== volume seed=2 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" diff --git a/acceptance/selftest/mutate_fuzz_config/script b/acceptance/selftest/mutate_fuzz_config/script new file mode 100644 index 00000000000..2d57c739261 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/script @@ -0,0 +1 @@ +mutate_fuzz_config_check.py diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 351c5a76499..26a5740a3b5 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -30,15 +30,21 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { StorageRoot: createRequest.StorageRoot, ProviderName: createRequest.ProviderName, ShareName: createRequest.ShareName, - Options: createRequest.Options, - Properties: createRequest.Properties, - FullName: createRequest.Name, - CreatedAt: nowMilli(), - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - Owner: s.CurrentUser().UserName, - CatalogType: catalog.CatalogTypeManagedCatalog, + // Round-trip these so a re-read matches the deployed config: connection_name is + // recreate_on_changes (immutable), so dropping it re-planned as a perpetual + // recreate, and managed_encryption_settings showed as drift. + ConnectionName: createRequest.ConnectionName, + ManagedEncryptionSettings: createRequest.ManagedEncryptionSettings, + CustomMaxRetentionHours: createRequest.CustomMaxRetentionHours, + Options: createRequest.Options, + Properties: createRequest.Properties, + FullName: createRequest.Name, + CreatedAt: nowMilli(), + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + Owner: s.CurrentUser().UserName, + CatalogType: catalog.CatalogTypeManagedCatalog, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5ae3141bfd2..475c92164b9 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -73,6 +73,11 @@ type Server struct { RequestCallback func(request *Request) ResponseCallback func(request *Request, response *EncodedResponse) + + // IgnoreUnhandledRequests returns 501 for a request with no handler instead of failing + // the test: the fuzzer emits resource types the testserver may not model, so the caller + // rejects the config. Curated tests leave it false so real gaps stay loud. + IgnoreUnhandledRequests bool } type Request struct { @@ -278,7 +283,11 @@ func New(t testutil.TestingT) *Server { body = fmt.Sprintf("[%d bytes] %s", len(bodyBytes), bodyBytes) } - t.Errorf(`No handler for URL: %s + if s.IgnoreUnhandledRequests { + // Coverage gap, not a CLI bug: log it but return the 501 so the caller can reject. + t.Logf("No handler for URL (ignored): %s", r.URL) + } else { + t.Errorf(`No handler for URL: %s Body: %s For acceptance tests, add this to test.toml: @@ -287,6 +296,7 @@ Pattern = %q Response.Body = '' # Response.StatusCode = `, r.URL, body, pattern) + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotImplemented) diff --git a/libs/testserver/server_test.go b/libs/testserver/server_test.go index 6c6dfe8c160..141a873c9bd 100644 --- a/libs/testserver/server_test.go +++ b/libs/testserver/server_test.go @@ -3,12 +3,54 @@ package testserver_test import ( "net/http" "net/http/httptest" + "sync" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// recordingT wraps a real TestingT but records Errorf calls instead of failing, +// so a test can assert whether the server would have failed the run. +type recordingT struct { + testutil.TestingT + mu sync.Mutex + errCount int +} + +func (r *recordingT) Errorf(format string, args ...any) { + r.mu.Lock() + defer r.mu.Unlock() + r.errCount++ +} + +func (r *recordingT) errors() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.errCount +} + +func TestIgnoreUnhandledRequests(t *testing.T) { + for _, ignore := range []bool{false, true} { + rt := &recordingT{TestingT: t} + s := testserver.New(rt) + s.IgnoreUnhandledRequests = ignore + + resp, err := http.Get(s.URL + "/api/2.0/no-such-endpoint") + require.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + + if ignore { + assert.Zero(t, rt.errors(), "unhandled request must not fail the test when ignored") + } else { + assert.Positive(t, rt.errors(), "unhandled request must fail the test by default") + } + } +} + func TestIsLocalhostProbe(t *testing.T) { tests := []struct { name string