{ "compute": [ { "query_id": "classic_clusters_config_current", "title": "Classic cluster configuration (current)", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.clusters" ], "requires": "SELECT on system.compute; GA", "params": [], "confidence": "confirmed", "confidence_note": "Columns verified against system.compute.clusters in a live workspace; cloud-specific attribute structs (aws_attributes/azure_attributes/gcp_attributes) are documented by example only.", "read_this": "One row = the latest known configuration for one classic cluster (all-purpose, job, Lakeflow SDP, or pipeline-maintenance) that has not been deleted. The columns that matter are data_security_mode (access-mode posture) and policy_id (NULL means the cluster is not governed by a compute policy).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "node_types_reference", "if": "to size vCPU/memory/GPU for driver_node_type/worker_node_type" }, { "query_id": "compute_idle_node_ratio", "if": "to see idle time on these classic clusters" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/classic_clusters_config_current.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-classic_clusters_config_current" }, { "query_id": "compute_idle_node_ratio", "title": "Idle classic clusters by idle-slice ratio", "domain": "compute", "tier": "standard", "stars": false, "reads": [ "system.compute.node_timeline", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.compute, system.billing; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days, capped at 90 in-SQL by node_timeline retention" }, { "name": "idle_cpu_pct", "default": 5, "meaning": "CPU busy percent below which a minute-slice counts as idle" }, { "name": "min_slices", "default": 60, "meaning": "minimum node-minutes before a cluster is judged (fewer -> NOT_ASSESSED)" }, { "name": "warn_idle_ratio", "default": 0.5, "meaning": "idle-slice fraction that flags WARN" }, { "name": "crit_idle_ratio", "default": 0.8, "meaning": "idle-slice fraction that flags CRITICAL" }, { "name": "top_n", "default": 200, "meaning": "row cap" } ], "confidence": "confirmed", "confidence_note": "node_timeline columns verified in a live workspace; the est_wasted_usd_list overlay is a directional list-price estimate, not a verified invoice figure.", "read_this": "One row = one classic cluster over the window. idle_ratio (idle minute-slices / total minute-slices) is the column that matters; est_wasted_usd_list scales the cluster's list-price DBU cost by that idle fraction so you can rank the biggest likely savings. High idle_ratio + meaningful est_wasted_usd_list = an auto-stop / downsizing candidate.", "healthy": "idle_ratio below :warn_idle_ratio (field heuristic - tune :warn_idle_ratio for your account).", "investigate_if": "idle_ratio at/above :warn_idle_ratio (WARN) or :crit_idle_ratio (CRITICAL) with real est_wasted_usd_list - field heuristic. BEFORE downsizing, check avg_mem_pct_all: high memory + low CPU means the cluster is memory-bound, NOT idle, so shrinking it will spill.", "actions": [ "set/shorten auto-termination on the cluster so idle time stops billing (free);", "enable autoscaling or lower the min workers if the idle is steady-state over-provisioning (config);", "move the workload to a smaller or memory-optimized node type once you have confirmed it is CPU-idle, not memory-bound (spend)." ], "next": [ { "query_id": "cost_by_compute_resource", "if": "to see the same cluster's full billed DBUs by day" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/compute_idle_node_ratio.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-compute_idle_node_ratio" }, { "query_id": "compute_warehouse_autoscale_churn", "title": "SQL warehouse autoscale churn (scale-up/scale-down thrash)", "domain": "compute", "tier": "standard", "stars": false, "reads": [ "system.compute.warehouse_events", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.compute, system.billing; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "min_observed_hours", "default": 1, "meaning": "minimum hours between a warehouse's first and last event in the window before a churn rate is reported (guards divide-by-zero and short-span spikes)" }, { "name": "warn_churn_per_hour", "default": 4, "meaning": "scaling events per observed hour that flags WARN" }, { "name": "crit_churn_per_hour", "default": 10, "meaning": "scaling events per observed hour that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Event-type enum and the cost-rollup join verified against system.compute.warehouse_events and system.billing.usage/list_prices in a live workspace.", "read_this": "One row = one SQL warehouse's scale-up/scale-down activity over the window. The columns that matter are scaling_events (SCALED_UP + SCALED_DOWN counts) and observed_hours (the span between this warehouse's first and last event in the window) - a warehouse thrashing clusters up and down wastes cold-start time and DBUs without necessarily showing up as idle.", "healthy": "status = OK - field heuristic; tune :warn_churn_per_hour / :crit_churn_per_hour for your account.", "investigate_if": "status = WARN or CRITICAL (scaling_events / observed_hours at or above the threshold) - field heuristic. status = NOT_ASSESSED means observed_hours is below :min_observed_hours, i.e. too little history to rate, not a clean bill of health.", "actions": [ "widen the warehouse's min_clusters/max_clusters autoscale bounds, or switch it to a fixed cluster count, so it stops thrashing (free);", "raise auto_stop_minutes or route bursty/spiky workloads to a dedicated warehouse so scaling settles (config);", "move the worst-churning warehouse to Serverless SQL, which absorbs scaling internally (spend)." ], "next": [ { "query_id": "sql_warehouse_config_current", "if": "to see this warehouse's min_clusters/max_clusters" }, { "query_id": "compute_warehouse_idle_gaps", "if": "the same warehouse also shows a long RUNNING idle tail" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/compute_warehouse_autoscale_churn.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-compute_warehouse_autoscale_churn" }, { "query_id": "compute_warehouse_idle_gaps", "title": "SQL warehouse idle tail before auto-stop", "domain": "compute", "tier": "standard", "stars": true, "reads": [ "system.compute.warehouse_events", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.compute, system.billing; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_idle_hours", "default": 4, "meaning": "hours of a single continuous RUNNING stretch that flags WARN" }, { "name": "crit_idle_hours", "default": 24, "meaning": "hours that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Event-type enum, the gap-window logic (LEAD across ALL events for a warehouse, not per event_type), and the cost-rollup join were all verified against system.compute.warehouse_events and system.billing.usage/list_prices in a live workspace.", "read_this": "One row = one SQL warehouse's time spent in each event state (RUNNING, STARTING, STOPPED) over the window, computed as the gap from each event to that warehouse's very next event (any type). The columns that matter are running_seconds (total time the warehouse sat RUNNING before its next event - a mix of active query time and the idle tail before auto-stop, since per-query activity is deliberately not joined into this query) and max_running_gap_seconds (the single longest continuous RUNNING stretch - an unusually long one usually means the warehouse never triggered its own auto-stop for a long time).", "healthy": "status = OK (max_running_gap_seconds below :warn_idle_hours hours) - field heuristic; tune :warn_idle_hours / :crit_idle_hours to your typical auto_stop_minutes settings.", "investigate_if": "status = WARN or CRITICAL (a single RUNNING stretch at or above :warn_idle_hours / :crit_idle_hours) - field heuristic. status = NOT_ASSESSED means this warehouse had zero RUNNING events in the window (running_seconds=0), which this query still surfaces rather than dropping, since an unused warehouse is itself worth a look.", "actions": [ "lower auto_stop_minutes on the warehouse in sql_warehouse_config_current so it suspends sooner after the last query (free);", "route the warehouse's workload onto a shared/Serverless warehouse if it is mostly idle between bursts (config);", "downsize warehouse_size if the long RUNNING stretch reflects genuinely low, steady load rather than a stuck auto-stop (spend)." ], "next": [ { "query_id": "sql_warehouse_config_current", "if": "to check this warehouse's auto_stop_minutes" }, { "query_id": "sql_warehouse_events_activity", "if": "for the raw event-type breakdown" }, { "query_id": "compute_warehouse_autoscale_churn", "if": "the same warehouse is also thrashing clusters up/down" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/compute_warehouse_idle_gaps.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-compute_warehouse_idle_gaps" }, { "query_id": "instance_events_idle_active", "title": "Classic-instance idle-vs-active event mix", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.instance_events" ], "requires": "SELECT on system.compute; Public Preview (system.compute.instance_events may be empty or disabled per workspace)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "system.compute.instance_events is Public Preview and may be empty or disabled in your workspace; if so, treat \"no rows\" as \"not populated\", not \"no idle instances\". Columns are confirmed, but a true idle-vs-active MINUTES calculation needs a per-instance windowed lead/lag over event_time (INSTANCE_READY vs INSTANCE_PLACED) that is not implemented here - this query only summarizes event COUNTS, a coarser signal.", "read_this": "One row = a workspace + node_type + availability_type + instance state + event_type combination and how many events/instances hit it over the window. The columns that matter are state (INSTANCE_READY = idle, INSTANCE_PLACED = in use) and instance_count - a node_type with a large INSTANCE_READY instance_count relative to its INSTANCE_PLACED instance_count is spending a lot of launches sitting idle before being used, but this query counts events, not idle minutes, so treat it as a screening signal, not a precise duration.", "healthy": "n/a - status not computed (see caveats); as a heuristic, INSTANCE_READY instance_count well below INSTANCE_PLACED instance_count for the same node_type/availability_type is the healthy shape.", "investigate_if": "n/a - status not computed (see caveats); as a heuristic, INSTANCE_READY instance_count at or above INSTANCE_PLACED instance_count for the same node_type/availability_type over the window is worth a look - field heuristic, not an authoritative threshold.", "actions": [ "cross-check the worst node_type/availability_type combos here against auto_termination_minutes in classic_clusters_config_current and lower it (free);", "shift the workload onto instance pools sized closer to actual concurrent demand, or onto Spot/preemptible where availability_type allows (config);", "if the same node_type is idle-heavy account-wide, right-size or retire that node type (spend)." ], "next": [ { "query_id": "classic_clusters_config_current", "if": "to see auto_termination_minutes for clusters using this node_type" }, { "query_id": "instance_pools_idle_capacity", "if": "these instances are pool-managed" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/instance_events_idle_active.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-instance_events_idle_active" }, { "query_id": "instance_pools_idle_capacity", "title": "Instance pool idle-capacity floor (min_idle_instances)", "domain": "compute", "tier": "standard", "stars": false, "reads": [ "system.compute.instance_pools", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.compute, system.billing; Public Preview (system.compute.instance_pools may be empty or disabled per workspace)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days for the cost rollup (this is a config snapshot with no time window of its own)" }, { "name": "warn_min_idle_instances", "default": 5, "meaning": "standing idle-instance floor that flags WARN" }, { "name": "crit_min_idle_instances", "default": 20, "meaning": "standing idle-instance floor that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "system.compute.instance_pools is Public Preview and may be empty or disabled in your workspace; the columns themselves are confirmed.", "read_this": "One row = the latest known configuration for one instance pool that has not been deleted, plus its compute spend over :period_days. The column that matters is min_idle_instances - the number of instances this pool keeps warm and idle at all times, which bills regardless of whether anything is using them.", "healthy": "status = OK (min_idle_instances below :warn_min_idle_instances) - field heuristic; tune for your account's pool sizes.", "investigate_if": "status = WARN or CRITICAL (min_idle_instances at or above :warn_min_idle_instances / :crit_min_idle_instances) - field heuristic. Also worth a look regardless of status: idle_instance_autotermination_minutes NULL or 0, which means instances above the floor never get reclaimed.", "actions": [ "lower min_idle_instances to match actual concurrent start-up demand (free);", "set or lower idle_instance_autotermination_minutes so instances above the floor get reclaimed instead of idling indefinitely (config);", "if est_usd_list is high and demand is genuinely bursty, move the workload to Serverless compute instead of a warm pool (spend)." ], "next": [ { "query_id": "classic_clusters_config_current", "if": "to see which clusters draw from this pool via driver_instance_pool_id/worker_instance_pool_id" }, { "query_id": "node_types_reference", "if": "to size the node_type this pool uses" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/instance_pools_idle_capacity.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-instance_pools_idle_capacity" }, { "query_id": "node_timeline_utilization", "title": "Per-cluster CPU / memory / network utilization profile", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.node_timeline" ], "requires": "SELECT on system.compute; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days, capped at 90 in-SQL by node_timeline retention" }, { "name": "min_slices", "default": 60, "meaning": "minimum node-minutes before a cluster is judged (fewer -> NOT_ASSESSED)" }, { "name": "oversized_cpu_pct", "default": 20, "meaning": "avg CPU percent below which a cluster looks oversized" }, { "name": "oversized_mem_pct", "default": 30, "meaning": "avg memory percent below which a cluster looks oversized" }, { "name": "top_n", "default": 200, "meaning": "row cap" } ], "confidence": "confirmed", "confidence_note": "node_timeline utilization columns verified in a live workspace.", "read_this": "One row = one cluster + node_type + driver/worker role, with average and peak CPU and memory over the window. avg_cpu_pct and avg_mem_pct are the columns that matter: both low over many slices means the node is oversized for its workload. This is a right-sizing profile; compute_idle_node_ratio is the dedicated idle finding.", "healthy": "avg_cpu_pct at/above :oversized_cpu_pct OR avg_mem_pct at/above :oversized_mem_pct - the box is being used (field heuristic).", "investigate_if": "avg_cpu_pct below :oversized_cpu_pct AND avg_mem_pct below :oversized_mem_pct across enough slices (WARN = oversized candidate) - field heuristic; peak_* matters too, a low average with a high peak may just be bursty.", "actions": [ "confirm the low utilization is steady (not a bursty job) from peak_cpu_pct / peak_mem_pct before acting (free);", "drop to a smaller node type or fewer workers, or enable autoscaling (config);", "switch to a memory- or compute-optimized family that matches the real bottleneck (spend)." ], "next": [ { "query_id": "compute_idle_node_ratio", "if": "to rank the same clusters by idle-slice ratio and wasted list-price $" }, { "query_id": "cost_by_compute_resource", "if": "to attach billed DBUs to a right-sizing candidate" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/node_timeline_utilization.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-node_timeline_utilization" }, { "query_id": "node_types_reference", "title": "Node type reference (vCPU / memory / GPU)", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.node_types" ], "requires": "SELECT on system.compute; GA", "params": [], "confidence": "confirmed", "confidence_note": "Columns verified against system.compute.node_types in a live workspace; this is a static reference dimension, not usage data.", "read_this": "One row = one node type's fixed specs (vCPU, memory, GPU count). Join node_type to classic_clusters_config_current.driver_node_type/worker_node_type or to node_timeline_utilization.node_type to turn a node type name into actual capacity.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "classic_clusters_config_current", "if": "to see which clusters use a given node_type" }, { "query_id": "node_timeline_utilization", "if": "for per-node utilization by node_type" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/node_types_reference.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-node_types_reference" }, { "query_id": "sql_warehouse_config_current", "title": "SQL warehouse configuration (current)", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.warehouses" ], "requires": "SELECT on system.compute; GA", "params": [], "confidence": "confirmed", "confidence_note": "Columns verified against system.compute.warehouses in a live workspace, cross-checked against the full column list (see caveats for a vendor-doc miscount).", "read_this": "One row = the latest known configuration for one SQL warehouse that has not been deleted. The columns that matter are warehouse_size, min_clusters/max_clusters (autoscaling bounds), and auto_stop_minutes (how long the warehouse waits before suspending).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "compute_warehouse_idle_gaps", "if": "to see this warehouse's actual RUNNING idle tail against its auto_stop_minutes" }, { "query_id": "compute_warehouse_autoscale_churn", "if": "to see if min_clusters/max_clusters is causing thrash" }, { "query_id": "sql_warehouse_events_activity", "if": "for the raw event history" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/sql_warehouse_config_current.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-sql_warehouse_config_current" }, { "query_id": "sql_warehouse_events_activity", "title": "SQL warehouse event activity (raw event-type breakdown)", "domain": "compute", "tier": "lite", "stars": false, "reads": [ "system.compute.warehouse_events" ], "requires": "SELECT on system.compute; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_stale_days", "default": 7, "meaning": "days since a warehouse's last RUNNING/STARTING event that flags WARN" }, { "name": "crit_stale_days", "default": 14, "meaning": "days that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Event-type enum verified against system.compute.warehouse_events in a live workspace; SCALING_UP/SCALING_DOWN values are undocumented so this query does not rely on them.", "read_this": "One row = one warehouse_id + event_type combination and its event count/cluster-count stats over the window. The columns that matter are event_type (RUNNING/STARTING show the warehouse was actually used; SCALED_UP/SCALED_DOWN show autoscaling churn; STOPPED/STOPPING show suspend behavior) and last_event_time - for a RUNNING or STARTING row, an old last_event_time means this warehouse has gone quiet.", "healthy": "status = OK on the RUNNING/STARTING rows (last RUNNING or STARTING event within :warn_stale_days) - field heuristic; tune for your account's usage cadence.", "investigate_if": "status = WARN or CRITICAL on a RUNNING/STARTING row (no RUNNING or STARTING event for :warn_stale_days / :crit_stale_days) - the warehouse has likely gone dormant. status is NOT_ASSESSED on SCALED_UP/SCALED_DOWN/STOPPED/STOPPING rows - staleness on those event types is not inherently good or bad from this table alone.", "actions": [ "if a warehouse shows no RUNNING/STARTING activity for a long stretch, confirm with its owner it is still needed before touching config (free);", "if confirmed unused, lower auto_stop_minutes or pause/decommission it in sql_warehouse_config_current (config);", "if it is a scheduled/rarely-used warehouse by design, move it to Serverless so idle time between runs costs nothing (spend)." ], "next": [ { "query_id": "sql_warehouse_config_current", "if": "to check auto_stop_minutes/warehouse_size for a dormant warehouse" }, { "query_id": "compute_warehouse_idle_gaps", "if": "for the RUNNING idle-tail duration instead of just event counts" }, { "query_id": "compute_warehouse_autoscale_churn", "if": "for the scaling-specific churn signal" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/compute/sql_warehouse_events_activity.sql", "learn_url": "/learn/tech/databricks/audit/compute#q-sql_warehouse_events_activity" } ], "cost": [ { "query_id": "cost_account_prices_raw", "title": "Raw account price history by SKU", "domain": "cost", "tier": "lite", "stars": false, "reads": [ "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.list_prices is generally available)", "params": [], "confidence": "needs_confirmation", "confidence_note": "Whether pricing.default (list rate) or pricing.effective_list.default (effective/negotiated rate) should anchor a discount-realization comparison is not fixed for this workspace; this query collects pricing.default raw so you can decide which to use yourself.", "read_this": "One row = a SKU's price at a point in time. The columns that matter are pricing_default (the list rate, cast to STRING because pricing.default is a high-precision decimal) and price_end_time (NULL means this is the currently active price for that SKU/cloud/currency).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_actual_vs_list_by_sku", "if": "to compare this rate against list/effective pricing" }, { "query_id": "pricing_list_prices_raw", "if": "for the full raw pricing struct including effective_list and promotional" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_account_prices_raw.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_account_prices_raw" }, { "query_id": "cost_actual_vs_list_by_sku", "title": "Actual vs list price by SKU (discount realization)", "domain": "cost", "tier": "deep", "stars": true, "reads": [ "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_realization_ratio", "default": 0.75, "meaning": "share of list price you are estimated to actually pay that flags WARN" }, { "name": "crit_realization_ratio", "default": 0.9, "meaning": "share of list price you are estimated to actually pay that flags CRITICAL - field heuristic" } ], "confidence": "needs_confirmation", "confidence_note": "There is no system.billing.account_prices table in this environment, so both the \"actual/default\" and \"list\" rates are approximated from system.billing.list_prices (pricing.default and pricing.effective_list.default respectively); treat every dollar here as default/list-unverified, not a confirmed negotiated rate.", "read_this": "One row = a cloud + SKU + usage_unit's DBU spend for the window, priced two ways. The columns that matter are net_default_cost (what you are estimated to actually pay) and net_list_cost (undiscounted list price) - the smaller the gap between them, the less discount you are realizing on that SKU.", "healthy": "net_default_cost stays well below net_list_cost - i.e. the realization ratio (net_default_cost / net_list_cost) is under :warn_realization_ratio (field heuristic; tune for your account's commit).", "investigate_if": "realization ratio at/above :warn_realization_ratio (WARN) or :crit_realization_ratio (CRITICAL) - you are paying close to list price on that SKU; or either cost column is NULL (NOT_ASSESSED - a priced-coverage gap, not $0).", "actions": [ "confirm the SKU actually needs its current tier/edition before assuming a pricing problem (free);", "route the workload to a discounted commitment tier or a cheaper SKU family for that workload (config);", "escalate the SKU to your Databricks account team for a better negotiated rate (spend)." ], "next": [ { "query_id": "cost_dollarized_by_sku_day", "if": "for the full dollarized cost series across all SKUs" }, { "query_id": "cost_account_prices_raw", "if": "for the underlying raw price rows behind this comparison" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_actual_vs_list_by_sku.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_actual_vs_list_by_sku" }, { "query_id": "cost_by_billing_origin_product", "title": "Usage by product line", "domain": "cost", "tier": "lite", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "billing_origin_product and usage_unit are documented system.billing.usage columns.", "read_this": "One row = a product line + usage unit + cloud's total usage for the window. The columns that matter are billing_origin_product (where the DBUs/bytes/tokens went - JOBS, SQL, MODEL_SERVING, DLT, and so on) and net_usage_quantity (the summed volume in that row's native unit).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "for the same window broken out by SKU and workspace" }, { "query_id": "cost_by_job", "if": "to drill a JOBS spike down to individual jobs" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_by_billing_origin_product.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_by_billing_origin_product" }, { "query_id": "cost_by_compute_resource", "title": "DBU cost by cluster, warehouse, and pool", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_resource_dbus_per_day", "default": 50, "meaning": "DBUs/day on a single cluster, warehouse, or instance pool that flags WARN" }, { "name": "crit_resource_dbus_per_day", "default": 200, "meaning": "DBUs/day that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "usage_metadata.{cluster_id, warehouse_id, instance_pool_id} and product_features.is_serverless are documented system.billing.usage columns.", "read_this": "One row = a day + workspace + compute resource's DBU cost. The columns that matter are cluster_id/warehouse_id/instance_pool_id (which resource) and net_usage_quantity (its DBU burn for that day) - a resource that stays above the WARN/CRITICAL band day after day is the one worth right-sizing or decommissioning first.", "healthy": "net_usage_quantity below :warn_resource_dbus_per_day DBUs/day per resource (field heuristic - tune :warn_resource_dbus_per_day for your account).", "investigate_if": "net_usage_quantity at/above :warn_resource_dbus_per_day (WARN) or :crit_resource_dbus_per_day (CRITICAL) DBUs/day - field heuristic; a single spike day matters less than the same resource showing up repeatedly.", "actions": [ "confirm the cluster/warehouse/pool is still needed and not an orphaned always-on resource (free);", "resolve the resource's name via classic_clusters_config_current or sql_warehouse_config_current and right-size its node type or auto-stop/auto-scale settings (config);", "move steady-state heavy workloads to a reserved/committed-use tier (spend)." ], "next": [ { "query_id": "classic_clusters_config_current", "if": "to resolve cluster_id to a name and config" }, { "query_id": "sql_warehouse_config_current", "if": "to resolve warehouse_id to a name and config" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_by_compute_resource.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_by_compute_resource" }, { "query_id": "cost_by_job", "title": "DBU cost by job", "domain": "cost", "tier": "standard", "stars": true, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_job_dbus_per_day", "default": 50, "meaning": "DBUs/day on a single job that flags WARN" }, { "name": "crit_job_dbus_per_day", "default": 200, "meaning": "DBUs/day that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "usage_metadata.job_id and product_features.is_serverless are documented system.billing.usage columns.", "read_this": "One row = a day + workspace + job's DBU cost. The columns that matter are job_id (which job) and net_usage_quantity (its DBU burn that day) - distinct_runs lets you tell a job that is expensive because it runs constantly apart from one that is expensive because a single run is heavy.", "healthy": "net_usage_quantity below :warn_job_dbus_per_day DBUs/day per job (field heuristic - tune :warn_job_dbus_per_day for your account).", "investigate_if": "net_usage_quantity at/above :warn_job_dbus_per_day (WARN) or :crit_job_dbus_per_day (CRITICAL) DBUs/day - field heuristic; a job that is consistently in-band and just large may be fine, a job whose per-run cost keeps climbing is the one to open first.", "actions": [ "resolve job_id to name/owner/run_as via system.lakeflow.jobs and confirm the job is still needed at this frequency (free);", "move the job off an always-on all-purpose cluster onto job-scoped compute, or tune its cluster size/autoscaling (config);", "if the job is genuinely compute-heavy and correctly sized, consider a committed-use discount for that capacity (spend)." ], "next": [ { "query_id": "lakeflow_jobs_on_all_purpose", "if": "this job is paying the all-purpose placement premium" }, { "query_id": "lakeflow_failed_runs", "if": "distinct_runs is high relative to net_usage_quantity - retries may be inflating the cost" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_by_job.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_by_job" }, { "query_id": "cost_by_notebook", "title": "DBU cost by notebook (ad-hoc usage)", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_notebook_dbus_per_day", "default": 10, "meaning": "DBUs/day on a single notebook that flags WARN" }, { "name": "crit_notebook_dbus_per_day", "default": 50, "meaning": "DBUs/day that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "usage_metadata.notebook_id is documented but populates only for notebook-attached all-purpose/interactive usage - confirm it is non-empty in your account before relying on a per-notebook view.", "read_this": "One row = a day + workspace + notebook's ad-hoc DBU cost. The column that matters is net_usage_quantity - ad-hoc/interactive spend is normally a small slice of total DBUs, so a notebook that clears the WARN/CRITICAL band is the runaway exception, not the norm.", "healthy": "net_usage_quantity below :warn_notebook_dbus_per_day DBUs/day per notebook (field heuristic - tune :warn_notebook_dbus_per_day for your account; ad-hoc spend is expected to be a small share of the total).", "investigate_if": "net_usage_quantity at/above :warn_notebook_dbus_per_day (WARN) or :crit_notebook_dbus_per_day (CRITICAL) DBUs/day - field heuristic for a runaway ad-hoc notebook left running on an expensive all-purpose cluster.", "actions": [ "ask the notebook's owner whether it is meant to run unattended at this size (free);", "attach the notebook to a smaller/auto-terminating all-purpose cluster, or move the workload into a scheduled job (config);", "if the workload is legitimately heavy interactive analysis, size the cluster deliberately rather than leaving it on defaults (spend)." ], "next": [ { "query_id": "cost_by_compute_resource", "if": "to see which cluster is absorbing the notebook's cost" }, { "query_id": "cost_by_job", "if": "to compare against scheduled-job DBU cost in the same window" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_by_notebook.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_by_notebook" }, { "query_id": "cost_by_serving_endpoint", "title": "Usage by model-serving / vector-search endpoint", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "usage_metadata.endpoint_id/endpoint_name are documented for MODEL_SERVING; same paths as cost_serving_mode_by_endpoint.", "read_this": "One row = a day + workspace + endpoint's serving/vector-search usage. The columns that matter are endpoint_id, usage_type (COMPUTE_TIME / GPU_TIME / TOKEN are different units), and net_usage_quantity in that unit - this is the size map behind the model-serving and vector-search share of spend, not a right-sizing verdict on its own.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_vector_search_spend", "if": "for the dollarized VECTOR_SEARCH cut" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_by_serving_endpoint.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_by_serving_endpoint" }, { "query_id": "cost_chargeback_by_identity", "title": "DBU usage by identity (chargeback)", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "identity_metadata fields are documented system.billing.usage columns.", "read_this": "One row = a day + workspace + product + masked identity's DBU usage. The columns that matter are identity_type (user vs service_principal vs unknown) and identity_run_as (the masked principal) - use this to see who is spending, and how much of the spend has no attributable identity at all.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_chargeback_by_tag", "if": "for a tag-based cut of the same window" }, { "query_id": "cost_workspace_names", "if": "to resolve workspace_id to a human name" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_chargeback_by_identity.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_chargeback_by_identity" }, { "query_id": "cost_chargeback_by_tag", "title": "DBU usage by custom tag (chargeback + coverage)", "domain": "cost", "tier": "standard", "stars": true, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "custom_tags is a documented map column on system.billing.usage.", "read_this": "One row = a day + workspace + product + tag key/value pair's DBU usage, including a row per tag_key = NULL where a resource carried no tags at all. The columns that matter are tag_key/tag_value (a real chargeback cut needs a key whose values actually vary) and net_usage_quantity - pair the NULL-tag rows against cost_totals_by_sku_day's total for the same window to get the untagged share.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "for the total DBU denominator to compute an untagged %" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_chargeback_by_tag.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_chargeback_by_tag" }, { "query_id": "cost_cloud_infra", "title": "Estimated billed cost by cloud (DBU-derived)", "domain": "cost", "tier": "deep", "stars": false, "reads": [ "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "system.billing.cloud_infra_cost does not exist in this workspace; this query substitutes a DBU-derived list-price estimate, so the dollar figures here are an estimate, not a reconciled cloud bill.", "read_this": "One row = a day + cloud + currency's estimated billed cost. The column that matters is net_billed_cost, an estimate = usage_quantity x list_prices.pricing.default matched to the price row that was active on that usage row - not a substitute for your cloud provider's own cost export.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "for the same window broken out by SKU/workspace instead of cloud/currency" }, { "query_id": "cost_networking_egress", "if": "for the DBU-billed egress slice specifically" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_cloud_infra.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_cloud_infra" }, { "query_id": "cost_dbsql_allocation_gap", "title": "DBSQL raw vs attributed DBU allocation gap", "domain": "cost", "tier": "deep", "stars": false, "reads": [ "system.billing.usage", "system.billing.attributed_usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.attributed_usage are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "system.billing.attributed_usage's quantity column is active_usage_quantity, not usage_quantity; confirm that column and the billing_origin_product = 'SQL' literal for DBSQL scope on your account before trusting the gap.", "read_this": "One row = either the raw or the attributed DBSQL DBU total for a day + cloud, labeled by source_kind. Difference the two source_kind rows for the same day/cloud yourself: raw minus attributed is the DBU volume Databricks' fair-split allocation could not attribute back to a consuming entity - a large, persistent gap points at cross-subsidized shared SQL-warehouse pools.", "healthy": "n/a - the gap is a difference you compute between this query's 'raw' and 'attributed' rows for the same key, not a single-row band; a small, stable gap is the healthy case (field heuristic).", "investigate_if": "a growing or large raw-minus-attributed gap for the same day/cloud - field heuristic; this needs the two rows differenced yourself, so no per-row status is computed here (see caveats).", "actions": [ "confirm which SQL warehouses are shared/multi-tenant vs single-team, since fair-split allocation struggles most on shared pools (free);", "split a heavily shared warehouse into per-team warehouses, or enable/verify usage-policy tagging on it so attribution has more to work with (config);", "if the gap is small and structural, accept it and document the residual as an allocation cost of shared infrastructure (spend, in the sense of accepted overhead)." ], "next": [ { "query_id": "cost_usage_policy_coverage", "if": "uncovered serverless DBSQL usage is one likely driver of the gap" }, { "query_id": "cost_chargeback_by_identity", "if": "for an identity-level cut of the same SQL usage" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_dbsql_allocation_gap.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_dbsql_allocation_gap" }, { "query_id": "cost_default_storage_dsu", "title": "Default storage (DSU) usage", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "The filter value billing_origin_product = 'DEFAULT_STORAGE' is unverified (billing_origin_product has no published closed enum); storage_api_type and catalog_id are confirmed columns and are used as the primary, safer filter here.", "read_this": "One row = a day + cloud + SKU + storage-API-type + catalog's default-storage usage. The columns that matter are storage_api_type (TIER_1 vs TIER_2 API operations) and net_usage_quantity - this is Unity Catalog default/managed storage cost, not a customer-managed external location.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "for the total spend context" }, { "query_id": "po_vacuum_reclaimed_bytes", "if": "for storage reclaimed by VACUUM in the same window" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_default_storage_dsu.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_default_storage_dsu" }, { "query_id": "cost_dollarized_by_sku_day", "title": "Dollarized cost by SKU and day (list price)", "domain": "cost", "tier": "deep", "stars": true, "reads": [ "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "The nested path list_prices.pricing.effective_list.default is not confirmed as a numerically castable scalar (the schema only types pricing.effective_list as an object); until confirmed, treat net_list_cost as directional and prefer dollarizing from pricing_list_prices_raw instead.", "read_this": "One row = a day + cloud + SKU + product + usage type/unit's usage, priced at list. The columns that matter are net_usage_quantity (the native-unit volume) and net_list_cost (usage_quantity x the list rate) - this is a pre-discount estimate, not what you actually pay.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_actual_vs_list_by_sku", "if": "for a DBU-only actual-vs-list comparison" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_dollarized_by_sku_day.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_dollarized_by_sku_day" }, { "query_id": "cost_genai_token_gpu", "title": "GenAI token and GPU usage", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "The usage_type enum (COMPUTE_TIME, STORAGE_SPACE, NETWORK_BYTE, NETWORK_HOUR, API_OPERATION, TOKEN, GPU_TIME, ANSWER) is documented.", "read_this": "One row = a day + cloud + SKU + usage type's GenAI usage. The columns that matter are usage_type (TOKEN vs GPU_TIME vs ANSWER are different units) and net_usage_quantity in that unit - price each usage_type against its own list_prices SKU row, never the DBU rate.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_vector_search_spend", "if": "for the dollarized VECTOR_SEARCH cut" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_genai_token_gpu.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_genai_token_gpu" }, { "query_id": "cost_networking_egress", "title": "Networking / egress usage", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "usage_metadata.{source_region, destination_region, networking_client, recipient_id} are documented system.billing.usage columns.", "read_this": "One row = a day + cloud + SKU + usage type's billed networking usage. The columns that matter are usage_type (NETWORK_BYTE vs NETWORK_HOUR are different units) and net_usage_quantity - this is the closest billed-egress signal available, not a full network cost reconciliation.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_cloud_infra", "if": "for the broader DBU-derived cloud cost estimate" }, { "query_id": "cost_by_billing_origin_product", "if": "for total usage by product line in the same window" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_networking_egress.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_networking_egress" }, { "query_id": "cost_premium_serverless_photon", "title": "Usage by serverless / Photon / tier premium lever", "domain": "cost", "tier": "standard", "stars": true, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "product_features.{is_serverless, is_photon, jobs_tier, sql_tier, dlt_tier, performance_target} are documented system.billing.usage columns.", "read_this": "One row = a day + cloud + SKU + product's usage cut by serverless/Photon/tier choice. The columns that matter are is_serverless, is_photon, and the *_tier columns (which premium lever, if any, applied) against net_usage_quantity (how much usage ran through that lever).", "healthy": "The lever columns (is_serverless / is_photon / *_tier / performance_target) mostly null or false for a workload that has not opted into a premium path, with net_usage_quantity concentrated on the classic/base rows - a qualitative judgement call, since usage_unit is not restricted here and volumes are not comparable across different sku_name families.", "investigate_if": "A sku_name/product with a large, growing net_usage_quantity on rows where is_serverless/is_photon = true or a *_tier is a premium tier - compare it against that SAME sku_name's classic rows before assuming the premium is unjustified, since raw volumes are not comparable across different SKUs' units.", "actions": [ "confirm the workload actually needs the premium tier (serverless start-up latency, Photon vectorization, DLT Advanced) rather than defaulting to it (free);", "step a workload down to Standard tier or classic compute where the premium is not earning its keep (config);", "if the premium genuinely pays for itself in latency/throughput, budget for it deliberately rather than discovering it after the fact (spend)." ], "next": [ { "query_id": "cost_by_compute_resource", "if": "to see which specific clusters/warehouses carry the premium usage" }, { "query_id": "cost_serving_mode_by_endpoint", "if": "for the model-serving-specific premium/cost-mode cut" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_premium_serverless_photon.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_premium_serverless_photon" }, { "query_id": "cost_restatement_trust_metric", "title": "Billing restatement / trust metric", "domain": "cost", "tier": "lite", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "The record_type enum (ORIGINAL/RETRACTION/RESTATEMENT) and ingestion_date are documented system.billing.usage columns.", "read_this": "One row = a cloud's usage totals for the window, split by record_type. The columns that matter are net_usage_quantity (the true net, corrections already applied) versus original_usage_quantity/retracted_abs_quantity/restatement_usage_quantity (how much of that net was corrected after the fact) - this is a trust/QA check on the billing data itself, not a cost or waste signal.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "for the same net-corrected totals broken out by SKU/workspace" }, { "query_id": "cost_by_billing_origin_product", "if": "for the net totals by product line" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_restatement_trust_metric.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_restatement_trust_metric" }, { "query_id": "cost_serving_mode_by_endpoint", "title": "Model-serving spend by endpoint and inferred cost mode", "domain": "cost", "tier": "deep", "stars": false, "reads": [ "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_endpoint_usd_per_day", "default": 50, "meaning": "estimated list-price $/day on a single endpoint + usage_type that flags WARN" }, { "name": "crit_endpoint_usd_per_day", "default": 250, "meaning": "$/day that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "The billing columns (billing_origin_product, product_features.serving_type, usage_type, usage_metadata.endpoint_name/endpoint_id, the LAUNCH SKU pattern) are all doc-confirmed; the list_rate path (pricing.effective_list.default) is not, so net_list_cost is a directional estimate, not a confirmed dollar figure.", "read_this": "One row = an endpoint + day + usage_type's model-serving usage with an inferred cost mode. The columns that matter are is_launch_sku (a TRUE-heavy endpoint is repeatedly cold-starting from scale-to-zero, which is itself a cost signal) and net_list_cost (est_usd_list, the estimated dollar exposure that drives the band below).", "healthy": "net_list_cost below :warn_endpoint_usd_per_day est_usd_list/day per endpoint + usage_type (field heuristic - tune for your account).", "investigate_if": "net_list_cost at/above :warn_endpoint_usd_per_day (WARN) or :crit_endpoint_usd_per_day (CRITICAL) est_usd_list/day (field heuristic); or net_list_cost is NULL (NOT_ASSESSED - the list-price join found no matching price row, not $0). A high is_launch_sku share alongside a high band is the strongest signal of scale-to-zero churn.", "actions": [ "confirm the endpoint's traffic pattern actually needs to scale from zero this often, or whether a minimum-provisioned-throughput floor would be cheaper (free);", "switch a steadily-busy endpoint from pay-per-token/scale-to-zero to provisioned throughput, or the reverse for a bursty one (config);", "right-size the endpoint's provisioned capacity or model choice (spend)." ], "next": [ { "query_id": "compute_serving_dormant_endpoints", "if": "to check whether a CRITICAL endpoint is actually seeing real traffic" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_serving_mode_by_endpoint.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_serving_mode_by_endpoint" }, { "query_id": "cost_totals_by_sku_day", "title": "Cost totals by SKU and day", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "The record_type enum and product_features.is_serverless are documented system.billing.usage columns.", "read_this": "One row = a day + cloud + workspace + SKU + product + usage type's net usage. The columns that matter are net_usage_quantity (the corrected net) and workspace_id (NULL means an account-level SKU, not a missing value) - this is the finest-grain total this repo collects, and most other cost cuts are a slice of it.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_by_billing_origin_product", "if": "for the same window rolled up to product line only" }, { "query_id": "cost_workspace_names", "if": "to resolve workspace_id to a human name" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_totals_by_sku_day.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_totals_by_sku_day" }, { "query_id": "cost_usage_policy_coverage", "title": "Usage policy coverage for serverless spend", "domain": "cost", "tier": "standard", "stars": false, "reads": [ "system.billing.usage" ], "requires": "SELECT on system.billing; GA (system.billing.usage is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "usage_metadata.{usage_policy_id, budget_policy_id} and product_features.is_serverless are documented system.billing.usage columns.", "read_this": "One row = a day + workspace + product's serverless usage cut by policy and tag coverage. The columns that matter are policy_coverage ('none' on a serverless row is the uncovered gap) and is_serverless (coverage is only meaningful for serverless spend) - tag_coverage is tracked separately, so a warehouse can be tagged yet still show 'none' on policy.", "healthy": "is_serverless = true rows show policy_coverage in ('usage_policy', 'budget_policy_legacy') - field heuristic.", "investigate_if": "is_serverless = true AND policy_coverage = 'none' (WARN) - field heuristic; the fix differs by product, an uncovered notebook/job needs a usage policy attached, an uncovered SQL warehouse needs a tag.", "actions": [ "identify the workspace/product combination driving the uncovered DBUs from this result directly (free);", "attach a usage policy to uncovered INTERACTIVE/JOBS serverless compute, or a tag to an uncovered SQL warehouse (config);", "make policy/tag attachment a provisioning gate (for example via Terraform or a cluster policy) so new serverless usage cannot go uncovered (spend, in the sense of process investment)." ], "next": [ { "query_id": "cost_chargeback_by_tag", "if": "tag coverage is tracked separately from policy coverage here" }, { "query_id": "cost_totals_by_sku_day", "if": "for the total DBU denominator" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_usage_policy_coverage.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_usage_policy_coverage" }, { "query_id": "cost_vector_search_spend", "title": "Vector Search spend by endpoint", "domain": "cost", "tier": "deep", "stars": false, "reads": [ "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.usage and system.billing.list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_endpoint_usd_per_day", "default": 25, "meaning": "estimated list-price $/day on a single Vector Search endpoint that flags WARN" }, { "name": "crit_endpoint_usd_per_day", "default": 100, "meaning": "$/day that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "billing_origin_product='VECTOR_SEARCH', usage_metadata.endpoint_name, and the usage_type enum are confirmed in docs; the list_rate path (pricing.effective_list.default) is not, so net_list_cost is a directional estimate.", "read_this": "One row = a Vector Search endpoint + day + usage type's spend. The columns that matter are usage_type (STORAGE_SPACE is DSU storage, everything else is serving DBUs - different units) and net_list_cost (est_usd_list, the dollarized driver behind the band, which is safe to compare across usage_type because it is already in dollars).", "healthy": "net_list_cost below :warn_endpoint_usd_per_day est_usd_list/day per endpoint (field heuristic - tune for your account).", "investigate_if": "net_list_cost at/above :warn_endpoint_usd_per_day (WARN) or :crit_endpoint_usd_per_day (CRITICAL) est_usd_list/day (field heuristic); or net_list_cost is NULL (NOT_ASSESSED - the list-price join found no matching price row, not $0).", "actions": [ "confirm the endpoint's index is still queried by something (free) - cross-check against real traffic before assuming it is idle;", "shrink or consolidate an oversized/underused index, or right-size its DSU tier (config);", "if traffic genuinely justifies the spend, budget for it deliberately (spend)." ], "next": [ { "query_id": "compute_serving_dormant_endpoints", "if": "to check whether a CRITICAL endpoint is actually receiving traffic before you shrink it" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_vector_search_spend.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_vector_search_spend" }, { "query_id": "cost_workspace_names", "title": "Workspace ID to name lookup", "domain": "cost", "tier": "lite", "stars": false, "reads": [ "system.access.workspaces_latest" ], "requires": "SELECT on system.access; GA (system.access.workspaces_latest is generally available)", "params": [], "confidence": "confirmed", "confidence_note": "workspace_id, workspace_name, and status are documented system.access.workspaces_latest columns.", "read_this": "One row = a workspace. The columns that matter are workspace_id (the join key every other cost/identity query keys on) and workspace_name (the human label to show instead of the numeric id) - status distinguishes an active workspace from a deleted one.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_totals_by_sku_day", "if": "to join workspace_id to workspace_name onto the cost totals" }, { "query_id": "cost_chargeback_by_identity", "if": "to join the same onto the identity chargeback cut" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/cost_workspace_names.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-cost_workspace_names" }, { "query_id": "pricing_list_prices_raw", "title": "Raw Databricks list price history", "domain": "cost", "tier": "lite", "stars": false, "reads": [ "system.billing.list_prices" ], "requires": "SELECT on system.billing; GA (system.billing.list_prices is generally available)", "params": [], "confidence": "confirmed", "confidence_note": "pricing.default, pricing.effective_list, and pricing.promotional are documented system.billing.list_prices columns.", "read_this": "One row = a SKU's price at a point in time, with the full pricing struct preserved as JSON. The columns that matter are pricing_default (the list rate) and price_end_time (NULL means this is the currently active price) - pricing_effective_list_json and pricing_promotional_json carry the raw structs for you to parse yourself.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_actual_vs_list_by_sku", "if": "for a ready-made actual-vs-list comparison built from this same table" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/cost/pricing_list_prices_raw.sql", "learn_url": "/learn/tech/databricks/audit/cost#q-pricing_list_prices_raw" } ], "governance_access": [ { "query_id": "access_admin_role_change_events", "title": "Admin and access-control role-change events", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.audit" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_admin_events", "default": 20, "meaning": "admin/role-change events by one actor+action in the window that flags WARN" }, { "name": "crit_admin_events", "default": 100, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "user_identity.subject_name (not subjectName) was confirmed against a live workspace on 2026-05-30.", "read_this": "One row = an actor (masked identity) x service x action combo in the window. The columns that matter are event_count (how often they did it) and distinct_source_ips (how many different locations it came from).", "healthy": "status = OK; event_count below :warn_admin_events per actor/action (field heuristic - tune :warn_admin_events for your account).", "investigate_if": "status = WARN at/above :warn_admin_events, CRITICAL at/above :crit_admin_events - field heuristic; also worth a look whenever distinct_source_ips is unusually high for one actor.", "actions": [ "confirm the actor and action are expected admin/automation activity and cross-check against your own change log (free);", "if unexpected, review the actor's current grants via access_grants_inventory and rotate credentials if compromise is suspected (config);", "if this is a legitimate but noisy automation script, move it to a dedicated service principal with scoped permissions so it stops tripping this alert (spend/eng time)." ], "next": [ { "query_id": "access_runas_escalation", "if": "the same actor also shows run-as differences" }, { "query_id": "access_grants_inventory", "if": "see what that actor's role change actually granted" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_admin_role_change_events.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_admin_role_change_events" }, { "query_id": "access_classified_unmasked", "title": "Classified-sensitive columns with no mask applied", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.data_classification.results", "system.information_schema.column_masks" ], "requires": "SELECT on system.data_classification and system.information_schema; system.data_classification is Public Preview and requires Unity Catalog; column_masks is Public Preview (DBR 12.2 LTS+)", "params": [], "confidence": "confirmed", "confidence_note": "Columns are doc-confirmed. The residual uncertainty is join completeness (column_masks is privilege-aware), not whether the columns exist.", "read_this": "One row = a HIGH-confidence classified (PII/sensitive) column and whether a column mask covers it. The column that matters is is_unmasked - true means the auto-classifier is confident the column is sensitive and no masking function was found on it.", "healthy": "status = OK; is_unmasked = false (a mask is applied) - field heuristic; confirm by reviewing MASK_NAME per column.", "investigate_if": "status = CRITICAL when is_unmasked = true - field heuristic; prioritize columns with the widest access (cross-check against access_grants_inventory).", "actions": [ "confirm the column is genuinely sensitive by reviewing the class_tag/confidence pair, not the raw values (free);", "attach a column mask function via ALTER TABLE ... ALTER COLUMN ... SET MASK (config);", "if masking many columns at scale, standardize on a small set of reusable mask functions instead of one-offs (spend/eng time)." ], "next": [ { "query_id": "access_column_masks_inventory", "if": "see every mask already in place" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_classified_unmasked.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_classified_unmasked" }, { "query_id": "access_column_lineage_sensitive_reach", "title": "Column-level lineage reach", "domain": "governance_access", "tier": "deep", "stars": false, "reads": [ "system.access.column_lineage" ], "requires": "SELECT on system.access; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_reach_principals", "default": 10, "meaning": "distinct principals driving one source-target column edge that flags WARN" }, { "name": "crit_reach_principals", "default": 50, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "All columns used (source/target table and column names, entity_type, direct_access, created_by, event_time) are confirmed against system.access.column_lineage.", "read_this": "One row = a source column x target column data-flow edge (plus entity_type and direct_access) in the window. The columns that matter are event_count (how often that flow ran) and distinct_principals - how many different identities drove it. This is per-edge, not a table-level rollup; pair it with access_table_lineage_blast_radius for the coarser view.", "healthy": "status = OK; distinct_principals below :warn_reach_principals for one source-target column edge - field heuristic.", "investigate_if": "status = WARN at/above :warn_reach_principals, CRITICAL at/above :crit_reach_principals - field heuristic; cross-check the source column against access_data_classification_inventory before prioritizing, since this query itself carries no sensitivity signal.", "actions": [ "cross-check the source column against access_data_classification_inventory or access_tags_inventory to see if it is actually sensitive (free);", "if it is, tighten grants on the source or add a mask, and check access_pii_propagation_untagged for the specific untagged-target case (config);", "if a column is a genuinely critical, widely-reached asset, formalize it as a governed data product with an owner (spend/eng time)." ], "next": [ { "query_id": "access_pii_propagation_untagged", "if": "the specific untagged-target finding this feeds" }, { "query_id": "access_table_lineage_blast_radius", "if": "the table-level rollup" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_column_lineage_sensitive_reach.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_column_lineage_sensitive_reach" }, { "query_id": "access_column_masks_inventory", "title": "Column masks inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.information_schema.column_masks" ], "requires": "SELECT on system.information_schema; Public Preview, DBR 12.2 LTS+, Unity Catalog required", "params": [], "confidence": "confirmed", "confidence_note": "Columns verified against a live workspace on 2026-05-30: table_catalog, table_schema, table_name, column_name, mask_name, using_columns.", "read_this": "One row = a column in the metastore that has a column mask attached. The columns that matter are mask_name (which masking function) and using_columns (what it reads to decide the mask).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_classified_unmasked", "if": "cross-check which classified columns are missing from this list" }, { "query_id": "access_row_filters_inventory", "if": "the row-level equivalent" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_column_masks_inventory.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_column_masks_inventory" }, { "query_id": "access_data_classification_inventory", "title": "Auto-classified sensitive column inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.data_classification.results" ], "requires": "SELECT on system.data_classification; Public Preview, Unity Catalog required, needs the data-classification feature AND the system.data_classification schema both enabled", "params": [], "confidence": "confirmed", "confidence_note": "Columns confirmed against Databricks documentation for system.data_classification.results.", "read_this": "One row = a column the auto-classifier tagged, deduplicated to one row per catalog/schema/table/column/class_tag/confidence/data_type. The columns that matter are class_tag (what kind of sensitive data) and max_frequency (how consistently it matched).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_classified_unmasked", "if": "see which of these are masked" }, { "query_id": "access_pii_propagation_untagged", "if": "see where this sensitive data propagates from here" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_data_classification_inventory.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_data_classification_inventory" }, { "query_id": "access_dead_table_candidates", "title": "Dead-table cleanup candidates", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.table_lineage", "system.information_schema.tables" ], "requires": "SELECT on system.access and system.information_schema; system.access.table_lineage is GA; information_schema requires Unity Catalog", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_dead_days", "default": 90, "meaning": "days since last altered for a table with no lineage-source hits that flags WARN" }, { "name": "crit_dead_days", "default": 365, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "Lineage undercounts reads that have no captured source (e.g. INSERT ... VALUES literals), so a table absent from lineage is a cleanup candidate to cross-check, not proof it is unused.", "read_this": "One row = a MANAGED or EXTERNAL table that never appeared as a lineage source (was never read) in the window. The columns that matter are days_since_altered (staleness) and table_owner (who to ask before dropping anything).", "healthy": "status = OK; days_since_altered below :warn_dead_days for a table that never appeared as a lineage source - field heuristic; a candidate can still be OK for a while (e.g. quarterly jobs).", "investigate_if": "status = WARN at/above :warn_dead_days, CRITICAL at/above :crit_dead_days, or NOT_ASSESSED when last_altered is NULL (age unknown - do not treat as a finding) - field heuristic; widen :period_days first if your account's retention allows it, since a short lineage window over-flags quarterly/long-tail tables.", "actions": [ "cross-check the table against system.access.audit, query.history, and system.storage.table_metrics_history before touching anything (free);", "if genuinely unused, revoke write access or move it to a to-be-deleted schema for a cooling-off period (config);", "once confirmed dead, formally retire and drop it to reclaim storage spend (spend)." ], "next": [ { "query_id": "access_table_lineage_blast_radius", "if": "see the table's full lineage graph if it does show up elsewhere" }, { "query_id": "po_vacuum_reclaimed_bytes", "if": "reclaim storage once you drop it" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_dead_table_candidates.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_dead_table_candidates" }, { "query_id": "access_grants_inventory", "title": "Table and catalog grants inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.information_schema.table_privileges", "system.information_schema.catalog_privileges" ], "requires": "SELECT on system.information_schema; Unity Catalog required", "params": [], "confidence": "confirmed", "confidence_note": "table_privileges and catalog_privileges are transcribed verbatim - the only two privilege views confirmed column-by-column for this query.", "read_this": "One row = an object scope (TABLE or CATALOG) x privilege type x grantee, rolled up. The columns that matter are grant_count (how many individual grants) and distinct_objects (how many different tables/catalogs that grantee touches).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_grants_inventory_extended", "if": "schema/connection/credential/external-location grants" }, { "query_id": "access_runas_escalation", "if": "cross-check a broadly-granted identity against run-as activity" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_grants_inventory.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_grants_inventory" }, { "query_id": "access_grants_inventory_extended", "title": "Schema, connection, credential, and external-location grants inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.information_schema.schema_privileges", "system.information_schema.connection_privileges", "system.information_schema.credential_privileges", "system.information_schema.external_location_privileges" ], "requires": "SELECT on system.information_schema; Unity Catalog required", "params": [], "confidence": "needs_confirmation", "confidence_note": "Only table_privileges and catalog_privileges (used in access_grants_inventory) are transcribed column-by-column; the object-name columns on these four sibling views (e.g. schema_privileges.catalog_name+schema_name, and the connection/credential/external-location name columns) are inferred, not verified. This query only reads GRANTEE and PRIVILEGE_TYPE, which are shared across every privilege view, so it is safe as written - do not add any object-name column from these views until you DESCRIBE it in your workspace.", "read_this": "One row = an object scope (SCHEMA, CONNECTION, CREDENTIAL, or EXTERNAL_LOCATION) x privilege type x grantee, with a grant count. Use it to see who holds broad integration-level access beyond plain table/catalog grants.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_grants_inventory", "if": "table/catalog grants" }, { "query_id": "access_runas_escalation", "if": "" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_grants_inventory_extended.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_grants_inventory_extended" }, { "query_id": "access_login_concentration", "title": "Login concentration and failed-authentication rollup", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.audit" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_failed_logins", "default": 5, "meaning": "non-success auth events for one principal+source_ip+service+action combo that flags WARN" }, { "name": "crit_failed_logins", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "service_name='accounts' is confirmed; specific action_name values (mfaLogin, tokenLogin, and others) are representative, not a complete enumeration - group by action_name rather than hardcoding a filter list. user_identity.subject_name (not subjectName) was confirmed live 2026-05-30 and is frequently NULL. Whether response.status_code is int or long is unverified, so the <>200 OR NULL comparison is written defensively either way.", "read_this": "One row = a masked principal x source IP x service x action combo in the window. The columns that matter are non_success_count (failed/non-200 attempts) and distinct_source_ips (how many locations that principal authenticated from).", "healthy": "status = OK; non_success_count below :warn_failed_logins for one principal+IP+action - field heuristic; some non-success events are normal (typos, expired tokens).", "investigate_if": "status = WARN at/above :warn_failed_logins, CRITICAL at/above :crit_failed_logins - field heuristic; also worth a look whenever distinct_source_ips is high for a single principal, which this query surfaces but does not score.", "actions": [ "confirm whether the failed attempts are a known automation/CI credential that rotated or expired (free);", "if unexplained, force a credential/token reset for that principal and require MFA (config);", "if this recurs across many principals, invest in a dedicated identity-threat-detection tool or SIEM integration ahead of Databricks-native monitoring (spend)." ], "next": [ { "query_id": "access_runas_escalation", "if": "check if the same principal shows run-as activity" }, { "query_id": "access_admin_role_change_events", "if": "check if they also touched admin roles" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_login_concentration.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_login_concentration" }, { "query_id": "access_network_inbound_denials", "title": "Inbound network policy denials", "domain": "governance_access", "tier": "deep", "stars": false, "reads": [ "system.access.inbound_network" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days (inbound retention caps at 30d regardless)" }, { "name": "warn_denial_count", "default": 10, "meaning": "denials for one rule+path+principal+IP that flags WARN" }, { "name": "crit_denial_count", "default": 50, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "The nested subfield source.ip is unverified - Databricks' docs describe the source struct as having subfields \"including ip, private-link\" but \"including\" is non-exhaustive, so the exact identifier (ip vs ip_address; private-link may need backtick-quoting) has not been confirmed column-by-column. All other columns (policy_outcome, rule_label, request_path, authenticated_as, event_time) are confirmed. A wrong subfield name errors the entire statement, so verify source.ip in your workspace (or drop it) before relying on this query.", "read_this": "One row = a policy outcome x rule x request path x principal x source IP combo in the window. The column that matters is denial_count - how many times that combination was denied at the network edge.", "healthy": "status = OK; denial_count below :warn_denial_count for one rule+path+principal+IP - field heuristic.", "investigate_if": "status = WARN at/above :warn_denial_count, CRITICAL at/above :crit_denial_count - field heuristic; a sudden spike of denials from one source IP against multiple paths is worth prioritizing even below the threshold.", "actions": [ "confirm whether the denied traffic is a misconfigured internal client (bad DNS, wrong endpoint) rather than an external actor (free);", "if the source is unexpected, tighten or add an explicit ingress rule and confirm the policy is not left in DENY_DRY_RUN (config);", "if denials persist from external ranges, engage network/security to add IP-range blocking upstream of Databricks (spend)." ], "next": [ { "query_id": "access_network_outbound_denials", "if": "check the egress side" }, { "query_id": "access_admin_role_change_events", "if": "see if the same principal also touched admin roles" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_network_inbound_denials.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_network_inbound_denials" }, { "query_id": "access_network_outbound_denials", "title": "Outbound network policy denials", "domain": "governance_access", "tier": "deep", "stars": false, "reads": [ "system.access.outbound_network" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days (outbound retention allows up to 365d)" }, { "name": "warn_denial_count", "default": 10, "meaning": "denials to one destination that flags WARN" }, { "name": "crit_denial_count", "default": 50, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "All columns (network_source_type, destination_type, access_type, destination, dns_event.rcode, storage_event.rejection_reason, event_time) are confirmed.", "read_this": "One row = a source type x destination type x access type x destination combo in the window. The column that matters is denial_count - how many times egress to that destination was blocked.", "healthy": "status = OK; denial_count below :warn_denial_count for one destination - field heuristic.", "investigate_if": "status = WARN at/above :warn_denial_count, CRITICAL at/above :crit_denial_count - field heuristic; a new destination_type=STORAGE denial with a non-null storage_rejection_reason is worth a look regardless of volume.", "actions": [ "confirm the destination is genuinely unexpected rather than a known SaaS/API dependency that needs an egress rule (free);", "add an explicit egress allow rule for legitimate destinations, or investigate the workload if it is not (config);", "if repeated denials point at exfiltration attempts, engage security to review the workload's code and rotate any credentials it held (spend)." ], "next": [ { "query_id": "access_network_inbound_denials", "if": "check the ingress side" }, { "query_id": "access_vector_search_traffic", "if": "the blocked destination is a Vector Search endpoint" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_network_outbound_denials.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_network_outbound_denials" }, { "query_id": "access_pii_propagation_untagged", "title": "PII propagation into untagged columns", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.column_lineage", "system.information_schema.column_tags" ], "requires": "SELECT on system.access and system.information_schema; system.access.column_lineage is GA; information_schema requires Unity Catalog", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_pii_gap_events", "default": 5, "meaning": "times a source-to-target untagged-propagation edge fired that flags WARN" }, { "name": "crit_pii_gap_events", "default": 50, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "This account's actual sensitivity tag_name/tag_value convention is not confirmed - the query matches common names (pii, sensitivity, sensitive, data_classification, classification, confidentiality, phi, pci) case-insensitively as a heuristic, written to under-detect rather than invent a match, which is the safe direction.", "read_this": "One row = a specific source column (tagged sensitive) that fed, via direct lineage, into a specific target column carrying no governance tag at all. The columns that matter are event_count (how often that flow ran) and created_by (who to talk to about tagging the target).", "healthy": "status = OK; event_count below :warn_pii_gap_events for one source-to-target propagation edge - field heuristic; any row here is already a gap, so \"healthy\" just means it is not yet a well-established pipeline.", "investigate_if": "status = WARN at/above :warn_pii_gap_events, CRITICAL at/above :crit_pii_gap_events - field heuristic; prioritize edges with a high event_count, since those are recurring, established pipelines, not one-off ad hoc queries.", "actions": [ "confirm the source column really is sensitive and the target genuinely lacks a tag (free);", "tag the target column to match the source's classification, or add a column mask if it should stay untagged but restricted (config);", "if this is a recurring ETL/pipeline pattern, fix the pipeline to propagate tags automatically instead of tagging targets one at a time (spend/eng time)." ], "next": [ { "query_id": "access_tags_inventory", "if": "see the full tag inventory" }, { "query_id": "access_classified_unmasked", "if": "check whether the untagged target is also unmasked" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_pii_propagation_untagged.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_pii_propagation_untagged" }, { "query_id": "access_row_filters_inventory", "title": "Row filters inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.information_schema.row_filters" ], "requires": "SELECT on system.information_schema; Public Preview, DBR 12.2 LTS+, Unity Catalog required", "params": [], "confidence": "confirmed", "confidence_note": "Columns verified against a live workspace on 2026-05-30: table_catalog, table_schema, table_name, filter_name, target_columns.", "read_this": "One row = a table in the metastore that has a row-level access filter attached. The columns that matter are filter_name (which filter function) and target_columns (what it evaluates to decide row visibility).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_column_masks_inventory", "if": "the column-level equivalent" }, { "query_id": "access_grants_inventory", "if": "see who has access to these filtered tables" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_row_filters_inventory.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_row_filters_inventory" }, { "query_id": "access_runas_escalation", "title": "Run-as escalation: initiator vs executed-as identity", "domain": "governance_access", "tier": "deep", "stars": true, "reads": [ "system.access.audit" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_runas_events", "default": 5, "meaning": "run_by != run_as events for one identity pair in the window that flags WARN" }, { "name": "crit_runas_events", "default": 25, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "identity_metadata.run_by and run_as are read directly off system.access.audit; no inferred column names here.", "read_this": "One row = a masked run_by/run_as identity pair x service x action, where the principal that initiated an action (run_by) differs from the identity it executed as (run_as). The column that matters is event_count - how often that specific pair fired in the window.", "healthy": "status = OK; event_count below :warn_runas_events for one run_by/run_as pair - field heuristic; a low, steady count is often a legitimate job running as a service principal.", "investigate_if": "status = WARN at/above :warn_runas_events, CRITICAL at/above :crit_runas_events - field heuristic; also treat a ZERO-ROW result with suspicion rather than relief - see caveats.", "actions": [ "confirm whether the run_as identity is a known service principal configured for that job/pipeline (free);", "if unexpected, revoke the run_by principal's ability to act as that identity and review its grants via access_grants_inventory (config);", "if run-as patterns are hard to reason about at scale, invest in a naming/tagging convention for service principals so legitimate run-as pairs are self-documenting (spend/eng time)." ], "next": [ { "query_id": "access_admin_role_change_events", "if": "check if the same run_by identity also touched admin roles" }, { "query_id": "access_grants_inventory", "if": "see what the run_as identity can do" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_runas_escalation.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_runas_escalation" }, { "query_id": "access_table_lineage_blast_radius", "title": "Table-level lineage blast radius", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.table_lineage" ], "requires": "SELECT on system.access; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_blast_principals", "default": 10, "meaning": "distinct principals driving one source-target edge that flags WARN" }, { "name": "crit_blast_principals", "default": 50, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "source_table_full_name is NULL for write-only events and target_table_full_name is NULL for read-only events; the access_class CASE reproduces Databricks' documented read/write rule from those two columns.", "read_this": "One row = a source table x target table data-flow edge (plus type, entity_type, and direct_access) in the window. The columns that matter are access_class (READ/WRITE/READ_WRITE/UNKNOWN) and distinct_principals - how many different identities drove that specific flow. This is per-edge, not a full fan-out count; to see everything one source table touches, aggregate this result by source_table_full_name yourself.", "healthy": "status = OK; distinct_principals below :warn_blast_principals for one source-target edge - field heuristic.", "investigate_if": "status = WARN at/above :warn_blast_principals, CRITICAL at/above :crit_blast_principals - field heuristic; an edge with direct_access=true, a sensitive-looking source, and high distinct_principals is the highest-priority combination to review.", "actions": [ "confirm the source table's sensitivity via access_data_classification_inventory before treating a wide edge as risky (free);", "tighten grants on the source table if the fan-out is broader than intended (config);", "if this is a genuinely critical shared table, formalize it as a governed data product with an owner and change-review process (spend/eng time)." ], "next": [ { "query_id": "access_column_lineage_sensitive_reach", "if": "the column-level equivalent" }, { "query_id": "access_pii_propagation_untagged", "if": "check if this edge is also an untagged PII gap" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_table_lineage_blast_radius.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_table_lineage_blast_radius" }, { "query_id": "access_tags_inventory", "title": "Manual governance tags inventory", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.information_schema.column_tags", "system.information_schema.table_tags" ], "requires": "SELECT on system.information_schema; Unity Catalog required", "params": [], "confidence": "confirmed", "confidence_note": "column_tags and table_tags are the two tag views transcribed verbatim.", "read_this": "One row = a tag name/value pair at column or table scope, with how many objects carry it. Use it to see your account's actual tagging vocabulary in practice.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "access_pii_propagation_untagged", "if": "see where tagged sensitive data flows into untagged targets" }, { "query_id": "access_data_classification_inventory", "if": "the auto-detected counterpart to these manual tags" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_tags_inventory.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_tags_inventory" }, { "query_id": "access_vector_search_traffic", "title": "Vector Search query and scan traffic by endpoint", "domain": "governance_access", "tier": "standard", "stars": false, "reads": [ "system.access.audit" ], "requires": "SELECT on system.access; Public Preview", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days." } ], "confidence": "confirmed", "confidence_note": "service_name='vectorSearch' and the query/scan action_name set (queryVectorIndex, queryVectorIndexNextPage, queryVectorIndexRouteOptimized, scanVectorIndex, scanVectorIndexRouteOptimized) are confirmed against Databricks' documentation on unused Vector Search endpoints.", "read_this": "One row = an endpoint x action x day, with how many query/scan events it received. The column that matters is event_count - join this against your own billing data to tell a provisioned-but-unqueried endpoint (bills spend, zero rows here) from a genuinely idle one.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "cost_vector_search_spend", "if": "join this against spend to find endpoints that bill but never show up here" }, { "query_id": "compute_serving_dormant_endpoints", "if": "the general serving-endpoint idle-detection sibling" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/governance_access/access_vector_search_traffic.sql", "learn_url": "/learn/tech/databricks/audit/governance_access#q-access_vector_search_traffic" } ], "jobs_pipelines": [ { "query_id": "lakeflow_failed_jobs_wasted_dbus", "title": "Failed jobs ranked by wasted DBUs", "domain": "jobs_pipelines", "tier": "deep", "stars": true, "reads": [ "system.lakeflow.job_run_timeline", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; GA (job_run_timeline and billing.usage/list_prices are generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_failed_runs", "default": 3, "meaning": "failed runs for a job in the window that flags WARN" }, { "name": "crit_failed_runs", "default": 10, "meaning": "failed runs that flags CRITICAL" }, { "name": "warn_wasted_dbus", "default": 10, "meaning": "wasted-DBU proxy that flags WARN" }, { "name": "crit_wasted_dbus", "default": 50, "meaning": "wasted-DBU proxy that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "usage_metadata carries job_id but not run_id, so wasted DBUs are apportioned across the job by failed-run share rather than measured per failed run; job_id is also unique only within a workspace, so every join here keys on (workspace_id, job_id).", "read_this": "One row = a job with at least one failed run in the window. The column that matters is wasted_dbus_proxy - the job's net DBUs scaled by the share of its runs that failed; est_usd_list turns that into a rough list-price dollar figure. last_failed_termination_code tells you why the most recent failure happened.", "healthy": "few failed_runs and a low wasted_dbus_proxy relative to :warn_wasted_dbus - field heuristic; tune :warn_failed_runs/:warn_wasted_dbus for your account.", "investigate_if": "failed_runs at/above :warn_failed_runs or wasted_dbus_proxy at/above :warn_wasted_dbus (WARN); either at/above :crit_failed_runs / :crit_wasted_dbus (CRITICAL) - field heuristic.", "actions": [ "read last_failed_termination_code and fix the root cause in the job code or config (free);", "add a retry cap or failure alert so a broken job stops burning DBUs unnoticed (config);", "if the job needs more headroom (timeouts, cluster sizing) to stop failing, resize its cluster (spend)." ], "next": [ { "query_id": "lakeflow_termination_taxonomy", "if": "for the org-wide termination_code breakdown" }, { "query_id": "lakeflow_retries_repairs", "if": "to see how many attempts each failing run actually took" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_failed_jobs_wasted_dbus.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_failed_jobs_wasted_dbus" }, { "query_id": "lakeflow_failed_runs", "title": "Failed job runs by workspace and termination code", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA (system.lakeflow.job_run_timeline is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_failed_run_rows", "default": 5, "meaning": "failed run rows in a group that flags WARN" }, { "name": "crit_failed_run_rows", "default": 20, "meaning": "failed run rows that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The result_state filter and the run_rows-vs-distinct_runs distinction were verified against system.lakeflow.job_run_timeline in a live workspace.", "read_this": "One row = a (workspace, run_type, trigger_type, result_state, termination_code) combination in the window. The column that matters is failed_run_rows - how many end rows landed in FAILED/ERROR/TIMED_OUT for that combination; distinct_runs corrects for retries repeating a run_id.", "healthy": "failed_run_rows low relative to distinct_runs, spread across many termination_codes rather than piled on one - field heuristic; tune :warn_failed_run_rows for your account.", "investigate_if": "failed_run_rows at/above :warn_failed_run_rows (WARN) or :crit_failed_run_rows (CRITICAL) for a single termination_code - field heuristic; one code dominating points at a systemic cause, not one-off flakiness.", "actions": [ "open the top termination_code and read its job's recent run logs for the actual error (free);", "fix retry/backoff or dependency config so transient causes stop recurring (config);", "if failures trace back to under-provisioned compute, resize the job cluster (spend)." ], "next": [ { "query_id": "lakeflow_termination_taxonomy", "if": "for the account-wide termination_code breakdown" }, { "query_id": "lakeflow_failed_jobs_wasted_dbus", "if": "for the DBUs each failing job is burning" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_failed_runs.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_failed_runs" }, { "query_id": "lakeflow_health_rule_coverage", "title": "Job health-rule coverage across active jobs", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.jobs" ], "requires": "SELECT on system.lakeflow; GA (system.lakeflow.jobs is generally available; health_rules was added late Nov 2025)", "params": [ { "name": "warn_coverage_pct", "default": 50, "meaning": "percent of active jobs with a health rule below which a workspace flags WARN" }, { "name": "crit_coverage_pct", "default": 20, "meaning": "percent below which it flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "health_rules is documented as an array/struct column; whether CARDINALITY(health_rules) > 0 is the right \"has a rule\" test on this account is unverified, and the column is not populated before late Nov 2025.", "read_this": "One row = a workspace. The columns that matter are jobs_with_health_rule vs active_jobs (coverage), and jobs_health_rules_null, which tells you whether the column is populated at all on this account before you trust the coverage number.", "healthy": "jobs_health_rules_null is 0 (column populated) and jobs_with_health_rule / active_jobs stays above :warn_coverage_pct percent - field heuristic; tune for your account.", "investigate_if": "jobs_health_rules_null > 0 (coverage is not assessable yet - column not populated), or coverage falls below :warn_coverage_pct percent (WARN) / :crit_coverage_pct percent (CRITICAL) - field heuristic.", "actions": [ "pick the highest-DBU jobs with no health rule and add a duration/failure health rule to each (free);", "make health-rule configuration part of your job-creation checklist or template (config);", "n/a - this finding does not itself justify new spend." ], "next": [ { "query_id": "lakeflow_job_ownership_orphans", "if": "for the related governance-coverage picture" }, { "query_id": "lakeflow_jobs_no_timeout", "if": "another active-job config gap worth checking alongside" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_health_rule_coverage.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_health_rule_coverage" }, { "query_id": "lakeflow_job_ownership_orphans", "title": "Job ownership mismatches and orphan run-as candidates", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.jobs" ], "requires": "SELECT on system.lakeflow; GA (system.lakeflow.jobs is generally available; creator_user_name/run_as_user_name were added late Nov 2025)", "params": [ { "name": "warn_orphan_jobs", "default": 5, "meaning": "combined mismatch+orphan job count in a workspace that flags WARN" }, { "name": "crit_orphan_jobs", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "creator_user_name and run_as_user_name are not populated before late Nov 2025, and FedRAMP/redacted workspaces emit '__REDACTED__' for masked identities; both cases are normalized to NULL before counting so they never register as a real owner or a real mismatch.", "read_this": "One row = a workspace. The columns that matter are jobs_owner_mismatch (creator and run-as identity both known and different - a handoff or service-principal pattern worth a look) and jobs_orphan_runas_unknown (run-as identity unknown while the creator is known - nobody clearly owns execution).", "healthy": "jobs_owner_mismatch and jobs_orphan_runas_unknown both low relative to active_jobs, and jobs_creator_null/jobs_runas_null near 0 so the picture is trustworthy - field heuristic; tune :warn_orphan_jobs for your account.", "investigate_if": "(jobs_owner_mismatch + jobs_orphan_runas_unknown) at/above :warn_orphan_jobs (WARN) or :crit_orphan_jobs (CRITICAL) - field heuristic; this is a governance posture signal, not a cost figure.", "actions": [ "list the flagged jobs and confirm each mismatch/orphan is an intentional handoff or service-principal run-as (free);", "require run_as to be set explicitly (not inherited) in your job-creation template (config);", "n/a - this finding does not itself justify new spend." ], "next": [ { "query_id": "lakeflow_health_rule_coverage", "if": "for the related governance-coverage picture" }, { "query_id": "lakeflow_stale_zombie_jobs", "if": "orphaned jobs are often also stale" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_job_ownership_orphans.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_job_ownership_orphans" }, { "query_id": "lakeflow_job_queue_time", "title": "Job queue time by workspace and job", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA (queue_duration_seconds was added late Nov 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_queue_p95_s", "default": 60, "meaning": "95th-percentile queue seconds that flags WARN" }, { "name": "crit_queue_p95_s", "default": 300, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "queue_duration_seconds is not populated before late Nov 2025, so on a short-history account it is NULL for every run; runs_queue_null exposes that so this query degrades instead of reading NULL as zero queue time.", "read_this": "One row = a job in the window. The column that matters is queue_s_p95 - the 95th-percentile seconds a run waited before starting; queue_s_total is the sum across all its runs. Consistently high p95 on the same job means it is waiting on capacity, not a one-off traffic spike.", "healthy": "queue_s_p95 below :warn_queue_p95_s seconds - field heuristic; tune :warn_queue_p95_s for your account.", "investigate_if": "queue_s_p95 at/above :warn_queue_p95_s (WARN) or :crit_queue_p95_s (CRITICAL) seconds - field heuristic; also check runs_queue_null before trusting a \"0\" queue time on a short-history account.", "actions": [ "stagger the job's schedule off the top of the hour and away from other jobs sharing the same warehouse (free);", "move the job onto its own job cluster or a warehouse with autoscaling headroom (config);", "add cluster capacity or move to serverless so runs stop queuing behind each other (spend)." ], "next": [ { "query_id": "lakeflow_phase_cold_start", "if": "for the full setup/queue/execution/cleanup breakdown" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_job_queue_time.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_job_queue_time" }, { "query_id": "lakeflow_job_tasks_no_timeout", "title": "Job tasks with no configured timeout, at-risk DBU exposure", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.job_tasks", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; GA (job_tasks.timeout_seconds was added late Nov 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "billing look-back window for the cost rollup only (does not change this query's grain/filters/counts)" }, { "name": "warn_no_timeout_tasks", "default": 5, "meaning": "no-timeout tasks per workspace that flags WARN" }, { "name": "crit_no_timeout_tasks", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "timeout_seconds is not populated before late Nov 2025; tasks_timeout_null exposes that so a short-history account degrades instead of reading NULL as \"no timeout\". net_dbus/est_usd_list here are an upper-bound exposure figure, not the exact cost of the untimed tasks - see caveats.", "read_this": "One row = a workspace. The column that matters is tasks_no_timeout - active tasks with no configured (or zero-second) timeout; net_dbus/est_usd_list next to it are the at-risk DBUs of any job that contains at least one such task.", "healthy": "tasks_no_timeout near 0 relative to active_tasks - field heuristic; tune :warn_no_timeout_tasks for your account.", "investigate_if": "tasks_no_timeout at/above :warn_no_timeout_tasks (WARN) or :crit_no_timeout_tasks (CRITICAL) - field heuristic.", "actions": [ "list the flagged tasks and set an explicit timeout on each in the job UI/API (free);", "add a default task timeout to your job-creation template or CI job-spec linter (config);", "n/a - fixing this is free; it prevents future spend rather than requiring new spend." ], "next": [ { "query_id": "lakeflow_jobs_no_timeout", "if": "the job-level version of this same gap" }, { "query_id": "lakeflow_tasks_near_timeout", "if": "for tasks that DO have a timeout but are running close to it" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_job_tasks_no_timeout.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_job_tasks_no_timeout" }, { "query_id": "lakeflow_jobs_no_timeout", "title": "Jobs with no configured timeout, at-risk DBU exposure", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.jobs", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; GA (timeout_seconds was added early Dec 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "cost-lookback window for the flagged-subset DBU/dollar rollup" }, { "name": "warn_no_timeout_jobs", "default": 5, "meaning": "no-timeout jobs per workspace that flags WARN" }, { "name": "crit_no_timeout_jobs", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "timeout_seconds is not populated before early Dec 2025; jobs_timeout_null is reported separately so this query degrades to \"not assessed\" instead of over-claiming \"no timeout\" on old records - confirm the NULL semantics on your account.", "read_this": "One row = a workspace. The column that matters is jobs_no_timeout - active jobs with no configured (or zero-second) timeout; net_dbus/est_usd_list next to it are the DBUs/list-price dollars attributed to just that flagged subset over the cost-lookback window.", "healthy": "jobs_no_timeout near 0 relative to active_jobs - field heuristic; tune :warn_no_timeout_jobs for your account.", "investigate_if": "jobs_no_timeout at/above :warn_no_timeout_jobs (WARN) or :crit_no_timeout_jobs (CRITICAL), especially where est_usd_list is also high - field heuristic.", "actions": [ "set an explicit timeout on each flagged job (free);", "add a default job timeout to your job-creation template or CI job-spec linter (config);", "n/a - fixing this is free; it prevents future spend from a runaway job rather than requiring new spend." ], "next": [ { "query_id": "lakeflow_job_tasks_no_timeout", "if": "the task-level version of this same gap" }, { "query_id": "lakeflow_stale_zombie_jobs", "if": "a no-timeout job that also never runs is lower priority" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_jobs_no_timeout.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_jobs_no_timeout" }, { "query_id": "lakeflow_jobs_on_all_purpose", "title": "Jobs running on all-purpose (interactive) clusters", "domain": "jobs_pipelines", "tier": "standard", "stars": true, "reads": [ "system.lakeflow.job_task_run_timeline", "system.compute.clusters", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.compute, system.billing; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "crit_share_usd", "default": 100, "meaning": "naive per-job list-$ share on an all-purpose cluster that flags CRITICAL" }, { "name": "top_n", "default": 500, "meaning": "row cap" } ], "confidence": "needs_confirmation", "confidence_note": "The cross-domain join to system.compute.clusters and whether compute_ids is populated on older rows are the two things to confirm on your own workspace.", "read_this": "One row = one (job, all-purpose cluster) placement over the window. cluster_source is the column that matters - UI or API means the job ran on an interactive/all-purpose cluster (the anti-pattern) instead of a cheaper jobs cluster. est_usd_list_share is a naive even split of the shared cluster's list-price cost across the jobs on it.", "healthy": "cluster_source = JOB, i.e. the task ran on dedicated jobs compute - est_usd_list_share is then informational only (field heuristic).", "investigate_if": "cluster_source IN ('UI','API') (WARN), especially with est_usd_list_share at/above :crit_share_usd (CRITICAL) - a job pinned to an always-on interactive cluster (field heuristic). The trailing NOT_ASSESSED summary row counts runs dropped for null compute_ids; do not read those as \"clean\".", "actions": [ "point the job at a job cluster / new-cluster-per-run instead of an existing all-purpose cluster (free, in the job definition);", "if the interactive cluster exists only to serve jobs, add auto-termination or delete it (config);", "if shared interactive compute is genuinely required, move the heavy jobs to a right-sized jobs cluster (spend / reshape)." ], "next": [ { "query_id": "cost_by_job", "if": "to dollarize the same jobs' DBUs" }, { "query_id": "lakeflow_failed_jobs_wasted_dbus", "if": "these jobs also fail" }, { "query_id": "compute_warehouse_idle_gaps", "if": "the shared cluster sits idle between runs" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_jobs_on_all_purpose.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_jobs_on_all_purpose" }, { "query_id": "lakeflow_never_started_runs", "title": "Runs that never started execution", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_never_started", "default": 3, "meaning": "never-started runs for a (job, termination_code) pair that flags WARN" }, { "name": "crit_never_started", "default": 10, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The period_start_time = period_end_time signature for a never-executed run, and termination_code as the reason, were verified against system.lakeflow.job_run_timeline in a live workspace.", "read_this": "One row = a (workspace, job, termination_code) combination. The column that matters is never_started_runs - runs that were created but never began executing (queue rejection, quota hit, config error) before terminating.", "healthy": "never_started_runs at/near 0 for every job - field heuristic; tune :warn_never_started for your account.", "investigate_if": "never_started_runs at/above :warn_never_started (WARN) or :crit_never_started (CRITICAL) for a job/termination_code pair - field heuristic; check termination_code first, it usually names the blocker (queue/quota/config).", "actions": [ "read the termination_code and, if it is a quota/limit code, check for a scheduling pile-up on that job (free);", "fix the underlying config error or raise the relevant workspace limit (config);", "if the block is capacity (queue/cluster limits), add capacity (spend)." ], "next": [ { "query_id": "lakeflow_termination_taxonomy", "if": "for the account-wide termination_code picture" }, { "query_id": "lakeflow_job_queue_time", "if": "queueing and never-started runs often share a capacity root cause" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_never_started_runs.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_never_started_runs" }, { "query_id": "lakeflow_phase_cold_start", "title": "Job run phase breakdown - setup, queue, execution, cleanup", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA (the five *_duration_seconds columns were added early Dec 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_setup_p95_s", "default": 60, "meaning": "95th-percentile setup seconds that flags WARN" }, { "name": "crit_setup_p95_s", "default": 300, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The five phase columns and the queue-vs-job_task_run_timeline scoping were verified against system.lakeflow.job_run_timeline in a live workspace.", "read_this": "One row = a job in the window. The column that matters is setup_s_p95 - the 95th-percentile cold-start (cluster setup) seconds before a run's code starts executing; execution_s_total is how much of the run's time was actual work.", "healthy": "setup_s_p95 below :warn_setup_p95_s seconds - field heuristic; tune :warn_setup_p95_s for your account.", "investigate_if": "setup_s_p95 at/above :warn_setup_p95_s (WARN) or :crit_setup_p95_s (CRITICAL) seconds - field heuristic; also check rows_setup_null before trusting a low number on a short-history account.", "actions": [ "switch the job to a job cluster/pool that is already warm, or share a cluster across tasks in the same job (free);", "enable a cluster pool or serverless compute to cut cold-start time (config);", "keep a small always-on pool of pre-warmed instances if setup latency is business-critical (spend)." ], "next": [ { "query_id": "lakeflow_workload_mix_hours", "if": "to see whether the affected job runs often enough to be worth the fix" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_phase_cold_start.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_phase_cold_start" }, { "query_id": "lakeflow_pipeline_cost", "title": "Per-pipeline DBU cost and update/refresh volume", "domain": "jobs_pipelines", "tier": "deep", "stars": false, "reads": [ "system.billing.usage", "system.billing.list_prices", "system.lakeflow.pipelines", "system.lakeflow.pipeline_update_timeline" ], "requires": "SELECT on system.billing, system.lakeflow; GA (system.billing.usage/list_prices); Public Preview (system.lakeflow.pipelines and pipeline_update_timeline - a missing/disabled table degrades names/updates to NULL, never the DBU attribution)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_pipeline_dbus", "default": 500, "meaning": "net pipeline DBUs in the window that flags WARN" }, { "name": "crit_pipeline_dbus", "default": 2000, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "the list_rate CAST path (pricing.effective_list.default) is unverified, so net_list_cost is a directional list-price estimate, never a billed dollar figure; confirm the struct path against your workspace's system.billing.list_prices schema.", "read_this": "One row = a pipeline. The column that matters is net_pipeline_dbus - net DBUs billed to that pipeline in the window; net_maintenance_dbus is the housekeeping share of that total, broken out separately; updates/active_seconds_total are the over-refresh signal (many short updates can mean the pipeline is triggered more often than the data actually changes).", "healthy": "net_pipeline_dbus below :warn_pipeline_dbus for the window, with maintenance a small share of the total - field heuristic; tune :warn_pipeline_dbus for your account.", "investigate_if": "net_pipeline_dbus at/above :warn_pipeline_dbus (WARN) or :crit_pipeline_dbus (CRITICAL) - field heuristic; also look at updates vs active_seconds_total for over-refresh.", "actions": [ "check the pipeline's trigger interval against how often the source data actually changes and space it out if it is over-refreshing (free);", "switch a continuously-triggered pipeline to a scheduled/triggered pipeline if continuous freshness is not required (config);", "if the pipeline is correctly sized and still expensive, revisit serverless/Photon settings or move it to a cheaper SKU tier (spend)." ], "next": [ { "query_id": "lakeflow_pipeline_idle_tail_duration", "if": "for cluster lingering after the pipeline's active window" }, { "query_id": "lakeflow_pipeline_update_failures_retries", "if": "for how much of that DBU spend is retries" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_pipeline_cost.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_pipeline_cost" }, { "query_id": "lakeflow_pipeline_idle_tail_duration", "title": "Pipeline update activity and idle-tail DBU exposure", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.pipeline_update_timeline", "system.lakeflow.pipelines", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; Public Preview (system.lakeflow.pipelines and pipeline_update_timeline); GA (system.billing.usage/list_prices)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_idle_dbus", "default": 100, "meaning": "net DBUs in the window on a non-continuous pipeline that flags WARN" }, { "name": "crit_idle_dbus", "default": 500, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "the settings.continuous / settings.development dot-access assumes settings is a STRUCT on system.lakeflow.pipelines; whether it is a STRUCT (dot-access) or a MAP (settings['key']) on your account is unverified.", "read_this": "One row = a pipeline. The columns that matter are updates/active_seconds_total (how much the pipeline actually ran) next to net_dbus (what it was billed) - a large gap between billed DBUs and active update time on a non-continuous pipeline suggests the cluster is lingering after the run (idle tail) rather than shutting down promptly.", "healthy": "net_dbus roughly tracks active_seconds_total for non-continuous pipelines, and stays below :warn_idle_dbus for the window - field heuristic; tune :warn_idle_dbus for your account.", "investigate_if": "net_dbus at/above :warn_idle_dbus (WARN) or :crit_idle_dbus (CRITICAL) on a pipeline where setting_continuous is false and active_seconds_total looks small relative to net_dbus - field heuristic.", "actions": [ "check the pipeline's cluster auto-termination / shutdown-delay setting (free);", "shorten the idle shutdown delay or move the pipeline to serverless DLT compute (config);", "if the workload genuinely needs a warm cluster between runs, that is a deliberate spend - just confirm it is intentional (spend)." ], "next": [ { "query_id": "lakeflow_pipeline_cost", "if": "for the full per-pipeline DBU picture" }, { "query_id": "lakeflow_pipelines_inventory_tier", "if": "for the pipeline's serverless/continuous/development settings" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_pipeline_idle_tail_duration.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_pipeline_idle_tail_duration" }, { "query_id": "lakeflow_pipeline_update_failures_retries", "title": "Pipeline update failures and retry triggers", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.pipeline_update_timeline" ], "requires": "SELECT on system.lakeflow; Public Preview (system.lakeflow.pipeline_update_timeline)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_failed_updates", "default": 3, "meaning": "failed update rows in a group that flags WARN" }, { "name": "crit_failed_updates", "default": 10, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "result_state values (COMPLETED/FAILED/CANCELED - one L), update_type values (FULL_REFRESH/REFRESH/VALIDATE), and trigger_type='RETRY_ON_FAILURE' were verified against system.lakeflow.pipeline_update_timeline in a live workspace.", "read_this": "One row = a (workspace, pipeline, update_type, trigger_type, result_state) combination. The column that matters is failed_update_rows; retry_triggered_rows next to it tells you how many of the group's updates were themselves a retry, not an original attempt.", "healthy": "failed_update_rows low relative to updates, with retry_triggered_rows explaining most of any repeats - field heuristic; tune :warn_failed_updates for your account.", "investigate_if": "failed_update_rows at/above :warn_failed_updates (WARN) or :crit_failed_updates (CRITICAL) for a pipeline - field heuristic; a pipeline that fails and keeps auto-retrying without ever completing is the priority case.", "actions": [ "open the pipeline's update history and read the failure reason on the most recent FAILED update (free);", "fix the upstream data/schema issue or expectation causing the failure (config);", "if failures trace back to under-provisioned compute, resize the pipeline's cluster (spend)." ], "next": [ { "query_id": "lakeflow_pipeline_cost", "if": "to see the DBU cost of a pipeline that keeps retrying" }, { "query_id": "lakeflow_pipeline_idle_tail_duration", "if": "for the same pipeline's idle/active time split" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_pipeline_update_failures_retries.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_pipeline_update_failures_retries" }, { "query_id": "lakeflow_pipelines_inventory_tier", "title": "Pipeline inventory by type, edition, and key settings", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.pipelines" ], "requires": "SELECT on system.lakeflow; Public Preview (system.lakeflow.pipelines)", "params": [], "confidence": "needs_confirmation", "confidence_note": "whether settings is a STRUCT (dot-access, used here) or a MAP (settings['key']) on your account is unverified; confirm before trusting the setting_* columns.", "read_this": "One row = a distinct combination of workspace, pipeline_type, and key settings, with pipelines as the count. Use this to see your pipeline mix (serverless vs classic, continuous vs triggered, dev vs prod) before drilling into a cost or idle-tail finding.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "lakeflow_pipeline_cost", "if": "drill into DBU cost per pipeline" }, { "query_id": "lakeflow_pipeline_idle_tail_duration", "if": "drill into idle-tail exposure per pipeline" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_pipelines_inventory_tier.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_pipelines_inventory_tier" }, { "query_id": "lakeflow_retries_repairs", "title": "Job run retries and repair attempts, with DBU cost", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.job_run_timeline", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_total_retries", "default": 5, "meaning": "total retries for a job in the window that flags WARN" }, { "name": "crit_total_retries", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The retry count formula (non-NULL-result_state rows per run_id, minus 1) was verified against the documented visibility caveat for system.lakeflow.job_run_timeline.", "read_this": "One row = a job in the window. The column that matters is total_retries - extra attempt rows beyond the first, summed across all the job's runs; runs_with_retry tells you how many distinct runs needed at least one retry, and net_dbus/est_usd_list is the job's overall DBU cost for context.", "healthy": "total_retries low relative to distinct_runs - field heuristic; tune :warn_total_retries for your account.", "investigate_if": "total_retries at/above :warn_total_retries (WARN) or :crit_total_retries (CRITICAL) - field heuristic; a job where runs_with_retry is close to distinct_runs is retrying on (almost) every run, which usually points at a systemic cause.", "actions": [ "read the failure reason on a recently-retried run and fix the root cause (free);", "tune the job's retry policy (max retries, backoff) so it fails fast instead of burning attempts on a non-transient error (config);", "if retries trace back to under-provisioned compute, resize the job cluster (spend)." ], "next": [ { "query_id": "lakeflow_failed_jobs_wasted_dbus", "if": "for the DBU-waste view of the same failing jobs" }, { "query_id": "lakeflow_termination_taxonomy", "if": "for the account-wide termination_code picture behind the retries" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_retries_repairs.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_retries_repairs" }, { "query_id": "lakeflow_stale_zombie_jobs", "title": "Stale / zombie jobs with no recent runs", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.jobs", "system.lakeflow.job_run_timeline", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.lakeflow, system.billing; GA", "params": [ { "name": "stale_days", "default": 30, "meaning": "days since last run that flags WARN" }, { "name": "crit_stale_days", "default": 90, "meaning": "days since last run that flags CRITICAL" }, { "name": "period_days", "default": 30, "meaning": "cost-lookback window for net_dbus/est_usd_list (a separate window from the staleness thresholds)" } ], "confidence": "confirmed", "confidence_note": "the SCD2-latest-row + delete_time filter and the last-run lookback (independent of the cost window) were verified against system.lakeflow.jobs and job_run_timeline in a live workspace.", "read_this": "One row = an active job. The column that matters is last_run_start - the most recent time the job ran, across all history; is_stale flags jobs that have not run in over :stale_days days (or never at all).", "healthy": "last_run_start recent (within :stale_days days) for every active job - field heuristic; tune :stale_days for your account.", "investigate_if": "is_stale = 1, especially where last_run_start is NULL (no run ever recorded) or older than :crit_stale_days days - field heuristic.", "actions": [ "confirm the job is genuinely unused and pause or delete it (free);", "if it is meant to run on a longer cadence, document that in the job name/description so it stops looking abandoned (config);", "n/a - a stale job is not itself a spend decision (it correctly shows ~0 net_dbus, see caveats)." ], "next": [ { "query_id": "lakeflow_job_ownership_orphans", "if": "stale jobs are often also ownership orphans" }, { "query_id": "lakeflow_health_rule_coverage", "if": "a stale job is a low priority for health-rule coverage effort" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_stale_zombie_jobs.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_stale_zombie_jobs" }, { "query_id": "lakeflow_succeeded_with_failed_tasks", "title": "Runs that succeeded overall despite a failed task", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline", "system.lakeflow.job_task_run_timeline" ], "requires": "SELECT on system.lakeflow; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_succeeded_with_failed", "default": 3, "meaning": "such runs for a job that flags WARN" }, { "name": "crit_succeeded_with_failed", "default": 10, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "the join key (job_task_run_timeline.job_run_id = job_run_timeline.run_id, plus workspace_id) and the end-row filter on both sides were verified against a live workspace.", "read_this": "One row = a job in the window. The column that matters is succeeded_runs_with_failed_task - runs where the overall job reported SUCCEEDED even though at least one of its tasks reported FAILED/ERROR/TIMED_OUT (usually an optional/best-effort task, or a masked failure worth a second look).", "healthy": "succeeded_runs_with_failed_task at/near 0 - field heuristic; tune :warn_succeeded_with_failed for your account.", "investigate_if": "succeeded_runs_with_failed_task at/above :warn_succeeded_with_failed (WARN) or :crit_succeeded_with_failed (CRITICAL) - field heuristic; check whether the failing task is intentionally marked non-blocking or is silently swallowing a real failure.", "actions": [ "open one of the flagged runs and check which task failed and why (free);", "if the task should block the run, remove its \"optional/continue on failure\" setting (config);", "n/a - this is a correctness/visibility fix, not a spend decision." ], "next": [ { "query_id": "lakeflow_tasks_near_timeout", "if": "a task timing out is one common cause of a masked task failure" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_succeeded_with_failed_tasks.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_succeeded_with_failed_tasks" }, { "query_id": "lakeflow_tasks_near_timeout", "title": "Tasks running near or past their configured timeout", "domain": "jobs_pipelines", "tier": "standard", "stars": false, "reads": [ "system.lakeflow.job_task_run_timeline", "system.lakeflow.job_tasks" ], "requires": "SELECT on system.lakeflow; GA (execution_duration_seconds and timeout_seconds were both added late Nov 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "near_timeout_ratio", "default": 0.8, "meaning": "fraction of the configured timeout that counts as \"near timeout\"" }, { "name": "warn_near_timeout_runs", "default": 3, "meaning": "near/over-timeout runs for a task that flags WARN" }, { "name": "crit_near_timeout_runs", "default": 10, "meaning": "that flags CRITICAL" } ], "confidence": "needs_confirmation", "confidence_note": "both execution_duration_seconds (job_task_run_timeline) and timeout_seconds (job_tasks) are not populated before late Nov 2025; runs_no_task_timeout / runs_exec_null expose that so a short-history account degrades to \"not assessed\" instead of reading a missing value as zero.", "read_this": "One row = a task within a job. The columns that matter are runs_near_timeout (execution reached at/above :near_timeout_ratio of the configured timeout) and runs_over_timeout (execution reached or passed the timeout and should have been killed).", "healthy": "runs_near_timeout and runs_over_timeout both at/near 0 - field heuristic; tune :near_timeout_ratio and :warn_near_timeout_runs for your account.", "investigate_if": "(runs_near_timeout + runs_over_timeout) at/above :warn_near_timeout_runs (WARN) or :crit_near_timeout_runs (CRITICAL) - field heuristic; runs_over_timeout > 0 is the more urgent signal since the task should already have been killed.", "actions": [ "look at whether the task's input volume grew and the timeout was never revisited (free);", "raise the task's configured timeout to a realistic value, or split the task into smaller steps (config);", "if the task is legitimately compute-bound, give it a faster node type or more parallelism (spend)." ], "next": [ { "query_id": "lakeflow_phase_cold_start", "if": "for the run-level setup/queue/execution breakdown" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_tasks_near_timeout.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_tasks_near_timeout" }, { "query_id": "lakeflow_termination_taxonomy", "title": "Termination code taxonomy across job runs", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "the documented termination_code value set (root-cause classification plus quota/limit-hit codes such as WORKSPACE_RUN_LIMIT_EXCEEDED, MAX_JOB_QUEUE_SIZE_EXCEEDED, CLUSTER_ERROR, STORAGE_ACCESS_ERROR) was verified against a live workspace.", "read_this": "One row = a (workspace, termination_code) combination in the window. Use run_rows / distinct_runs to see which termination reasons dominate before drilling into the specific failing jobs.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "lakeflow_failed_runs", "if": "drill into which jobs/triggers a given termination_code hits" }, { "query_id": "lakeflow_failed_jobs_wasted_dbus", "if": "drill into the DBU cost of jobs terminating this way" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_termination_taxonomy.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_termination_taxonomy" }, { "query_id": "lakeflow_termination_type_probe", "title": "Termination type discovery probe (runtime value enumeration)", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA (the column exists; its value enum is unverified - see caveats)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "termination_type is confirmed to exist as a column, but its distinct value list is unverified and it is not populated before early Dec 2025; this query is a runtime discovery probe rather than one that assumes any specific value.", "read_this": "One row = a (workspace, termination_type) combination in the window, with a raw row count. Run this once to discover the actual termination_type values on your account before building any logic that filters or bands on them.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "lakeflow_failed_runs", "if": "drill into runs behind a given termination_type once confirmed" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_termination_type_probe.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_termination_type_probe" }, { "query_id": "lakeflow_workload_mix_hours", "title": "Job run workload mix by run type, trigger, and day", "domain": "jobs_pipelines", "tier": "lite", "stars": false, "reads": [ "system.lakeflow.job_run_timeline" ], "requires": "SELECT on system.lakeflow; GA (execution_duration_seconds was added early Dec 2025)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "the run_type values (JOB_RUN/SUBMIT_RUN/WORKFLOW_RUN) and the WORKFLOW_RUN compute-attribution note were verified against system.lakeflow.job_run_timeline in a live workspace.", "read_this": "One row = a (workspace, run_type, trigger_type, day) combination. Use distinct_runs and execution_s_total to see your run-type/trigger mix and daily volume before drilling into a specific failure or cost finding.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "lakeflow_failed_runs", "if": "drill into failures within a given run_type/trigger_type" }, { "query_id": "lakeflow_pipeline_cost", "if": "drill into DBU cost for the pipeline-driven share of this mix" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/jobs_pipelines/lakeflow_workload_mix_hours.sql", "learn_url": "/learn/tech/databricks/audit/jobs_pipelines#q-lakeflow_workload_mix_hours" } ], "performance": [ { "query_id": "audit_self_cost", "title": "Cost of running this audit itself", "domain": "performance", "tier": "lite", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "Measured magnitude - query count and runtime are directly observable columns in system.query.history; Databricks exposes no per-query DBU column, so the dollar cost of running this audit stays \"not assessed\" by design, not by omission.", "read_this": "One row = a workspace + statement type describing how many queries this audit itself issued and how long they ran. The columns that matter are query_count and total_duration_secs - how much of your own query-history footprint running this audit adds.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "query_costly_statements", "if": "you want to see the audit's own queries ranked individually" }, { "query_id": "query_workload_mix_hours", "if": "to see how the audit's load compares to the rest of your workload" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/audit_self_cost.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-audit_self_cost" }, { "query_id": "query_cache_coldstart", "title": "Result-cache hit rate and IO-cache cold-start latency", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_io_cache_pct", "default": 50, "meaning": "average read_io_cache_percent below which a day+warehouse flags WARN, and the per-query low-cache threshold" }, { "name": "crit_io_cache_pct", "default": 20, "meaning": "average read_io_cache_percent below which it flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse whose queries were checked for cache reuse. The columns that matter are read_io_cache_percent_avg (average share of scanned bytes served from disk/IO cache) and waiting_for_compute_ms_sum (cumulative cold-start/provisioning wait). A day with a low average and a high low_io_cache_count is spending compute re-reading data it should be caching.", "healthy": "read_io_cache_percent_avg at/above :warn_io_cache_pct (field heuristic - tune :warn_io_cache_pct for your account).", "investigate_if": "read_io_cache_percent_avg below :warn_io_cache_pct (WARN) or below :crit_io_cache_pct (CRITICAL) - field heuristic.", "actions": [ "re-run a representative query back-to-back to confirm it is actually cold rather than just infrequently reused (free);", "pin hot tables to a warehouse that stays warm between runs, or extend the warehouse's auto-stop timeout to avoid cold starts between queries (config);", "move latency-sensitive workloads to a warehouse that never idles down, such as a dedicated always-on warehouse (spend)." ], "next": [ { "query_id": "query_queuing_waits", "if": "waiting_for_compute_ms_sum is also high" }, { "query_id": "query_local_spillage", "if": "the same warehouse also spills" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_cache_coldstart.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_cache_coldstart" }, { "query_id": "query_costly_statements", "title": "Heaviest individual SQL statements (with per-warehouse framing)", "domain": "performance", "tier": "standard", "stars": true, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_wh_share", "default": 0.1, "meaning": "fraction of a warehouse's total execution time one statement must eat to flag WARN" }, { "name": "crit_wh_share", "default": 0.25, "meaning": "same for CRITICAL" }, { "name": "top_n", "default": 1000, "meaning": "row cap" } ], "confidence": "confirmed", "confidence_note": "system.query.history columns verified in a live workspace.", "read_this": "One row = one heavy FINISHED statement. execution_duration_ms is the column that matters (Databricks has no per-query dollar column; warehouse DBUs are allocated in proportion to it, so ranking by it ~= ranking by cost within one warehouse). pct_of_warehouse_exec_ms shows how much of that warehouse's total execution time this single statement ate; statement_fingerprint groups repeat shapes.", "healthy": "no single statement dominates its warehouse - pct_of_warehouse_exec_ms below :warn_wh_share (field heuristic - tune for your account).", "investigate_if": "pct_of_warehouse_exec_ms at/above :warn_wh_share (WARN) or :crit_wh_share (CRITICAL) - one statement shape eating a large share of a warehouse's execution time (field heuristic).", "actions": [ "tune the statement - add filters / partition pruning, avoid full scans, cache reused CTEs (free);", "route heavy recurring shapes to a right-sized dedicated warehouse or schedule them off-peak (config);", "scale the warehouse up/out only after tuning, if the statement is genuinely large (spend)." ], "next": [ { "query_id": "query_costly_statements_grouped", "if": "to roll repeat shapes up by fingerprint" }, { "query_id": "query_pruning_effectiveness", "if": "read_files far exceeds pruned_files" }, { "query_id": "query_local_spillage", "if": "spilled_local_bytes is high" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_costly_statements.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_costly_statements" }, { "query_id": "query_costly_statements_grouped", "title": "Heaviest SQL statement shapes, rolled up by fingerprint", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_total_hours", "default": 1, "meaning": "cumulative execution hours for a single shape that flags WARN" }, { "name": "crit_total_hours", "default": 5, "meaning": "same for CRITICAL" }, { "name": "top_n", "default": 500, "meaning": "row cap" } ], "confidence": "confirmed", "confidence_note": "system.query.history columns verified in a live workspace; the sibling of query_costly_statements.", "read_this": "One row = one distinct statement SHAPE (statement_fingerprint) run one or more times. total_exec_ms (cumulative execution time across every run) is the column that matters - a cheap statement run 10,000 times can outweigh one heavy query. runs and avg_exec_ms tell you whether to fix the shape or its frequency.", "healthy": "total_exec_ms below :warn_total_hours hours over the window (field heuristic - tune for your account).", "investigate_if": "total_exec_ms at/above :warn_total_hours (WARN) or :crit_total_hours (CRITICAL) cumulative hours - a hot repeated shape (field heuristic). High runs + low avg_exec_ms means fix the caller (frequency / caching); high avg_exec_ms means tune the statement.", "actions": [ "cache or materialize the result when the same shape reruns with the same inputs, and dedupe callers (free);", "batch / schedule the repeated shape, or move it to a right-sized warehouse (config);", "scale only the genuinely heavy shapes, after tuning (spend)." ], "next": [ { "query_id": "query_costly_statements", "if": "to see the individual runs behind a fingerprint" }, { "query_id": "query_pruning_effectiveness", "if": "a hot shape scans far more files than it prunes" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_costly_statements_grouped.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_costly_statements_grouped" }, { "query_id": "query_failed_queries_daily", "title": "Failed and canceled queries by day", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_failed_count", "default": 5, "meaning": "failed/canceled query count per day+warehouse+statement_type that flags WARN" }, { "name": "crit_failed_count", "default": 20, "meaning": "... that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The execution_status enum (FINISHED/FAILED/CANCELED) and error_message behavior under customer-managed keys were verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse + status + statement_type of failed or canceled queries, broken out by who ran them. The columns that matter are query_count (how many failed or were canceled that day) and error_message_sample (a de-identified shape of the most recent error) - a repeated high count on the same warehouse/statement_type points to a systemic issue (bad credentials, a broken job, schema drift), not one-off flakiness.", "healthy": "query_count below :warn_failed_count per day+warehouse+statement_type (field heuristic - tune :warn_failed_count for your account).", "investigate_if": "query_count at/above :warn_failed_count (WARN) or :crit_failed_count (CRITICAL) - field heuristic; a spike concentrated on one warehouse or statement_type is the real signal.", "actions": [ "read error_message_sample for the shape of the failure and check the source job/notebook logs for the un-redacted detail (free);", "fix the query, permission, or schema issue it points to, or add a retry/backoff on the calling job (config);", "if failures cluster around an unstable warehouse (capacity errors), resize it or move the workload to a more resilient one (spend)." ], "next": [ { "query_id": "query_costly_statements", "if": "you want the individual failed statements" }, { "query_id": "query_queuing_waits", "if": "failures cluster with queuing or cold-start" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_failed_queries_daily.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_failed_queries_daily" }, { "query_id": "query_local_spillage", "title": "Queries spilling to local disk", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_spill_gb", "default": 1, "meaning": "daily local-spill GB per warehouse that flags WARN" }, { "name": "crit_spill_gb", "default": 10, "meaning": "daily local-spill GB that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse + user whose queries spilled to local disk. The column that matters is spilled_local_bytes_sum - bytes a shuffle/sort/join had to write to local SSD because it did not fit in memory. Steady spill on the same warehouse means under-provisioned memory or a skewed query, not a one-off.", "healthy": "spilled_local_bytes_sum below :warn_spill_gb GB/day per warehouse (field heuristic - tune :warn_spill_gb for your account).", "investigate_if": "spilled_local_bytes_sum at/above :warn_spill_gb GB/day (WARN) or :crit_spill_gb GB/day (CRITICAL) - field heuristic; sustained daily spill on the same warehouse is the real signal.", "actions": [ "rewrite the query to cut shuffle - broadcast the small side, filter earlier, avoid exploding joins (free);", "raise shuffle partitions or enable disk caching, or move the workload to a warehouse with more memory per node (config);", "size up to a memory-optimized node type or a larger warehouse (spend)." ], "next": [ { "query_id": "query_shuffle_write_amplification", "if": "shuffle_read_bytes_sum is also high" }, { "query_id": "query_costly_statements", "if": "you want the exact statements behind a spilling warehouse" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_local_spillage.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_local_spillage" }, { "query_id": "query_per_query_estimate_lane", "title": "Per-query duration and bytes for cost-estimate allocation", "domain": "performance", "tier": "deep", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace; this query returns raw per-statement rows for per-statement cost allocation, not an aggregate.", "read_this": "One row = one finished, non-cached query on a SQL warehouse or serverless compute. The columns that matter are execution_duration_ms (used to allocate warehouse DBU cost across queries) and waiting_for_compute_duration_ms (should be netted out before allocating, since it is provisioning wait, not query work). There is no per-query dollar or DBU column here - use this as the weighting input for spreading system.billing.usage's hourly warehouse DBU total across the queries that ran in that hour, not as a cost figure on its own.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "query_costly_statements", "if": "for the same per-query granularity with pruning/shuffle/spill signals attached" }, { "query_id": "cost_dollarized_by_sku_day", "if": "for the aggregate dollar figure this table feeds into" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_per_query_estimate_lane.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_per_query_estimate_lane" }, { "query_id": "query_provenance_by_source", "title": "Query volume and duration by source (job, dashboard, notebook, ad-hoc)", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "The nested query_source struct's dotted-path access (query_source.job_info.job_id, query_source.dashboard_id, etc.) and the single-winner CASE precedence used to pick one source_kind per query are not verified verbatim against a live workspace. The subfield names themselves are confirmed, but Databricks' own documentation says multiple subfields can be populated on the same query simultaneously and are not execution-ordered, so this CASE is a heuristic, not authoritative attribution. Confirm the struct path resolves in your workspace, and consider emitting all non-null source flags instead of a single winner if you need exact attribution.", "read_this": "One row = a workspace + compute + identity + source_kind (job, dashboard, legacy_dashboard, notebook, alert, genie, sql_editor, or other) describing where queries came from. The columns that matter are source_kind (which surface issued the query) and query_count - a workspace dominated by \"other\" or split evenly between ad-hoc (sql_editor/notebook) and scheduled (job) sources tells you whether spend is coming from production pipelines or exploratory work.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "query_workload_mix_hours", "if": "for the same volume broken out by hour-of-day instead of source" }, { "query_id": "query_per_query_estimate_lane", "if": "to turn this volume into a cost estimate" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_provenance_by_source.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_provenance_by_source" }, { "query_id": "query_pruning_effectiveness", "title": "File-pruning effectiveness per query group", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_prune_ratio", "default": 0.5, "meaning": "pruning ratio (pruned_files / (pruned_files + read_files)) below which a group flags WARN" }, { "name": "crit_prune_ratio", "default": 0.2, "meaning": "... below which it flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse + identity + statement_type whose queries pruned or read files during a scan. The columns that matter are pruned_files_sum and read_files_sum - together they give the pruning ratio (pruned / (pruned + read)); a low ratio on a repeating group means partition or file layout, or the query's own predicates, are not letting Databricks skip files it should be skipping.", "healthy": "pruning ratio at/above :warn_prune_ratio (field heuristic - tune :warn_prune_ratio for your account).", "investigate_if": "pruning ratio below :warn_prune_ratio (WARN) or below :crit_prune_ratio (CRITICAL) - field heuristic; a stable low ratio on the same table/warehouse combination over several days is the real signal, not one bad day.", "actions": [ "check whether your WHERE/JOIN predicates actually align with the table's partition, Z-order, or liquid-clustering columns (free);", "run OPTIMIZE / re-cluster the table on the columns these queries filter by (config);", "if the table is still poorly organized after re-clustering, repartition or rewrite it with a layout that matches the query pattern (spend - a rewrite cost)." ], "next": [ { "query_id": "query_shuffle_write_amplification", "if": "the same reads are also shuffle-heavy" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_pruning_effectiveness.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_pruning_effectiveness" }, { "query_id": "query_queuing_waits", "title": "Queries queued for capacity or compute provisioning", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_queue_secs", "default": 60, "meaning": "combined queued-for-capacity + waiting-for-compute seconds per day+warehouse that flags WARN" }, { "name": "crit_queue_secs", "default": 600, "meaning": "... that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse whose queries waited before or during execution. The columns that matter are waiting_at_capacity_ms_sum (queued because the warehouse was already at max concurrency/scale) and waiting_for_compute_ms_sum (compute was still provisioning - cold start). A warehouse that shows up here repeatedly on the same day-of-week/hour pattern is under-sized or under-scheduled for its load, not just unlucky once.", "healthy": "waiting_at_capacity_ms_sum + waiting_for_compute_ms_sum below :warn_queue_secs seconds/day per warehouse (field heuristic - tune :warn_queue_secs for your account).", "investigate_if": "combined wait at/above :warn_queue_secs seconds (WARN) or :crit_queue_secs seconds (CRITICAL) - field heuristic; waits that recur on the same warehouse are the real signal, not a single spike.", "actions": [ "check whether the wait clusters at specific hours before changing anything, e.g. against query_workload_mix_hours (free);", "raise the warehouse's max clusters/scaling limit, or keep it warm through the busy window with a longer auto-stop (config);", "split the workload onto a second warehouse, or size up the existing one, if concurrency is structurally too high for one warehouse (spend)." ], "next": [ { "query_id": "query_workload_mix_hours", "if": "to see if waits line up with a load spike by hour" }, { "query_id": "query_local_spillage", "if": "the same warehouse also spills once queries do get compute" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_queuing_waits.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_queuing_waits" }, { "query_id": "query_shuffle_write_amplification", "title": "Shuffle-heavy queries and small-file write amplification", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_shuffle_gb", "default": 50, "meaning": "daily shuffle_read_bytes per group that flags WARN" }, { "name": "crit_shuffle_gb", "default": 200, "meaning": "... that flags CRITICAL" }, { "name": "warn_avg_file_mb", "default": 32, "meaning": "average written-file size in MB below which a group flags WARN (small-file amplification)" }, { "name": "crit_avg_file_mb", "default": 8, "meaning": "... below which it flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + warehouse + identity + statement_type whose queries shuffled data or wrote files. Two independent signals live in this row: shuffle_read_bytes_sum (a shuffle-heavy or skewed-join signal) and written_files_sum versus written_bytes_sum (write amplification - lots of small files instead of a few well-sized ones). Check both; a query can be shuffle-heavy without writing anything, or write badly without shuffling.", "healthy": "shuffle_read_bytes_sum below :warn_shuffle_gb GB/day AND average written-file size at/above :warn_avg_file_mb MB (field heuristics - tune :warn_shuffle_gb / :warn_avg_file_mb for your account).", "investigate_if": "shuffle_read_bytes_sum at/above :warn_shuffle_gb GB (WARN) or :crit_shuffle_gb GB (CRITICAL), OR average written-file size below :warn_avg_file_mb MB (WARN) or :crit_avg_file_mb MB (CRITICAL) - field heuristics; either signal alone is enough to flag the row.", "actions": [ "for shuffle, check the join/aggregation for a skewed key or a missing broadcast hint; for small files, check whether the write has a narrow partition-by or a streaming micro-batch producing many tiny files (free);", "tune shuffle partitions and the broadcast-join threshold, or enable auto-compaction / optimized writes on the target table (config);", "run OPTIMIZE to compact existing small files, or move the write to a warehouse with more memory to reduce spill-driven shuffle (spend)." ], "next": [ { "query_id": "query_local_spillage", "if": "the same warehouse also spills - shuffle and spill often co-occur" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_shuffle_write_amplification.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_shuffle_write_amplification" }, { "query_id": "query_workload_mix_hours", "title": "Workload mix by hour of day", "domain": "performance", "tier": "standard", "stars": false, "reads": [ "system.query.history" ], "requires": "SELECT on system.query; GA (system.query.history is generally available)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "Columns verified against system.query.history in a live workspace.", "read_this": "One row = a day + hour-of-day + compute + statement_type + identity slice of your query workload. The columns that matter are query_count and total_duration_ms_sum - stack these by hour to see when your workload actually peaks, and whether that peak lines up with warehouse auto-scaling or scheduled auto-stop.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "query_queuing_waits", "if": "a peak hour also shows queuing" }, { "query_id": "compute_warehouse_autoscale_churn", "if": "to check whether autoscaling tracks this hourly pattern" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/performance/query_workload_mix_hours.sql", "learn_url": "/learn/tech/databricks/audit/performance#q-query_workload_mix_hours" } ], "serving_ai": [ { "query_id": "compute_ai_gateway_usage", "title": "AI Gateway usage, tokens, and latency by endpoint", "domain": "serving_ai", "tier": "standard", "stars": false, "reads": [ "system.ai_gateway.usage" ], "requires": "SELECT on system.ai_gateway; must be enabled per-metastore (empty until AI Gateway is enabled on an endpoint)", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "Column names (event_time, workspace_id, endpoint_name, requester, status_code, latency_ms, input_tokens, output_tokens) and the 2xx/429/error status-code grouping are verified against a live workspace system-schema dump but not against published Databricks documentation, so treat the request classification as a working assumption until you confirm it against your own workspace.", "read_this": "One row = a day + workspace + endpoint + requester's AI Gateway traffic. The columns that matter are total_requests, rate_limited_requests, and error_requests - a rise in rate_limited_requests means you are hitting Gateway or provider rate limits, and a rise in error_requests means calls are failing outright; both mean tokens and latency spent on calls that did not succeed.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "compute_serving_endpoint_usage", "if": "you want the underlying serving-endpoint request/error/token detail" }, { "query_id": "cost_by_compute_resource", "if": "you want this endpoint's DBU cost" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/serving_ai/compute_ai_gateway_usage.sql", "learn_url": "/learn/tech/databricks/audit/serving_ai#q-compute_ai_gateway_usage" }, { "query_id": "compute_serving_dormant_endpoints", "title": "Serving endpoints with no traffic in the window (dormant / wasted spend)", "domain": "serving_ai", "tier": "deep", "stars": false, "reads": [ "system.serving.served_entities", "system.serving.endpoint_usage", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.serving, system.billing; system.serving must be enabled per-metastore (empty until Model Serving is in use); system.billing is GA.", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_low_requests", "default": 10, "meaning": "total requests over the window at/below which a served entity with some traffic still flags WARN as near-dormant" } ], "confidence": "needs_confirmation", "confidence_note": "The dedup-to-latest-config-row join and the endpoint-level cost rollup are verified against a live workspace, but whether served_entities exposes an explicit creation flag - needed to exclude endpoints created inside the window from the dormant label - is not confirmed; verify against your own workspace before acting on borderline rows.", "read_this": "One row = a served entity (a model/version behind an endpoint). The columns that matter are last_request_date and net_dbus: last_request_date NULL means no request landed on this entity in the window, and net_dbus > 0 on that same row means you are paying DBUs for capacity nobody is calling.", "healthy": "last_request_date is not NULL and total_requests is above :warn_low_requests (field heuristic - tune :warn_low_requests for your account).", "investigate_if": "last_request_date IS NULL with net_dbus > 0 (CRITICAL - paying for zero traffic), or last_request_date IS NULL with no cost signal, or total_requests at/below :warn_low_requests (WARN) - field heuristic; tune :warn_low_requests for your account.", "actions": [ "confirm with the owning team that the endpoint has no real consumers, then delete the served entity / endpoint (free);", "if it must stay available for occasional use, move it to a scale-to-zero or pay-per-token serving mode so idle time is not billed (config);", "if it must stay warm for latency SLAs, right-size the provisioned throughput to the smallest tier that meets that SLA (spend, but lower than today)." ], "next": [ { "query_id": "compute_serving_endpoint_usage", "if": "you want the daily request/error/token trend before deciding" }, { "query_id": "cost_by_compute_resource", "if": "you want this endpoint's full cost history" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/serving_ai/compute_serving_dormant_endpoints.sql", "learn_url": "/learn/tech/databricks/audit/serving_ai#q-compute_serving_dormant_endpoints" }, { "query_id": "compute_serving_endpoint_usage", "title": "Serving endpoint request volume, errors, tokens, and cost by day", "domain": "serving_ai", "tier": "deep", "stars": false, "reads": [ "system.serving.served_entities", "system.serving.endpoint_usage", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.serving, system.billing; system.serving must be enabled per-metastore (empty until Model Serving is in use); system.billing is GA.", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "needs_confirmation", "confidence_note": "Column names (request_time, workspace_id, served_entity_id, status_code, input_token_count, output_token_count) and the served_entities-to-endpoint_usage join are verified against a live workspace system-schema dump, but the 2xx-as-success classification is a working assumption, not vendor-documented.", "read_this": "One row = a day + endpoint + served entity's request volume. The columns that matter are total_requests, error_requests, and net_dbus - use this as the detail behind a dormancy or cost finding, not as a standalone health signal (this file mixes several metrics with no single waste driver).", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "compute_serving_dormant_endpoints", "if": "you want to know which of these endpoints have zero traffic" }, { "query_id": "cost_by_compute_resource", "if": "you want the endpoint's cost trend outside the serving grain" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/serving_ai/compute_serving_endpoint_usage.sql", "learn_url": "/learn/tech/databricks/audit/serving_ai#q-compute_serving_endpoint_usage" }, { "query_id": "serving_endpoint_traffic_by_endpoint", "title": "Serving endpoint traffic totals over the window (request/error/token rollup)", "domain": "serving_ai", "tier": "deep", "stars": false, "reads": [ "system.serving.endpoint_usage", "system.serving.served_entities", "system.billing.usage", "system.billing.list_prices" ], "requires": "SELECT on system.serving, system.billing; system.serving must be enabled per-metastore (empty until Model Serving is in use); system.billing is GA.", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" } ], "confidence": "confirmed", "confidence_note": "Columns were verified against the AI Gateway usage-schema documentation on 2026-06-21 - re-verify if you are running this well after that date, since system-table schemas evolve.", "read_this": "One row = one endpoint's total traffic over the window. The columns that matter are request_count and last_request_date - use this as the join input for a cost-per-request or dormancy check, not as a standalone health signal.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "compute_serving_dormant_endpoints", "if": "you want the per-served-entity dormancy read" }, { "query_id": "cost_by_compute_resource", "if": "you want the underlying billed cost trend" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/serving_ai/serving_endpoint_traffic_by_endpoint.sql", "learn_url": "/learn/tech/databricks/audit/serving_ai#q-serving_endpoint_traffic_by_endpoint" } ], "storage": [ { "query_id": "iceberg_uniform_metadata", "title": "Iceberg / UniForm metadata detection (DESCRIBE EXTENDED)", "domain": "storage", "tier": "deep", "stars": false, "reads": [ "none - DESCRIBE EXTENDED output; not a system table" ], "requires": "DESCRIBE / SELECT privilege on the target table; Unity Catalog required (UniForm / managed-Iceberg detection is a UC feature)", "params": [], "confidence": "needs_confirmation", "confidence_note": "The 'Delta Uniform Iceberg' section exists per Databricks docs, but the exact output row/key strings to parse are unverified - confirm the section key strings the first time you run this against a UniForm-enabled table in your own workspace.", "read_this": "One row per key = a DESCRIBE EXTENDED output line for the target table. The rows under the 'Delta Uniform Iceberg' section are what matter - their presence (and their format-name value) tells you whether the table is UniForm-enabled / has a synced Iceberg metadata pointer.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "table_inventory_type", "if": "to find EXTERNAL / non-DELTA tables worth checking" }, { "query_id": "table_props_time_travel_config", "if": "for a fuller per-table metadata pass" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/iceberg_uniform_metadata.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-iceberg_uniform_metadata" }, { "query_id": "po_clustering_activity", "title": "Liquid clustering activity and DBU cost by table", "domain": "storage", "tier": "standard", "stars": false, "reads": [ "system.storage.predictive_optimization_operations_history" ], "requires": "SELECT on system.storage; Public Preview (system.storage.predictive_optimization_operations_history), regional", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_clustering_dbu", "default": 50, "meaning": "estimated clustering DBUs per table over the window that flags WARN" }, { "name": "crit_clustering_dbu", "default": 200, "meaning": "estimated clustering DBUs per table over the window that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "Column names and the four CLUSTERING operation_metrics subfields queried here are verified against Databricks system-table docs.", "read_this": "One row = a table's incremental liquid-clustering (CLUSTERING) activity over the window. The column that matters is clustering_estimated_dbu - the DBUs Predictive Optimization spent compacting/re-clustering that table. Some clustering activity is normal; a table burning outsized DBUs on it repeatedly is the signal worth chasing.", "healthy": "clustering_estimated_dbu below :warn_clustering_dbu DBUs per table over the window (field heuristic - tune :warn_clustering_dbu for your account).", "investigate_if": "clustering_estimated_dbu at/above :warn_clustering_dbu (WARN) or :crit_clustering_dbu (CRITICAL) - field heuristic; also compare clustered_bytes against removed_bytes - if clustered_bytes is small relative to removed_bytes and DBU is high, clustering may be thrashing rather than compacting.", "actions": [ "check po_clustering_column_churn for the same table to see if the clustering keys are unstable (free);", "if the table clusters constantly because of small/streaming writes, batch writes to reduce trigger frequency or lengthen the target file size (config);", "re-evaluate the CLUSTER BY keys for the table's actual query patterns to reduce re-clustering work (redesign/spend)." ], "next": [ { "query_id": "po_clustering_column_churn", "if": "churn is suspected" }, { "query_id": "po_maintenance_cost_by_table", "if": "for the full per-table maintenance-cost picture across all PO operation types" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/po_clustering_activity.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-po_clustering_activity" }, { "query_id": "po_clustering_column_churn", "title": "Clustering-key selection churn by table", "domain": "storage", "tier": "deep", "stars": false, "reads": [ "system.storage.predictive_optimization_operations_history" ], "requires": "SELECT on system.storage; Public Preview (system.storage.predictive_optimization_operations_history), regional", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_churn_events", "default": 2, "meaning": "column-selection-changed events per table over the window that flags WARN" }, { "name": "crit_churn_events", "default": 5, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The four operation_metrics subfields (has_column_selection_changed, old_clustering_columns, new_clustering_columns, additional_reason) and their string typing are verified against Databricks system-table docs.", "read_this": "One row = a table + old-to-new clustering-column signature (or \"no change\") seen at least once in the window. selection_event_count is how many times that exact old-to-new (or unchanged) selection happened; has_column_selection_changed = 'true' rows are where Predictive Optimization actually swapped the clustering keys - repeated true rows on the same table with a rising event count mean the keys keep getting relitigated, which is layout instability, not a one-off tuning pass.", "healthy": "no has_column_selection_changed = 'true' rows, or fewer than :warn_churn_events change events per table over the window (field heuristic - a single key selection/change is normal, especially on a newly clustered table).", "investigate_if": "has_column_selection_changed = 'true' with selection_event_count at/above :warn_churn_events (WARN) or :crit_churn_events (CRITICAL) - field heuristic; repeated churn on the same table means the clustering keys are unstable, which burns DBUs in po_clustering_activity without settling into a stable layout.", "actions": [ "look at additional_reason and old/new_clustering_columns to see what is driving the reselection (free);", "if churn tracks a recurring write pattern (e.g. a daily batch that shifts filter columns), pin CLUSTER BY explicitly instead of leaving it to automatic selection (config);", "if the table's query patterns are genuinely heterogeneous, consider splitting it, or accept the DBU cost as a known trade-off (redesign/spend)." ], "next": [ { "query_id": "po_clustering_activity", "if": "see the DBU cost this churn is generating" }, { "query_id": "table_props_time_travel_config", "if": "check other per-table config while you are looking at this table" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/po_clustering_column_churn.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-po_clustering_column_churn" }, { "query_id": "po_data_skipping_backfill", "title": "Data-skipping stats backfill by table", "domain": "storage", "tier": "deep", "stars": false, "reads": [ "system.storage.predictive_optimization_operations_history" ], "requires": "SELECT on system.storage; Public Preview (system.storage.predictive_optimization_operations_history), regional", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_scan_gb", "default": 100, "meaning": "GB scanned by a backfill event that added no new data-skipping columns, over the window, that flags WARN" }, { "name": "crit_scan_gb", "default": 500, "meaning": "GB that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "DATA_SKIPPING_COLUMN_SELECTION's operation_metrics subfields queried here (added/removed/new_data_skipping_columns, amount_of_scanned_bytes, number_of_scanned_files) are verified against Databricks system-table docs; a 6th real subfield, old_data_skipping_columns, exists but is intentionally not selected here.", "read_this": "One row = a table + distinct added/removed/new data-skipping-column set seen in the window. scanned_bytes is how many bytes Predictive Optimization had to scan to decide on (or confirm) data-skipping columns for that table; new_data_skipping_columns is what it landed on. A row with a large scanned_bytes but an empty new_data_skipping_columns means the backfill ran and paid its scan cost but did not end up adding any pruning columns for that table.", "healthy": "new_data_skipping_columns is populated whenever a backfill event scans data, or scanned_bytes stays under :warn_scan_gb GB for events with no new columns (field heuristic - tune :warn_scan_gb for your account).", "investigate_if": "no new_data_skipping_columns AND scanned_bytes at/above :warn_scan_gb GB (WARN) or :crit_scan_gb GB (CRITICAL) - field heuristic; this only tells you stats were (re)computed, not whether pruning actually improved - cross-check query_pruning_effectiveness for the payoff.", "actions": [ "check query_pruning_effectiveness for the table to see if existing data-skipping columns are actually being used for pruning (free);", "if the table has many high-cardinality or rarely-filtered columns, explicitly set delta.dataSkippingStatsColumns to a smaller, deliberate list instead of leaving it to automatic selection (config);", "if this backfill activity recurs expensively with no coverage gain on a large table, consider a clustering/Z-order redesign around the columns you actually filter on (redesign/spend)." ], "next": [ { "query_id": "query_pruning_effectiveness", "if": "see whether data-skipping columns are actually cutting scanned bytes at query time" }, { "query_id": "po_clustering_activity", "if": "compaction/clustering activity on the same table" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/po_data_skipping_backfill.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-po_data_skipping_backfill" }, { "query_id": "po_maintenance_cost_by_table", "title": "Predictive Optimization maintenance cost and status by table", "domain": "storage", "tier": "standard", "stars": false, "reads": [ "system.storage.predictive_optimization_operations_history" ], "requires": "SELECT on system.storage; Public Preview (system.storage.predictive_optimization_operations_history), regional, 180-day retention", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_maint_dbu", "default": 20, "meaning": "estimated maintenance DBUs per table+operation+status group over the window that flags WARN" }, { "name": "crit_maint_dbu", "default": 100, "meaning": "that flags CRITICAL (a FAILED operation_status always flags CRITICAL regardless of these)" } ], "confidence": "confirmed", "confidence_note": "usage_quantity as ESTIMATED_DBU, the operation_status enum values (including the 'FAILED: INTERNAL_ERROR' string with its embedded colon), and the up-to-24h billing-population lag are verified against Databricks docs.", "read_this": "One row = a table x operation_type (CLUSTERING/VACUUM/etc) x operation_status combo over the window. estimated_dbu is the DBUs Predictive Optimization spent maintaining that table for that operation type - the columns that matter are operation_status (SUCCESSFUL vs a 'FAILED: INTERNAL_ERROR' row, note the embedded colon) and estimated_dbu; a table with FAILED rows means maintenance is not actually completing even though you may be paying for the attempts.", "healthy": "no FAILED rows for the table, and estimated_dbu under :warn_maint_dbu per operation+status group over the window (field heuristic - tune :warn_maint_dbu for your account).", "investigate_if": "any operation_status LIKE 'FAILED%' row (CRITICAL - maintenance is not completing on that table), or estimated_dbu at/above :warn_maint_dbu (WARN) / :crit_maint_dbu (CRITICAL) - field heuristic.", "actions": [ "for FAILED rows, check the table for the underlying error (schema drift, concurrent writers, permissions) before anything else (free);", "for high-DBU SUCCESSFUL tables, check po_clustering_activity / po_vacuum_reclaimed_bytes for that table_id to see which operation type is driving the cost (free/config);", "if a large table's maintenance cost is structural, consider a partitioning/clustering redesign, or exclude it from automatic PO and schedule maintenance in an off-peak window instead (spend/redesign)." ], "next": [ { "query_id": "po_clustering_activity", "if": "drill into CLUSTERING rows for a specific table" }, { "query_id": "po_vacuum_reclaimed_bytes", "if": "drill into VACUUM rows for a specific table" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/po_maintenance_cost_by_table.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-po_maintenance_cost_by_table" }, { "query_id": "po_vacuum_reclaimed_bytes", "title": "VACUUM reclaimed bytes and DBU cost by table", "domain": "storage", "tier": "standard", "stars": false, "reads": [ "system.storage.predictive_optimization_operations_history" ], "requires": "SELECT on system.storage; Public Preview (system.storage.predictive_optimization_operations_history), regional", "params": [ { "name": "period_days", "default": 30, "meaning": "rolling window in days" }, { "name": "warn_noop_dbu", "default": 5, "meaning": "estimated VACUUM DBUs spent on a table that reclaimed zero bytes over the window that flags WARN" }, { "name": "crit_noop_dbu", "default": 20, "meaning": "that flags CRITICAL" } ], "confidence": "confirmed", "confidence_note": "The two VACUUM operation_metrics subfields (number_of_deleted_files, amount_of_data_deleted_bytes) and their map typing are verified against Databricks system-table docs.", "read_this": "One row = a table's successful VACUUM activity over the window: how many VACUUM operations ran (vacuum_op_count), how many files and bytes they deleted (total_deleted_files, total_deleted_bytes), and the estimated DBUs they cost (vacuum_estimated_dbu). The pairing that matters is total_deleted_bytes against vacuum_estimated_dbu - a table that keeps vacuuming successfully but reclaiming (near) zero bytes is paying maintenance cost for no storage benefit.", "healthy": "total_deleted_bytes > 0 for vacuum activity, or vacuum_estimated_dbu under :warn_noop_dbu when reclaimed bytes are zero (field heuristic - tune :warn_noop_dbu for your account; a well-tuned table that rarely deletes rows can legitimately show low/zero reclaim).", "investigate_if": "total_deleted_bytes = 0 (no reclaim) with vacuum_estimated_dbu at/above :warn_noop_dbu (WARN) or :crit_noop_dbu (CRITICAL) - field heuristic; note this query only covers tables that WERE successfully vacuumed - a table missing entirely from this result may simply never run VACUUM, which is a coverage gap you can only see by pairing this against storage_breakdown_analyze's vacuumable_bytes for the same table.", "actions": [ "for zero-reclaim tables, check whether Predictive Optimization VACUUM is running on a schedule shorter than your actual delete/update cadence (free);", "lengthen the VACUUM interval or disable automatic VACUUM for tables that rarely delete data (config);", "for large tables with genuinely high churn, confirm delta.deletedFileRetentionDuration is not set so short that VACUUM runs (and costs DBU) far more often than needed (config/spend, see table_props_time_travel_config)." ], "next": [ { "query_id": "storage_breakdown_analyze", "if": "pair reclaimed bytes against vacuumable_bytes still on disk for the same table" }, { "query_id": "table_props_time_travel_config", "if": "check the retention config driving VACUUM eligibility" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/po_vacuum_reclaimed_bytes.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-po_vacuum_reclaimed_bytes" }, { "query_id": "storage_breakdown_analyze", "title": "Per-table storage breakdown: active vs time-travel vs vacuumable", "domain": "storage", "tier": "deep", "stars": false, "reads": [ "none - ANALYZE TABLE COMPUTE STORAGE METRICS output; not a system table" ], "requires": "privilege sufficient to run ANALYZE TABLE on the target table; GA but requires DBR 18.0+ (COMPUTE STORAGE METRICS is not available on older runtimes)", "params": [], "confidence": "confirmed", "confidence_note": "All 8 result columns and the DBR 18.0+ gate are verbatim-verified against Databricks docs.", "read_this": "One result row per run = one table's true on-disk footprint at the moment you ran it, split into active_bytes (live data), time_travel_bytes (retained history), and vacuumable_bytes (already-deleted files still on disk, reclaimable by VACUUM). The columns that matter are vacuumable_bytes and time_travel_bytes relative to total_bytes - a high share of either means you are paying to store data nobody is reading.", "healthy": "n/a - status not computed (see caveats); as a heuristic, vacuumable_bytes + time_travel_bytes under roughly 20 percent of total_bytes is the healthy shape.", "investigate_if": "n/a - status not computed (see caveats); as a heuristic, vacuumable_bytes + time_travel_bytes at or above roughly 20 percent of total_bytes is worth a look - field heuristic, not an authoritative threshold. A large vacuumable_bytes alone means VACUUM has not run recently or retention is too long; a large time_travel_bytes alone means retention is deliberately long (or should be shortened).", "actions": [ "run VACUUM on the table to reclaim vacuumable_bytes, or check whether Predictive Optimization VACUUM is enabled for it (free/config);", "shorten delta.deletedFileRetentionDuration / delta.logRetentionDuration if the retention window is longer than your compliance need requires (config, see table_props_time_travel_config);", "if time_travel_bytes is large because of a genuine long-retention requirement, tier the underlying cloud storage location to a cheaper class via a lifecycle rule rather than treating it as reclaimable (spend/infra change)." ], "next": [ { "query_id": "po_vacuum_reclaimed_bytes", "if": "see whether VACUUM already reclaimed this table recently" }, { "query_id": "table_props_time_travel_config", "if": "check the retention config driving time_travel_bytes" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/storage_breakdown_analyze.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-storage_breakdown_analyze" }, { "query_id": "table_inventory_type", "title": "Table and view inventory by type and format", "domain": "storage", "tier": "deep", "stars": false, "reads": [ "system.information_schema.tables" ], "requires": "SELECT on system.information_schema; Unity Catalog required", "params": [], "confidence": "needs_confirmation", "confidence_note": "Only table_type (MANAGED/EXTERNAL/VIEW) is verbatim-confirmed against Databricks docs; table_catalog, table_schema, and data_source_format are plausible information_schema column names but unverified until you run this in your own workspace and confirm they resolve.", "read_this": "One row = a catalog + schema + table_type + data_source_format combination, with the count of tables/views matching it. table_type tells you managed vs external vs view; data_source_format (when it resolves) flags DELTA vs ICEBERG. EXTERNAL tables are not eligible for Predictive Optimization, so a high EXTERNAL count here caps how much of po_maintenance_cost_by_table's coverage can ever reach 100%.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "iceberg_uniform_metadata", "if": "drill into EXTERNAL / non-DELTA tables to confirm UniForm" }, { "query_id": "po_maintenance_cost_by_table", "if": "cross-check PO coverage against the EXTERNAL count here" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/table_inventory_type.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-table_inventory_type" }, { "query_id": "table_props_time_travel_config", "title": "Delta time-travel and optimization table properties", "domain": "storage", "tier": "standard", "stars": false, "reads": [ "none - DESCRIBE EXTENDED / SHOW TBLPROPERTIES output; not a system table" ], "requires": "DESCRIBE / SELECT privilege on the target table; GA (standard Delta Lake table properties)", "params": [], "confidence": "confirmed", "confidence_note": "All 7 property names and their documented defaults are verbatim-verified against Delta Lake / Databricks docs.", "read_this": "One row block per target table = its current time-travel retention (delta.logRetentionDuration, delta.deletedFileRetentionDuration), data-skipping indexing config, auto-optimize flags, and deletion-vector flag. The properties that matter most for storage bloat are delta.logRetentionDuration and delta.deletedFileRetentionDuration - values are CalendarInterval strings (e.g. 'interval 30 days') that you need to parse to days yourself.", "healthy": "n/a - inventory", "investigate_if": "n/a - inventory", "actions": [], "next": [ { "query_id": "storage_breakdown_analyze", "if": "pairs this retention config with the actual time-travel / vacuumable bytes it produces" }, { "query_id": "po_vacuum_reclaimed_bytes", "if": "see whether VACUUM is actually running against this retention window" } ], "sql_url": "https://github.com/darshanmeel/crosshire-audit-databricks-admin/blob/master/queries/storage/table_props_time_travel_config.sql", "learn_url": "/learn/tech/databricks/audit/storage#q-table_props_time_travel_config" } ] }