Skip to content

Refactor out inlined values in builtin materialized views #39468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SangJunBak wants to merge 7 commits into from
+595 −677
Open

Refactor out inlined values in builtin materialized views #39468

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
47b1421
catalog: extract builtin index rows of mz_indexes into mz_builtin_ind…
SangJunBak Oct 1, 2026
66cf390
catalog: read the builtin log keys of mz_indexes from mz_builtin_log_…
SangJunBak Oct 1, 2026
67ce561
catalog: read the builtin rows of mz_sources from mz_builtin_sources
SangJunBak Oct 1, 2026
0adac32
catalog: read the builtin log rows of mz_builtin_sources from mz_buil…
SangJunBak Oct 2, 2026
33de8ef
catalog: make mz_sources a static builtin
SangJunBak Oct 1, 2026
cc06d19
catalog: filter builtin_items inside each mz_builtin view generator
SangJunBak Oct 1, 2026
b3e9ac9
test: update expected outputs for the generated mz_builtin views
SangJunBak Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions doc/user/content/sql/system-catalog/mz_internal.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1632,6 +1632,8 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys
[`timestamp with time zone`]: /sql/types/timestamp

<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_activity_log_thinned -->
<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_indexes -->
<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_log_indexes -->
<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_materialized_views -->
<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_sources -->
<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_tables -->
Expand Down
18 changes: 18 additions & 0 deletions src/adapter/src/catalog/open/builtin_schema_migration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,24 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
MZ_INTERNAL_SCHEMA,
"mz_replacements",
),
// mz_indexes reads builtin indexes from mz_builtin_indexes
// and builtin logs from mz_builtin_log_indexes instead of
// inlining them, which changes its SQL fingerprint.
MigrationStep::replacement(
"26.46.0-dev.0",
CatalogItemType::MaterializedView,
MZ_CATALOG_SCHEMA,
"mz_indexes",
),
// mz_sources reads builtin sources from mz_builtin_sources
// and builtin logs from mz_builtin_log_indexes
// instead of inlining them, which changes its SQL fingerprint.
MigrationStep::replacement(
"26.46.0-dev.0",
CatalogItemType::MaterializedView,
MZ_CATALOG_SCHEMA,
"mz_sources",
),
]
});

Expand Down
189 changes: 3 additions & 186 deletions src/catalog/src/builtin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -782,7 +782,7 @@ impl Fingerprint for SqlRelationType {

/// Asserts that `name` is safe to embed unquoted inside a `'...'`-quoted SQL literal
/// or inside a `"..."`-quoted SQL identifier. Generated builtin relations
/// (`make_mz_indexes`, `make_mz_object_dependencies_raw`, ...) concatenate builtin names
/// (`make_builtin_indexes`, `make_mz_object_dependencies_raw`, ...) concatenate builtin names
/// into SQL fragments, so a quote or backslash would produce malformed SQL. Builtin
/// names should always be plain ASCII identifiers.
pub(super) fn assert_safe_builtin_name(name: &str, kind: &str) {
Expand Down Expand Up @@ -1155,10 +1155,10 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
Builtin::MaterializedView(&MZ_DATABASES),
Builtin::MaterializedView(&MZ_SCHEMAS),
Builtin::Table(&MZ_COLUMNS),
// mz_indexes is generated dynamically below with inlined builtin VALUES.
Builtin::MaterializedView(&MZ_INDEXES),
Builtin::Table(&MZ_INDEX_COLUMNS),
Builtin::MaterializedView(&MZ_TABLES),
// mz_sources is generated dynamically below with inlined builtin VALUES.
Builtin::MaterializedView(&MZ_SOURCES),
Builtin::MaterializedView(&MZ_SOURCE_REFERENCES),
Builtin::MaterializedView(&MZ_POSTGRES_SOURCES),
Builtin::MaterializedView(&MZ_POSTGRES_SOURCE_TABLES),
Expand Down Expand Up @@ -1513,61 +1513,6 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne

builtin_items.extend(notice::builtins());

// Generate mz_sources with builtin source/log entries inlined as VALUES so
// that its SQL fingerprint changes whenever a builtin source is added or
// removed, forcing an explicit MigrationStep::replacement.
//
// Must happen BEFORE ontology::generate_views so that mz_sources's ontology
// annotation (entity_name = "source") is visible to the ontology index views.
// All sources/logs are already present in builtin_items at this point.
{
let source_iter = builtin_items.iter().filter_map(|b| match b {
Builtin::Source(x) => Some(*x),
_ => None,
});
let log_iter = builtin_items.iter().filter_map(|b| match b {
Builtin::Log(x) => Some(*x),
_ => None,
});
let mz_sources = builtin::make_mz_sources(source_iter, log_iter);
let mz_sources_ref: &'static BuiltinMaterializedView = Box::leak(Box::new(mz_sources));
// Insert at the original position of the old static MZ_SOURCES —
// right before mz_source_references — to preserve stable IDs for
// all items that follow it in the list.
let insert_pos = builtin_items
.iter()
.position(|b| b.name() == "mz_source_references")
.expect("mz_source_references must be present in builtin_items");
builtin_items.insert(insert_pos, Builtin::MaterializedView(mz_sources_ref));
}

// Generate mz_indexes with builtin index/log entries inlined as VALUES so
// that its SQL fingerprint changes whenever a builtin index or log is added or
// removed, forcing an explicit MigrationStep::replacement.
//
// Must happen AFTER all builtin indexes and logs have been pushed into
// builtin_items, so that make_mz_indexes sees the complete set. Must happen
// BEFORE ontology::generate_views so the ontology generator sees mz_indexes
// as a materialized view participating in catalog ontology, rather than
// being absent from builtin_items.
{
let index_iter = builtin_items.iter().filter_map(|b| match b {
Builtin::Index(x) => Some(*x),
_ => None,
});
let log_iter = builtin_items.iter().filter_map(|b| match b {
Builtin::Log(x) => Some(*x),
_ => None,
});
let mz_indexes = mz_catalog::make_mz_indexes(index_iter, log_iter);
let mz_indexes_ref: &'static BuiltinMaterializedView = Box::leak(Box::new(mz_indexes));
let insert_pos = builtin_items
.iter()
.position(|b| matches!(b, Builtin::Table(t) if t.name == "mz_index_columns"))
.expect("mz_index_columns must be present in builtin_items");
builtin_items.insert(insert_pos, Builtin::MaterializedView(mz_indexes_ref));
}

// Generate mz_object_dependencies_raw, which inlines every builtin's
// dependency edges as VALUES.
//
Expand Down Expand Up @@ -2347,134 +2292,6 @@ mod tests {
);
}

/// Verifies that the `mz_sources` materialized view fingerprint changes
/// whenever a new builtin source or log is added.
///
/// This is the correctness property that `make_mz_sources` provides: by
/// inlining the full set of builtin sources/logs as VALUES in its SQL, any
/// change to those sets is reflected in `fingerprint()`. A stale fingerprint
/// would prevent the catalog migration from replacing `mz_sources`, leaving
/// it with out-of-date data, silently serving stale builtin source rows.
#[mz_ore::test]
#[cfg_attr(miri, ignore)]
fn test_mz_sources_fingerprint_changes_with_new_builtin_source() {
let sources: Vec<&'static BuiltinSource> = BUILTINS_STATIC
.iter()
.filter_map(|b| match b {
Builtin::Source(x) => Some(*x),
_ => None,
})
.collect();
let logs: Vec<&'static BuiltinLog> = BUILTINS_STATIC
.iter()
.filter_map(|b| match b {
Builtin::Log(x) => Some(*x),
_ => None,
})
.collect();

// The fingerprint from make_mz_sources must match the live BUILTINS_STATIC entry.
let mv_base = builtin::make_mz_sources(sources.iter().copied(), logs.iter().copied());
let fp_base = Fingerprint::fingerprint(&&mv_base);

let mz_sources_static = BUILTINS_STATIC
.iter()
.find_map(|b| match b {
Builtin::MaterializedView(mv) if mv.name == "mz_sources" => Some(*mv),
_ => None,
})
.expect("mz_sources must be present in BUILTINS_STATIC");
assert_eq!(
fp_base,
Fingerprint::fingerprint(&mz_sources_static),
"make_mz_sources fingerprint must match the BUILTINS_STATIC mz_sources fingerprint"
);

// Adding an extra source must change the fingerprint, proving that
// make_mz_sources inlines the source list into its SQL.
let extra_source = sources[0];
let mv_extra = builtin::make_mz_sources(
sources.iter().copied().chain(std::iter::once(extra_source)),
logs.iter().copied(),
);
assert_ne!(
fp_base,
Fingerprint::fingerprint(&&mv_extra),
"mz_sources fingerprint must change when a builtin source is added"
);
}

/// Verifies that the `mz_indexes` materialized view fingerprint changes
/// whenever a new builtin index or log is added.
///
/// This is the correctness property that `make_mz_indexes` provides: by
/// inlining the full set of builtin indexes/logs as VALUES in its SQL,
/// any change to those sets is reflected in `fingerprint()`. A stale
/// fingerprint would prevent the catalog migration from replacing
/// `mz_indexes`, leaving it with out-of-date data, silently serving
/// stale builtin index rows.
#[mz_ore::test]
#[cfg_attr(miri, ignore)]
fn test_mz_indexes_fingerprint_changes_with_new_builtin_index() {
let indexes: Vec<&'static BuiltinIndex> = BUILTINS_STATIC
.iter()
.filter_map(|b| match b {
Builtin::Index(x) => Some(*x),
_ => None,
})
.collect();
let logs: Vec<&'static BuiltinLog> = BUILTINS_STATIC
.iter()
.filter_map(|b| match b {
Builtin::Log(x) => Some(*x),
_ => None,
})
.collect();

// The fingerprint from make_mz_indexes must match the live BUILTINS_STATIC entry.
let mv_base = mz_catalog::make_mz_indexes(indexes.iter().copied(), logs.iter().copied());
let fp_base = Fingerprint::fingerprint(&&mv_base);

let mz_indexes_static = BUILTINS_STATIC
.iter()
.find_map(|b| match b {
Builtin::MaterializedView(mv) if mv.name == "mz_indexes" => Some(*mv),
_ => None,
})
.expect("mz_indexes must be present in BUILTINS_STATIC");
assert_eq!(
fp_base,
Fingerprint::fingerprint(&mz_indexes_static),
"make_mz_indexes fingerprint must match the BUILTINS_STATIC mz_indexes fingerprint"
);

// Adding an extra index must change the fingerprint, proving that
// make_mz_indexes inlines the index list into its SQL.
let extra_index = indexes[0];
let mv_extra_index = mz_catalog::make_mz_indexes(
indexes.iter().copied().chain(std::iter::once(extra_index)),
logs.iter().copied(),
);
assert_ne!(
fp_base,
Fingerprint::fingerprint(&&mv_extra_index),
"mz_indexes fingerprint must change when a builtin index is added"
);

// Adding an extra log must also change the fingerprint, because the
// log set feeds the introspection-source-indexes CTE.
let extra_log = logs[0];
let mv_extra_log = mz_catalog::make_mz_indexes(
indexes.iter().copied(),
logs.iter().copied().chain(std::iter::once(extra_log)),
);
assert_ne!(
fp_base,
Fingerprint::fingerprint(&&mv_extra_log),
"mz_indexes fingerprint must change when a builtin log is added"
);
}

/// Because `mz_object_dependencies_raw`` is built from prior copies of the ontology and builtin
/// reporter views to get their edges since they must be built afterwards. Thus we test
/// that regenerating the view using all builtins (excluding itself) equals the SQL of the real
Expand Down
Loading
Loading