Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .schema/pgdog.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@
"query_log_stdout": false,
"query_parser": "auto",
"query_parser_enabled": false,
"query_parser_engine": "pg_query_protobuf",
"query_parser_engine": "pg_query_raw",
"query_size_limit": null,
"query_size_limit_action": "warn",
"query_timeout": 9223372036854775807,
Expand Down Expand Up @@ -1094,7 +1094,7 @@
"query_parser_engine": {
"description": "Underlying parser implementation used to analyze SQL queries.",
"$ref": "#/$defs/QueryParserEngine",
"default": "pg_query_protobuf"
"default": "pg_query_raw"
},
"query_size_limit": {
"description": "Maximum size, in bytes, of a query message (`Query` or `Parse`)\nreceived from a client, including the 5-byte message header.\nProtects the query parser from very large SQL texts; other\nprotocol messages (e.g. `Bind`, `CopyData`) are not affected.\nDepending on the setting `query_size_limit_action` oversized messages are\neither logged or blocked.\n\n_Default:_ `None` (disabled)\n\n<https://docs.pgdog.dev/configuration/pgdog.toml/general/#query_size_limit>",
Expand Down Expand Up @@ -1740,7 +1740,7 @@
"engine": {
"description": "Query parser engine used.",
"$ref": "#/$defs/QueryParserEngine",
"default": "pg_query_protobuf"
"default": "pg_query_raw"
},
"level": {
"description": "Query parser level.",
Expand All @@ -1757,12 +1757,12 @@
"description": "Underlying parser implementation used to analyze SQL queries.",
"oneOf": [
{
"description": "Use the protobuf parse tree from `pg_query` (default).",
"description": "Use the protobuf parse tree from `pg_query`.",
"type": "string",
"const": "pg_query_protobuf"
},
{
"description": "Use the raw JSON parse tree from `pg_query`.",
"description": "Use the raw JSON parse tree from `pg_query` (default).",
"type": "string",
"const": "pg_query_raw"
}
Expand Down
4 changes: 4 additions & 0 deletions pgdog-config/src/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,10 @@ impl Config {
);
}

if !raw_query_parser {
warn!(r#""query_parser_engine" is deprecated and uses the "pg_query_raw" option now"#)
}

if !self.sharded_mappings.is_empty() {
warn!(
"`[[sharded_mappings]]` config is deprecated, use `[[sharded_tables.mapping]]` instead"
Expand Down
8 changes: 4 additions & 4 deletions pgdog-config/src/sharding.rs
Original file line number Diff line number Diff line change
Expand Up @@ -473,10 +473,10 @@ pub enum QueryParserLevel {
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, Default, JsonSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum QueryParserEngine {
/// Use the protobuf parse tree from `pg_query` (default).
#[default]
/// Use the protobuf parse tree from `pg_query`.
PgQueryProtobuf,
/// Use the raw JSON parse tree from `pg_query`.
/// Use the raw JSON parse tree from `pg_query` (default).
#[default]
PgQueryRaw,
}

Expand Down Expand Up @@ -727,7 +727,7 @@ database = "production"
assert_eq!(config.query_parsers[0].level, QueryParserLevel::Auto);
assert_eq!(
config.query_parsers[0].engine,
QueryParserEngine::PgQueryProtobuf
QueryParserEngine::PgQueryRaw
);
}
}
13 changes: 2 additions & 11 deletions pgdog/src/backend/pool/cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
use futures::future::try_join_all;
use parking_lot::Mutex;
use pgdog_config::{
LoadSchema, PreparedStatements, QueryParser, QueryParserEngine, QueryParserLevel, Rewrite,
RewriteMode, users::PasswordKind,
LoadSchema, PreparedStatements, QueryParser, QueryParserLevel, Rewrite, RewriteMode,
users::PasswordKind,
};
use std::{sync::Arc, time::Duration};

Expand Down Expand Up @@ -68,7 +68,6 @@ pub struct Cluster {
expanded_explain: bool,
query_parser: QueryParserLevel,
client_connection_recovery: ConnectionRecovery,
query_parser_engine: QueryParserEngine,
log_min_duration_parse: Option<Duration>,
log_query_sample_length: usize,
reload_schema_on_ddl: bool,
Expand Down Expand Up @@ -118,7 +117,6 @@ impl Default for Cluster {
expanded_explain: Default::default(),
query_parser: Default::default(),
client_connection_recovery: Default::default(),
query_parser_engine: Default::default(),
log_min_duration_parse: Default::default(),
log_query_sample_length: Default::default(),
reload_schema_on_ddl: Default::default(),
Expand Down Expand Up @@ -150,8 +148,6 @@ pub struct ShardingSchema {
pub schemas: ShardedSchemas,
/// Rewrite config.
pub rewrite: Rewrite,
/// Query parser engine.
pub query_parser_engine: QueryParserEngine,
pub log_min_duration_parse: Option<Duration>,
pub log_query_sample_length: usize,
}
Expand Down Expand Up @@ -206,7 +202,6 @@ pub struct ClusterConfig<'a> {
dry_run: bool,
expanded_explain: bool,
query_parser: QueryParserLevel,
query_parser_engine: QueryParserEngine,
log_min_duration_parse: Option<Duration>,
log_query_sample_length: usize,
client_connection_recovery: ConnectionRecovery,
Expand Down Expand Up @@ -276,7 +271,6 @@ impl<'a> ClusterConfig<'a> {
dry_run: general.dry_run,
expanded_explain: general.expanded_explain,
query_parser: query_parser.level,
query_parser_engine: query_parser.engine,
log_min_duration_parse: general.log_min_duration_parse(),
log_query_sample_length: general.log_query_sample_length,
client_connection_recovery: general.client_connection_recovery,
Expand Down Expand Up @@ -327,7 +321,6 @@ impl Cluster {
query_parser,
client_connection_recovery,
lsn_check_interval,
query_parser_engine,
log_min_duration_parse,
log_query_sample_length,
reload_schema_on_ddl,
Expand Down Expand Up @@ -396,7 +389,6 @@ impl Cluster {
expanded_explain,
query_parser,
client_connection_recovery,
query_parser_engine,
log_min_duration_parse,
log_query_sample_length,
reload_schema_on_ddl,
Expand Down Expand Up @@ -615,7 +607,6 @@ impl Cluster {
tables: self.sharded_tables.clone(),
schemas: self.sharded_schemas.clone(),
rewrite: self.rewrite.clone(),
query_parser_engine: self.query_parser_engine,
log_min_duration_parse: self.log_min_duration_parse,
log_query_sample_length: self.log_query_sample_length,
}
Expand Down
12 changes: 2 additions & 10 deletions pgdog/src/backend/replication/logical/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,7 @@ impl Orchestrator {
/// Replace the publisher entirely (discards LSN state). Only valid
/// when starting a fresh replication phase, e.g. after cutover.
fn refresh_publisher(&mut self) {
let publisher = Publisher::new(
&self.publication,
config().config.general.query_parser_engine,
self.replication_slot.clone(),
);
let publisher = Publisher::new(&self.publication, self.replication_slot.clone());
self.publisher = Arc::new(Mutex::new(publisher));
}

Expand Down Expand Up @@ -562,11 +558,7 @@ mod tests {
let cluster = Cluster::new_test(config);
let publication = "test_pub".to_owned();
let replication_slot = "test_slot".to_owned();
let publisher = Publisher::new(
&publication,
config.config.general.query_parser_engine,
replication_slot.clone(),
);
let publisher = Publisher::new(&publication, replication_slot.clone());
Self {
source: cluster.clone(),
destination: cluster,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ mod test {
use crate::net::messages::replication::logical::tuple_data::{
TupleData, text_col, toasted_col,
};
use pgdog_config::QueryParserEngine;
use pgdog_postgres_types::Oid;

fn make_table(columns: Vec<(&str, bool)>) -> Table {
Expand Down Expand Up @@ -124,7 +123,6 @@ mod test {
})
.collect(),
lsn: Lsn::default(),
query_parser_engine: QueryParserEngine::default(),
}
}

Expand Down
30 changes: 6 additions & 24 deletions pgdog/src/backend/replication/logical/publisher/publisher_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ use std::time::Duration;

use futures::stream::{FuturesUnordered, StreamExt};
use parking_lot::Mutex;
use pgdog_config::QueryParserEngine;
use tokio::select;
use tokio::task::JoinHandle;
use tokio::time::Instant;
Expand Down Expand Up @@ -51,8 +50,6 @@ pub struct Publisher {
tables: HashMap<usize, Vec<Table>>,
/// Replication slots.
slots: HashMap<usize, ReplicationSlot>,
/// Query parser engine.
query_parser_engine: QueryParserEngine,
/// Replication lag.
replication_lag: Arc<Mutex<HashMap<usize, i64>>>,
/// Last transaction.
Expand All @@ -62,16 +59,11 @@ pub struct Publisher {
}

impl Publisher {
pub fn new(
publication: &str,
query_parser_engine: QueryParserEngine,
slot_name: String,
) -> Self {
pub fn new(publication: &str, slot_name: String) -> Self {
Self {
publication: publication.to_string(),
tables: HashMap::new(),
slots: HashMap::new(),
query_parser_engine,
replication_lag: Arc::new(Mutex::new(HashMap::new())),
last_transaction: Arc::new(Mutex::new(None)),
slot_name,
Expand Down Expand Up @@ -126,8 +118,7 @@ impl Publisher {
for (number, shard) in source.shards().iter().enumerate() {
// Load tables from publication.
let mut primary = shard.primary(&Request::default()).await?;
let tables =
Table::load(&self.publication, &mut primary, self.query_parser_engine).await?;
let tables = Table::load(&self.publication, &mut primary).await?;

// For data sync, split omni tables evenly between shards.
if data_sync {
Expand Down Expand Up @@ -562,7 +553,6 @@ mod test {
identity: true,
}],
lsn: Lsn::from_i64(lsn),
query_parser_engine: QueryParserEngine::default(),
}
}

Expand Down Expand Up @@ -594,7 +584,7 @@ mod test {
fn distribute_omnisharded_tables_initializes_missing_shards() {
let config = config();
let cluster = Cluster::new_test(&config);
let mut publisher = Publisher::new("test", QueryParserEngine::default(), "slot".into());
let mut publisher = Publisher::new("test", "slot".into());
let table = make_table("public", "omni_only", 0);

publisher.distribute_omnisharded_tables(HashMap::from([(table.key(), table)]), &cluster);
Expand Down Expand Up @@ -638,11 +628,7 @@ mod test {
source.launch();
let dest = Cluster::new_test(&config());

let mut publisher = Publisher::new(
"publication_no_pk_validation",
QueryParserEngine::default(),
"sync_test_slot".into(),
);
let mut publisher = Publisher::new("publication_no_pk_validation", "sync_test_slot".into());

// Validation must fire before the copy begins.
let result = publisher
Expand Down Expand Up @@ -700,11 +686,8 @@ mod test {
source.launch();
let dest = Cluster::new_test(&config());

let mut publisher = Publisher::new(
"publication_sync_only_no_pk",
QueryParserEngine::default(),
"sync_only_no_pk_slot".into(),
);
let mut publisher =
Publisher::new("publication_sync_only_no_pk", "sync_only_no_pk_slot".into());

let cancel = CancellationToken::new();
cancel.cancel();
Expand Down Expand Up @@ -752,7 +735,6 @@ mod test {

let mut publisher = Publisher::new(
"pub_full_identity_nothing_test",
QueryParserEngine::default(),
"pub_full_identity_nothing_slot".into(),
);

Expand Down
23 changes: 4 additions & 19 deletions pgdog/src/backend/replication/logical/publisher/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

use std::time::Duration;

use pgdog_config::QueryParserEngine;
use tokio::select;
use tracing::error;

Expand Down Expand Up @@ -38,8 +37,6 @@ pub struct Table {
pub columns: Vec<PublicationTableColumn>,
/// Table data as of this LSN.
pub lsn: Lsn,
/// Query parser engine.
pub query_parser_engine: QueryParserEngine,
}

/// An enumerated view over a subset of a table's columns.
Expand Down Expand Up @@ -141,11 +138,7 @@ where
}

impl Table {
pub async fn load(
publication: &str,
server: &mut Server,
query_parser_engine: QueryParserEngine,
) -> Result<Vec<Self>, Error> {
pub async fn load(publication: &str, server: &mut Server) -> Result<Vec<Self>, Error> {
let tables = PublicationTable::load(publication, server).await?;
let mut results = vec![];

Expand All @@ -159,7 +152,6 @@ impl Table {
identity,
columns,
lsn: Lsn::default(),
query_parser_engine,
});
}

Expand Down Expand Up @@ -526,7 +518,6 @@ mod test {
};
use pg_raw_parse::parse;

use crate::config::config;
use crate::net::messages::replication::logical::tuple_data::{
TupleData, text_col, toasted_col,
};
Expand Down Expand Up @@ -558,7 +549,6 @@ mod test {
})
.collect(),
lsn: Lsn::default(),
query_parser_engine: QueryParserEngine::default(),
}
}

Expand Down Expand Up @@ -834,13 +824,9 @@ mod test {
crate::logger();

let mut publication = setup_publication().await;
let tables = Table::load(
"publication_test",
&mut publication.server,
config().config.general.query_parser_engine,
)
.await
.unwrap();
let tables = Table::load("publication_test", &mut publication.server)
.await
.unwrap();

assert_eq!(tables.len(), 2);

Expand Down Expand Up @@ -878,7 +864,6 @@ mod test {
identity,
columns,
lsn: Lsn::default(),
query_parser_engine: QueryParserEngine::default(),
}
}

Expand Down
2 changes: 0 additions & 2 deletions pgdog/src/backend/replication/logical/subscriber/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,6 @@ mod test {
Column, Identifier, TupleData, text_col, toasted_col,
},
};
use pgdog_config::QueryParserEngine;
use pgdog_postgres_types::{Format, Oid};

use super::*;
Expand Down Expand Up @@ -142,7 +141,6 @@ mod test {
})
.collect(),
lsn: Default::default(),
query_parser_engine: QueryParserEngine::default(),
}
}

Expand Down
Loading
Loading