PostgreSQL: add support for CREATE AGGREGATE - #2316
fmguerreiro wants to merge 4 commits into
Conversation
LucaCappelletti94
left a comment
There was a problem hiding this comment.
Just a couple of preliminary clarification requests.
| impl FunctionParallel { | ||
| /// Returns the bare keyword for this parallel mode, without the `PARALLEL` prefix. | ||
| pub fn as_str(&self) -> &'static str { | ||
| match self { | ||
| FunctionParallel::Unsafe => write!(f, "PARALLEL UNSAFE"), | ||
| FunctionParallel::Restricted => write!(f, "PARALLEL RESTRICTED"), | ||
| FunctionParallel::Safe => write!(f, "PARALLEL SAFE"), | ||
| FunctionParallel::Unsafe => "UNSAFE", | ||
| FunctionParallel::Restricted => "RESTRICTED", | ||
| FunctionParallel::Safe => "SAFE", | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl fmt::Display for FunctionParallel { | ||
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | ||
| write!(f, "PARALLEL {}", self.as_str()) | ||
| } | ||
| } | ||
|
|
There was a problem hiding this comment.
What is the reason for this edit?
There was a problem hiding this comment.
the aggregate option prints PARALLEL = SAFE while the function form prints PARALLEL SAFE, so as_str gives the bare keyword without duplicating the strings.
Codecov Report❌ Patch coverage is
Additional details and impacted files@@ Coverage Diff @@
## main #2316 +/- ##
==========================================
+ Coverage 81.04% 81.06% +0.02%
==========================================
Files 42 42
Lines 33593 33905 +312
Branches 33593 33905 +312
==========================================
+ Hits 27224 27484 +260
- Misses 2799 2810 +11
- Partials 3570 3611 +41 ☔ View full report in Codecov by Harness. 🚀 New features to boost your workflow:
|
| } else if or_replace { | ||
| self.expected_ref( | ||
| "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE", | ||
| "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or AGGREGATE or SCHEMA or WAREHOUSE after CREATE OR REPLACE", |
There was a problem hiding this comment.
Please add a test to cover this case
There was a problem hiding this comment.
covered by parse_create_aggregate_or_replace_with_parallel.
| Keyword::READ_ONLY => Ok(AggregateModifyKind::ReadOnly), | ||
| Keyword::SHAREABLE => Ok(AggregateModifyKind::Shareable), | ||
| Keyword::READ_WRITE => Ok(AggregateModifyKind::ReadWrite), | ||
| _ => self.expected_ref( |
There was a problem hiding this comment.
Also this appears not to be covered by tests
There was a problem hiding this comment.
covered across the parse_create_aggregate_* tests, including legacy, star, empty-args, and the malformed, unclosed, and duplicate error paths.
| fn span(&self) -> Span { | ||
| match self { | ||
| CreateAggregateOption::StateTransitionFunction(name) | ||
| | CreateAggregateOption::FinalFunction(name) |
There was a problem hiding this comment.
I am unsure whether these are covered (and llvm does not instrument the lines) or only a subset of these are tested
There was a problem hiding this comment.
the span's pinned now by parse_create_aggregate_span, from the name through the last option value.
a95f12c to
822c8fc
Compare
822c8fc to
f01ccf2
Compare
LucaCappelletti94
left a comment
There was a problem hiding this comment.
I have rebased onto main. I believe that some of the errors I am reporting in this review round could have been caught by the fuzzer, so please add CREATE AGGREGATE seeds to fuzz/fuzz_seeds/ (such as the one below), run fuzz_parse_roundtrip and fuzz_postgres_accepts over them as described in docs/fuzzing.md. If you happen to find other interesting but PR-unrelated bugs by all means do file them as issues and/or PRs.
The seeds below follow the shape pg_dump writes and all of them parse in PostgreSQL 18. The last one fails on this branch today, which is the SORTOP issue from the inline notes.
-- fuzz/fuzz_seeds/create_aggregate.sql
CREATE AGGREGATE public.my_avg(numeric) (
SFUNC = numeric_avg_accum,
STYPE = internal,
SSPACE = 128,
INITCOND = '0',
FINALFUNC = numeric_avg,
FINALFUNC_EXTRA,
FINALFUNC_MODIFY = SHAREABLE,
COMBINEFUNC = numeric_avg_combine,
SERIALFUNC = numeric_avg_serialize,
DESERIALFUNC = numeric_avg_deserialize,
MSFUNC = numeric_avg_accum,
MINVFUNC = numeric_accum_inv,
MSTYPE = internal,
MSSPACE = 64,
MFINALFUNC = numeric_avg,
MFINALFUNC_EXTRA,
MFINALFUNC_MODIFY = READ_WRITE,
MINITCOND = '0',
PARALLEL = safe
)
-- fuzz/fuzz_seeds/create_aggregate_star.sql
CREATE OR REPLACE AGGREGATE my_count(*) (SFUNC = int8inc, STYPE = bigint, INITCOND = '0')
-- fuzz/fuzz_seeds/create_aggregate_legacy.sql
CREATE AGGREGATE my_sum (BASETYPE = int4, SFUNC = int4pl, STYPE = int4, INITCOND = '0')
-- fuzz/fuzz_seeds/create_aggregate_args.sql
CREATE AGGREGATE s.my_agg(IN a numeric(10,2), VARIADIC b text[]) (SFUNC = s.f, STYPE = integer[], INITCOND = '{}', SORTOP = <, HYPOTHETICAL)
-- fuzz/fuzz_seeds/create_aggregate_sortop.sql
CREATE AGGREGATE public.my_min(integer) (SFUNC = int4smaller, STYPE = integer, SORTOP = OPERATOR(pg_catalog.<))| Keyword::MINITCOND => { | ||
| CreateAggregateOption::MovingInitialCondition(self.parse_value()?) | ||
| } | ||
| Keyword::SORTOP => CreateAggregateOption::SortOperator(self.parse_operator_name()?), |
There was a problem hiding this comment.
You should accept the OPERATOR(schema.op) form here, as COMMUTATOR already does in CREATE OPERATOR. pg_dump writes every sort operator that way (SORTOP = OPERATOR(pg_catalog.<)), so dumped aggregates currently fail with Expected: ), found: (.
| Keyword::SORTOP => CreateAggregateOption::SortOperator(self.parse_operator_name()?), | |
| Keyword::SORTOP => { | |
| let operator = if self.parse_keyword(Keyword::OPERATOR) { | |
| self.expect_token(&Token::LParen)?; | |
| let operator = self.parse_operator_name()?; | |
| self.expect_token(&Token::RParen)?; | |
| operator | |
| } else { | |
| self.parse_operator_name()? | |
| }; | |
| CreateAggregateOption::SortOperator(operator) | |
| } |
| Self::MovingFinalFunctionExtra => write!(f, "MFINALFUNC_EXTRA"), | ||
| Self::MovingFinalFunctionModify(kind) => write!(f, "MFINALFUNC_MODIFY = {kind}"), | ||
| Self::MovingInitialCondition(cond) => write!(f, "MINITCOND = {cond}"), | ||
| Self::SortOperator(name) => write!(f, "SORTOP = {name}"), |
There was a problem hiding this comment.
You should render a qualified sort operator inside OPERATOR(...), as ORDER BY ... USING does in query.rs. PostgreSQL rejects the bare SORTOP = pg_catalog.< with syntax error at or near "<".
| Self::SortOperator(name) => write!(f, "SORTOP = {name}"), | |
| Self::SortOperator(name) if name.0.len() > 1 => write!(f, "SORTOP = OPERATOR({name})"), | |
| Self::SortOperator(name) => write!(f, "SORTOP = {name}"), |
| "CREATE AGGREGATE my_min (INT) (SFUNC = my_sfunc, STYPE = INT, SSPACE = 128, SORTOP = <)", | ||
| ); | ||
| pg_and_generic().verified_stmt( | ||
| "CREATE AGGREGATE my_min2 (INT) (SFUNC = my_sfunc, STYPE = INT, SORTOP = pg_catalog.<)", |
There was a problem hiding this comment.
You should round-trip the form PostgreSQL accepts and pg_dump emits. The current string is a syntax error in PostgreSQL 18, and this one fails on the current parser.
| "CREATE AGGREGATE my_min2 (INT) (SFUNC = my_sfunc, STYPE = INT, SORTOP = pg_catalog.<)", | |
| "CREATE AGGREGATE my_min2 (INT) (SFUNC = my_sfunc, STYPE = INT, SORTOP = OPERATOR(pg_catalog.<))", |
| CreateAggregateArgs::List( | ||
| self.parse_comma_separated0(Parser::parse_function_arg, Token::RParen)?, | ||
| ) |
There was a problem hiding this comment.
You should require at least one argument. PostgreSQL rejects CREATE AGGREGATE my_agg () (...) with syntax error at or near ")", and a zero-argument aggregate is spelled (*), which Star already covers.
| CreateAggregateArgs::List( | |
| self.parse_comma_separated0(Parser::parse_function_arg, Token::RParen)?, | |
| ) |
There was a problem hiding this comment.
fixed in 41ef183. () is a parse error now.
| /// An explicit argument list, possibly empty: `()`, `(NUMERIC)`, | ||
| /// `(input INT, VARIADIC tail TEXT)`. |
There was a problem hiding this comment.
You should drop () from the documented forms, since PostgreSQL does not accept it.
| /// An explicit argument list, possibly empty: `()`, `(NUMERIC)`, | |
| /// `(input INT, VARIADIC tail TEXT)`. | |
| /// An explicit argument list: `(NUMERIC)`, `(input INT, VARIADIC tail TEXT)`. |
| fn parse_create_aggregate_empty_args() { | ||
| let stmt = pg_and_generic() | ||
| .verified_stmt("CREATE AGGREGATE my_agg () (SFUNC = my_sfunc, STYPE = INT)"); | ||
| match stmt { | ||
| Statement::CreateAggregate(agg) => { | ||
| assert_eq!(agg.args, CreateAggregateArgs::List(vec![])); | ||
| } | ||
| _ => panic!("Expected CreateAggregate, got: {stmt:?}"), | ||
| } |
There was a problem hiding this comment.
You should turn this test into the negative case. It fails on the current parser because () parses.
| fn parse_create_aggregate_empty_args() { | |
| let stmt = pg_and_generic() | |
| .verified_stmt("CREATE AGGREGATE my_agg () (SFUNC = my_sfunc, STYPE = INT)"); | |
| match stmt { | |
| Statement::CreateAggregate(agg) => { | |
| assert_eq!(agg.args, CreateAggregateArgs::List(vec![])); | |
| } | |
| _ => panic!("Expected CreateAggregate, got: {stmt:?}"), | |
| } | |
| fn parse_create_aggregate_rejects_empty_args() { | |
| assert_eq!( | |
| pg_and_generic() | |
| .parse_sql_statements("CREATE AGGREGATE my_agg () (SFUNC = my_sfunc, STYPE = INT)") | |
| .unwrap_err() | |
| .to_string(), | |
| "sql parser error: Expected: a data type name, found: )" | |
| ); |
There was a problem hiding this comment.
done in 41ef183, asserts the error your suggestion had.
Accept and render OPERATOR(schema.op) for SORTOP, matching pg_dump. Require at least one explicit argument; empty () is a syntax error. Add CREATE AGGREGATE fuzz seeds.
| } else { | ||
| CreateAggregateArgs::List(self.parse_comma_separated(Parser::parse_function_arg)?) | ||
| }; |
There was a problem hiding this comment.
You should accept ordered-set argument lists. pg_dump writes every ordered-set aggregate as CREATE AGGREGATE public.my_pct(double precision ORDER BY double precision) (...), and PostgreSQL 18 rejects HYPOTHETICAL anywhere else (only ordered-set aggregates can be hypothetical), yet (FLOAT8 ORDER BY FLOAT8), (ORDER BY anyelement) and (VARIADIC "any" ORDER BY VARIADIC "any") all fail to parse here.
Could look something like this, I will try to prepare some red tests shortly.
| } else { | |
| CreateAggregateArgs::List(self.parse_comma_separated(Parser::parse_function_arg)?) | |
| }; | |
| } else { | |
| let direct = if self.peek_keyword(Keyword::ORDER) { | |
| vec![] | |
| } else { | |
| self.parse_comma_separated(Parser::parse_function_arg)? | |
| }; | |
| if self.parse_keywords(&[Keyword::ORDER, Keyword::BY]) { | |
| CreateAggregateArgs::OrderedSet { | |
| direct, | |
| aggregated: self.parse_comma_separated(Parser::parse_function_arg)?, | |
| } | |
| } else { | |
| CreateAggregateArgs::List(direct) | |
| } | |
| }; |
There was a problem hiding this comment.
You should stop the name/type disambiguation at ORDER, since it currently reads FLOAT8 ORDER as an argument named FLOAT8 of type ORDER. ORDER is reserved in PostgreSQL, so no valid CREATE FUNCTION argument changes.
| // DEFAULT and ORDER will be parsed as `DataType::Custom`, which is undesirable in this context | |
| fn parse_data_type_no_default(parser: &mut Parser) -> Result<DataType, ParserError> { | |
| if parser.peek_keyword(Keyword::DEFAULT) || parser.peek_keyword(Keyword::ORDER) { |
| Star, | ||
| /// An explicit argument list: `(NUMERIC)`, | ||
| /// `(input INT, VARIADIC tail TEXT)`. | ||
| List(Vec<OperateFunctionArg>), |
There was a problem hiding this comment.
Adding an ordered-set variant I proposed in the other suggestions. I am unsure whether it could be representable in some other manner.
| List(Vec<OperateFunctionArg>), | |
| List(Vec<OperateFunctionArg>), | |
| /// An ordered-set argument list: `(FLOAT8 ORDER BY FLOAT8)`, | |
| /// `(ORDER BY anyelement)`. | |
| OrderedSet { | |
| /// The direct arguments before `ORDER BY`, possibly empty. | |
| direct: Vec<OperateFunctionArg>, | |
| /// The aggregated arguments after `ORDER BY`. | |
| aggregated: Vec<OperateFunctionArg>, | |
| }, |
| match &self.args { | ||
| CreateAggregateArgs::Legacy => {} | ||
| CreateAggregateArgs::Star => write!(f, " (*)")?, | ||
| CreateAggregateArgs::List(args) => write!(f, " ({})", display_comma_separated(args))?, |
There was a problem hiding this comment.
And, following the above refactoring, also the display is needed:
| CreateAggregateArgs::List(args) => write!(f, " ({})", display_comma_separated(args))?, | |
| CreateAggregateArgs::List(args) => write!(f, " ({})", display_comma_separated(args))?, | |
| CreateAggregateArgs::OrderedSet { direct, aggregated } => { | |
| write!(f, " (")?; | |
| if !direct.is_empty() { | |
| write!(f, "{} ", display_comma_separated(direct))?; | |
| } | |
| write!(f, "ORDER BY {})", display_comma_separated(aggregated))?; | |
| } |
| fn span(&self) -> Span { | ||
| match self { | ||
| CreateAggregateArgs::Legacy | CreateAggregateArgs::Star => Span::empty(), | ||
| CreateAggregateArgs::List(args) => union_spans(args.iter().map(|arg| arg.span())), |
There was a problem hiding this comment.
| CreateAggregateArgs::List(args) => union_spans(args.iter().map(|arg| arg.span())), | |
| CreateAggregateArgs::List(args) => union_spans(args.iter().map(|arg| arg.span())), | |
| CreateAggregateArgs::OrderedSet { direct, aggregated } => { | |
| union_spans(direct.iter().chain(aggregated).map(|arg| arg.span())) | |
| } |
| pg_and_generic().verified_stmt( | ||
| "CREATE AGGREGATE percentile (FLOAT8) (SFUNC = ordered_set_transition, STYPE = internal, FINALFUNC = percentile_final, FINALFUNC_MODIFY = READ_WRITE, HYPOTHETICAL)", | ||
| ); |
There was a problem hiding this comment.
Adding red test and moving HYPOTHETICAL onto the hypothetical-set one.
| pg_and_generic().verified_stmt( | |
| "CREATE AGGREGATE percentile (FLOAT8) (SFUNC = ordered_set_transition, STYPE = internal, FINALFUNC = percentile_final, FINALFUNC_MODIFY = READ_WRITE, HYPOTHETICAL)", | |
| ); | |
| pg_and_generic().verified_stmt( | |
| "CREATE AGGREGATE percentile (FLOAT8 ORDER BY FLOAT8) (SFUNC = ordered_set_transition, STYPE = internal, FINALFUNC = percentile_final, FINALFUNC_MODIFY = READ_WRITE)", | |
| ); | |
| pg_and_generic().verified_stmt( | |
| "CREATE AGGREGATE my_mode (ORDER BY anyelement) (SFUNC = ordered_set_transition, STYPE = internal, FINALFUNC = mode_final, FINALFUNC_EXTRA)", | |
| ); | |
| pg_and_generic().verified_stmt( | |
| "CREATE AGGREGATE my_rank (VARIADIC \"any\" ORDER BY VARIADIC \"any\") (SFUNC = ordered_set_transition_multi, STYPE = internal, FINALFUNC = rank_final, FINALFUNC_EXTRA, HYPOTHETICAL)", | |
| ); |
|
|
||
| let duplicate = pg_and_generic() | ||
| .parse_sql_statements("CREATE AGGREGATE foo (INT) (SFUNC = f, SFUNC = g, STYPE = INT)") | ||
| .unwrap_err(); | ||
| assert_eq!( | ||
| duplicate.to_string(), | ||
| "sql parser error: Duplicate CREATE AGGREGATE option: SFUNC" | ||
| ); |
There was a problem hiding this comment.
You should remove the duplicate-option assertion together with the check.
| let duplicate = pg_and_generic() | |
| .parse_sql_statements("CREATE AGGREGATE foo (INT) (SFUNC = f, SFUNC = g, STYPE = INT)") | |
| .unwrap_err(); | |
| assert_eq!( | |
| duplicate.to_string(), | |
| "sql parser error: Duplicate CREATE AGGREGATE option: SFUNC" | |
| ); |
| let mut seen: Vec<Keyword> = Vec::new(); | ||
| let options = self.parse_comma_separated(|parser| { | ||
| let start = parser.peek_token_ref().span.start; | ||
| let keyword = parser.parse_create_aggregate_option_key()?; | ||
| if seen.contains(&keyword) { | ||
| return parser_err!( | ||
| format!("Duplicate CREATE AGGREGATE option: {keyword:?}"), | ||
| start | ||
| ); | ||
| } | ||
| seen.push(keyword); | ||
| parser.parse_create_aggregate_option(keyword) | ||
| })?; |
There was a problem hiding this comment.
You should drop the duplicate-option check. PostgreSQL 18 accepts CREATE AGGREGATE dup(int4) (SFUNC = int4mi, STYPE = int4, SFUNC = int4pl) and keeps the last value (aggtransfn is int4pl), so this rejects valid SQL.
| let mut seen: Vec<Keyword> = Vec::new(); | |
| let options = self.parse_comma_separated(|parser| { | |
| let start = parser.peek_token_ref().span.start; | |
| let keyword = parser.parse_create_aggregate_option_key()?; | |
| if seen.contains(&keyword) { | |
| return parser_err!( | |
| format!("Duplicate CREATE AGGREGATE option: {keyword:?}"), | |
| start | |
| ); | |
| } | |
| seen.push(keyword); | |
| parser.parse_create_aggregate_option(keyword) | |
| })?; | |
| let options = self.parse_comma_separated(|parser| { | |
| let keyword = parser.parse_create_aggregate_option_key()?; | |
| parser.parse_create_aggregate_option(keyword) | |
| })?; |
|
cool, didnt know about the fuzzing in the codebase. added your five seeds in 41ef183. ran fuzz_parse_roundtrip and fuzz_postgres_accepts over them, both clean. |
Parses PostgreSQL
CREATE [OR REPLACE] AGGREGATE, both the modernname (arg_types) (options)form and the legacyBASETYPE = ...form. Options cover the full set from the docs: state and final functions, the moving-aggregate variants,PARALLEL,SORTOP,HYPOTHETICAL,FINALFUNC_MODIFY.Ordered-set aggregates (
ORDER BYin the argument list) are out of scope.Round-trip tests cover both syntaxes, star and variadic args, the moving-aggregate options, and an unknown-option error.