From 8b11d9cfbb5bdf702e02f61f61b50e7578361227 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Fri, 31 Jul 2026 21:44:18 +0530 Subject: [PATCH 1/3] HIVE-29786: Support SELECT * EXCLUDE syntax for column omission --- .../hadoop/hive/ql/parse/FromClauseParser.g | 8 +- .../hadoop/hive/ql/parse/HiveLexerParent.g | 1 + .../hadoop/hive/ql/parse/IdentifiersParser.g | 1 + .../hadoop/hive/ql/parse/CalcitePlanner.java | 5 +- .../hive/ql/parse/SemanticAnalyzer.java | 41 ++- .../queries/clientpositive/select_exclude.q | 37 +++ .../clientpositive/llap/select_exclude.q.out | 292 ++++++++++++++++++ 7 files changed, 376 insertions(+), 9 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/select_exclude.q create mode 100644 ql/src/test/results/clientpositive/llap/select_exclude.q.out diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index ad34eddbc1ae..2822c66d1c1e 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -48,10 +48,10 @@ catch (RecognitionException e) { //----------------------------------------------------------------------------------- tableAllColumns - : STAR - -> ^(TOK_ALLCOLREF) - | tableName DOT STAR - -> ^(TOK_ALLCOLREF tableName) + : STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)? + -> ^(TOK_ALLCOLREF columnNameList?) + | tableName DOT STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)? + -> ^(TOK_ALLCOLREF tableName columnNameList?) ; // (table|column) diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g index a96812d49698..d784056d7831 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g @@ -399,6 +399,7 @@ KW_SYSTEM_TIME: 'SYSTEM_TIME'; KW_SYSTEM_VERSION: 'SYSTEM_VERSION'; KW_EXPIRE_SNAPSHOTS: 'EXPIRE_SNAPSHOTS'; KW_REWRITE_MANIFESTS: 'REWRITE_MANIFESTS'; +KW_EXCLUDE: 'EXCLUDE'; KW_SET_CURRENT_SNAPSHOT: 'SET_CURRENT_SNAPSHOT'; KW_BRANCH: 'BRANCH'; KW_SNAPSHOTS: 'SNAPSHOTS'; diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 37fd6187d16d..c7e63cd286ac 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -1026,6 +1026,7 @@ nonReserved | KW_TRIM | KW_SPEC | KW_SYSTEM_TIME | KW_SYSTEM_VERSION + | KW_EXCLUDE | KW_EXPIRE_SNAPSHOTS | KW_REWRITE_MANIFESTS | KW_SET_CURRENT_SNAPSHOT diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 25489f477f55..8a79011d4614 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -4366,8 +4366,9 @@ private Pair internalGenSelectLogicalPlan(QB qb, RelNode s // 6.4 Build ExprNode corresponding to colums if (expr.getType() == HiveParser.TOK_ALLCOLREF) { - pos = genRexNodeRegex(".*", - expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), + // Parse SELECT * EXCLUDE columns and pass them to the Calcite engine for exclusion + String starTabAlias = SemanticAnalyzer.processAllColRefAndExclude(expr, inputRR, excludedColumns); + pos = genRexNodeRegex(".*", starTabAlias, expr, columnList, excludedColumns, inputRR, starRR, pos, outputRR, qb.getAliases(), true); } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7042cff53874..d24515bd951e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -4737,6 +4737,35 @@ static boolean isRegex(String pattern, HiveConf conf) { return false; } + public static String processAllColRefAndExclude( + ASTNode expr, RowResolver inputRR, Set excludedColumns) throws SemanticException { + // Check if the query uses SELECT * EXCLUDE. If it does, grab the table + // alias (like t.*) and build a list of the columns the user wants to exclude. + String starTabAlias = null; + ASTNode excludeNode = null; + if (expr.getChildCount() > 0) { + ASTNode firstChild = (ASTNode) expr.getChild(0); + if (firstChild.getType() == HiveParser.TOK_TABCOLNAME) { + excludeNode = firstChild; + } else { + starTabAlias = getUnescapedName(firstChild).toLowerCase(); + if (expr.getChildCount() > 1) { + excludeNode = (ASTNode) expr.getChild(1); + } + } + } + + if (excludeNode != null) { + for (int e = 0; e < excludeNode.getChildCount(); e++) { + String excludeColName = unescapeIdentifier(excludeNode.getChild(e).getText()).toLowerCase(); + ColumnInfo colInfo = inputRR.get(starTabAlias, excludeColName); + if (colInfo != null) { + excludedColumns.add(colInfo); + } + } + } + return starTabAlias; + } private Operator genSelectPlan(String dest, QB qb, Operator input, Operator inputForSelectStar) throws SemanticException { @@ -4914,9 +4943,15 @@ private Operator genSelectPlan(String dest, ASTNode selExprList, QB qb, Opera // The real expression if (expr.getType() == HiveParser.TOK_ALLCOLREF) { int initPos = pos; - pos = genExprNodeDescRegex(".*", expr.getChildCount() == 0 ? null - : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + + Set excludeCols = new HashSet<>(); + String starTabAlias = processAllColRefAndExclude(expr, inputRR, excludeCols); + if (excludeCols.isEmpty()) { + excludeCols = null; + } + + pos = genExprNodeDescRegex(".*", starTabAlias, + expr, colList, excludeCols, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); if (unparseTranslator.isEnabled()) { offset += pos - initPos - 1; } diff --git a/ql/src/test/queries/clientpositive/select_exclude.q b/ql/src/test/queries/clientpositive/select_exclude.q new file mode 100644 index 000000000000..1ed816fd786f --- /dev/null +++ b/ql/src/test/queries/clientpositive/select_exclude.q @@ -0,0 +1,37 @@ +CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +); + +INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'); +INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200'); + +-- Exclude a single column +EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude; +SELECT * EXCLUDE (email) FROM test_exclude; + +-- Exclude multiple columns +EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude; +SELECT * EXCLUDE (email, address, phone) FROM test_exclude; + +-- Exclude with table alias +EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t; +SELECT t.* EXCLUDE (id, phone) FROM test_exclude t; + +-- Exclude with JOIN +CREATE TABLE test_exclude_join ( + id INT, + department STRING +); +INSERT INTO test_exclude_join VALUES (1, 'Engineering'); +INSERT INTO test_exclude_join VALUES (2, 'Sales'); + +EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id; + +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id; diff --git a/ql/src/test/results/clientpositive/llap/select_exclude.q.out b/ql/src/test/results/clientpositive/llap/select_exclude.q.out new file mode 100644 index 000000000000..852e58541792 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/select_exclude.q.out @@ -0,0 +1,292 @@ +PREHOOK: query: CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_exclude +POSTHOOK: query: CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_exclude +PREHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude +POSTHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude +POSTHOOK: Lineage: test_exclude.address SCRIPT [] +POSTHOOK: Lineage: test_exclude.email SCRIPT [] +POSTHOOK: Lineage: test_exclude.id SCRIPT [] +POSTHOOK: Lineage: test_exclude.name SCRIPT [] +POSTHOOK: Lineage: test_exclude.phone SCRIPT [] +PREHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude +POSTHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude +POSTHOOK: Lineage: test_exclude.address SCRIPT [] +POSTHOOK: Lineage: test_exclude.email SCRIPT [] +POSTHOOK: Lineage: test_exclude.id SCRIPT [] +POSTHOOK: Lineage: test_exclude.name SCRIPT [] +POSTHOOK: Lineage: test_exclude.phone SCRIPT [] +PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: test_exclude + Select Operator + expressions: id (type: int), name (type: string), address (type: string), phone (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +1 Alice 123 Apple St 555-0100 +2 Bob 456 Banana Ave 555-0200 +PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: test_exclude + Select Operator + expressions: id (type: int), name (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +1 Alice +2 Bob +PREHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: t + Select Operator + expressions: name (type: string), email (type: string), address (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +Alice alice@test.com 123 Apple St +Bob bob@test.com 456 Banana Ave +PREHOOK: query: CREATE TABLE test_exclude_join ( + id INT, + department STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: CREATE TABLE test_exclude_join ( + id INT, + department STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_exclude_join +PREHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude_join +POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] +POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] +PREHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude_join +POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] +POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] +PREHOOK: query: EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +PREHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +POSTHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), name (type: string), email (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), department (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col4 + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +PREHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +POSTHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +1 Alice alice@test.com Engineering +2 Bob bob@test.com Sales From cf31c1bb858b967989ced0599ea9a3739c8f7133 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Wed, 19 Aug 2026 21:01:21 +0530 Subject: [PATCH 2/3] Address review comments --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 4 +- .../hive/ql/parse/SemanticAnalyzer.java | 32 +++++---- .../queries/clientpositive/select_exclude.q | 10 +-- .../clientpositive/llap/select_exclude.q.out | 67 ++++++++----------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 8a79011d4614..20ce47258f5a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -4367,7 +4367,9 @@ private Pair internalGenSelectLogicalPlan(QB qb, RelNode s // 6.4 Build ExprNode corresponding to colums if (expr.getType() == HiveParser.TOK_ALLCOLREF) { // Parse SELECT * EXCLUDE columns and pass them to the Calcite engine for exclusion - String starTabAlias = SemanticAnalyzer.processAllColRefAndExclude(expr, inputRR, excludedColumns); + ExcludeResult excludeResult = processAllColRefAndExclude(expr, inputRR); + String starTabAlias = excludeResult.tableAlias(); + excludedColumns.addAll(excludeResult.excludedColumns()); pos = genRexNodeRegex(".*", starTabAlias, expr, columnList, excludedColumns, inputRR, starRR, pos, outputRR, qb.getAliases(), true); } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d24515bd951e..b5d21fa846b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -4737,20 +4737,25 @@ static boolean isRegex(String pattern, HiveConf conf) { return false; } - public static String processAllColRefAndExclude( - ASTNode expr, RowResolver inputRR, Set excludedColumns) throws SemanticException { + protected record ExcludeResult(String tableAlias, Set excludedColumns) {} + + protected ExcludeResult processAllColRefAndExclude(ASTNode expr, RowResolver inputRR) + throws SemanticException { // Check if the query uses SELECT * EXCLUDE. If it does, grab the table // alias (like t.*) and build a list of the columns the user wants to exclude. String starTabAlias = null; ASTNode excludeNode = null; - if (expr.getChildCount() > 0) { - ASTNode firstChild = (ASTNode) expr.getChild(0); - if (firstChild.getType() == HiveParser.TOK_TABCOLNAME) { - excludeNode = firstChild; - } else { - starTabAlias = getUnescapedName(firstChild).toLowerCase(); - if (expr.getChildCount() > 1) { - excludeNode = (ASTNode) expr.getChild(1); + Set excludedColumns = new HashSet<>(); + + if (expr.getChildren() != null) { + for (Node childNode : expr.getChildren()) { + ASTNode child = (ASTNode) childNode; + switch (child.getType()) { + case HiveParser.TOK_TABNAME -> starTabAlias = getUnescapedName(child).toLowerCase(); + case HiveParser.TOK_TABCOLNAME -> excludeNode = child; + default -> + throw new SemanticException( + "Unexpected node type in TOK_ALLCOLREF: " + child.getType()); } } } @@ -4764,7 +4769,7 @@ public static String processAllColRefAndExclude( } } } - return starTabAlias; + return new ExcludeResult(starTabAlias, excludedColumns); } private Operator genSelectPlan(String dest, QB qb, Operator input, @@ -4944,8 +4949,9 @@ private Operator genSelectPlan(String dest, ASTNode selExprList, QB qb, Opera if (expr.getType() == HiveParser.TOK_ALLCOLREF) { int initPos = pos; - Set excludeCols = new HashSet<>(); - String starTabAlias = processAllColRefAndExclude(expr, inputRR, excludeCols); + ExcludeResult excludeResult = processAllColRefAndExclude(expr, inputRR); + String starTabAlias = excludeResult.tableAlias(); + Set excludeCols = excludeResult.excludedColumns(); if (excludeCols.isEmpty()) { excludeCols = null; } diff --git a/ql/src/test/queries/clientpositive/select_exclude.q b/ql/src/test/queries/clientpositive/select_exclude.q index 1ed816fd786f..3159ea299fc7 100644 --- a/ql/src/test/queries/clientpositive/select_exclude.q +++ b/ql/src/test/queries/clientpositive/select_exclude.q @@ -1,3 +1,5 @@ +set hive.cli.print.header=true; + CREATE TABLE test_exclude ( id INT, name STRING, @@ -6,8 +8,9 @@ CREATE TABLE test_exclude ( phone STRING ); -INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'); -INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200'); +INSERT INTO test_exclude VALUES +(1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'), +(2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200'); -- Exclude a single column EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude; @@ -26,8 +29,7 @@ CREATE TABLE test_exclude_join ( id INT, department STRING ); -INSERT INTO test_exclude_join VALUES (1, 'Engineering'); -INSERT INTO test_exclude_join VALUES (2, 'Sales'); +INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2, 'Sales'); EXPLAIN SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) diff --git a/ql/src/test/results/clientpositive/llap/select_exclude.q.out b/ql/src/test/results/clientpositive/llap/select_exclude.q.out index 852e58541792..0bb6d2422279 100644 --- a/ql/src/test/results/clientpositive/llap/select_exclude.q.out +++ b/ql/src/test/results/clientpositive/llap/select_exclude.q.out @@ -18,24 +18,15 @@ POSTHOOK: query: CREATE TABLE test_exclude ( POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@test_exclude -PREHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') +PREHOOK: query: INSERT INTO test_exclude VALUES +(1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'), +(2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@test_exclude -POSTHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_exclude -POSTHOOK: Lineage: test_exclude.address SCRIPT [] -POSTHOOK: Lineage: test_exclude.email SCRIPT [] -POSTHOOK: Lineage: test_exclude.id SCRIPT [] -POSTHOOK: Lineage: test_exclude.name SCRIPT [] -POSTHOOK: Lineage: test_exclude.phone SCRIPT [] -PREHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_exclude -POSTHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') +POSTHOOK: query: INSERT INTO test_exclude VALUES +(1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'), +(2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@test_exclude @@ -44,6 +35,7 @@ POSTHOOK: Lineage: test_exclude.email SCRIPT [] POSTHOOK: Lineage: test_exclude.id SCRIPT [] POSTHOOK: Lineage: test_exclude.name SCRIPT [] POSTHOOK: Lineage: test_exclude.phone SCRIPT [] +col1 col2 col3 col4 col5 PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude PREHOOK: type: QUERY PREHOOK: Input: default@test_exclude @@ -52,6 +44,7 @@ POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +Explain STAGE DEPENDENCIES: Stage-0 is a root stage @@ -75,6 +68,7 @@ POSTHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +test_exclude.id test_exclude.name test_exclude.address test_exclude.phone 1 Alice 123 Apple St 555-0100 2 Bob 456 Banana Ave 555-0200 PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude @@ -85,6 +79,7 @@ POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_excl POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +Explain STAGE DEPENDENCIES: Stage-0 is a root stage @@ -108,6 +103,7 @@ POSTHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +test_exclude.id test_exclude.name 1 Alice 2 Bob PREHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t @@ -118,6 +114,7 @@ POSTHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +Explain STAGE DEPENDENCIES: Stage-0 is a root stage @@ -141,6 +138,7 @@ POSTHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude #### A masked pattern was here #### +t.name t.email t.address Alice alice@test.com 123 Apple St Bob bob@test.com 456 Banana Ave PREHOOK: query: CREATE TABLE test_exclude_join ( @@ -157,26 +155,17 @@ POSTHOOK: query: CREATE TABLE test_exclude_join ( POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@test_exclude_join -PREHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_exclude_join -POSTHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_exclude_join -POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] -POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] -PREHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +PREHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2, 'Sales') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@test_exclude_join -POSTHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +POSTHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2, 'Sales') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@test_exclude_join POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] +col1 col2 PREHOOK: query: EXPLAIN SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id @@ -191,6 +180,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude POSTHOOK: Input: default@test_exclude_join #### A masked pattern was here #### +Explain STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -208,20 +198,20 @@ STAGE PLANS: TableScan alias: a filterExpr: id is not null (type: boolean) - Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 378 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: id is not null (type: boolean) - Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 378 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), name (type: string), email (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 378 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 378 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -230,20 +220,20 @@ STAGE PLANS: TableScan alias: b filterExpr: id is not null (type: boolean) - Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: id is not null (type: boolean) - Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), department (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -257,14 +247,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 562 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 562 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 562 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -288,5 +278,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_exclude POSTHOOK: Input: default@test_exclude_join #### A masked pattern was here #### +a.id a.name a.email b.department 1 Alice alice@test.com Engineering 2 Bob bob@test.com Sales From 937f2c4829c582bf9d9f7a8ffeab07c0084a4bcb Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Tue, 25 Aug 2026 14:04:48 +0530 Subject: [PATCH 3/3] Address view comments 2 --- .../hive/ql/parse/SemanticAnalyzer.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b5d21fa846b4..a239ce483c16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -4737,38 +4737,50 @@ static boolean isRegex(String pattern, HiveConf conf) { return false; } + /** + * Helper method to parse the excluded columns from an EXCLUDE AST node. Returns an unmodifiable + * set to ensure the caller cannot accidentally mutate the result. + */ + private Set processExcludeColumns( + ASTNode excludeNode, String starTabAlias, RowResolver inputRR) throws SemanticException { + Set localExcluded = new HashSet<>(); + for (int e = 0; e < excludeNode.getChildCount(); e++) { + String excludeColName = unescapeIdentifier(excludeNode.getChild(e).getText()).toLowerCase(); + ColumnInfo colInfo = inputRR.get(starTabAlias, excludeColName); + if (colInfo != null) { + localExcluded.add(colInfo); + } + } + return Collections.unmodifiableSet(localExcluded); + } + protected record ExcludeResult(String tableAlias, Set excludedColumns) {} + /** + * Parses a TOK_ALLCOLREF node (e.g. `*` or `t.* EXCLUDE (a)`) to extract the table alias and the + * set of columns to be excluded. + */ protected ExcludeResult processAllColRefAndExclude(ASTNode expr, RowResolver inputRR) throws SemanticException { - // Check if the query uses SELECT * EXCLUDE. If it does, grab the table - // alias (like t.*) and build a list of the columns the user wants to exclude. + String starTabAlias = null; - ASTNode excludeNode = null; - Set excludedColumns = new HashSet<>(); + + // Zero-allocation initialization for queries that don't use EXCLUDE. + Set excludedColumns = Set.of(); if (expr.getChildren() != null) { for (Node childNode : expr.getChildren()) { ASTNode child = (ASTNode) childNode; switch (child.getType()) { case HiveParser.TOK_TABNAME -> starTabAlias = getUnescapedName(child).toLowerCase(); - case HiveParser.TOK_TABCOLNAME -> excludeNode = child; + case HiveParser.TOK_TABCOLNAME -> + excludedColumns = processExcludeColumns(child, starTabAlias, inputRR); default -> throw new SemanticException( "Unexpected node type in TOK_ALLCOLREF: " + child.getType()); } } } - - if (excludeNode != null) { - for (int e = 0; e < excludeNode.getChildCount(); e++) { - String excludeColName = unescapeIdentifier(excludeNode.getChild(e).getText()).toLowerCase(); - ColumnInfo colInfo = inputRR.get(starTabAlias, excludeColName); - if (colInfo != null) { - excludedColumns.add(colInfo); - } - } - } return new ExcludeResult(starTabAlias, excludedColumns); }