From 78f3529d8d113716047644d9848a6500354a6217 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Fri, 4 Sep 2026 11:49:22 +1000 Subject: [PATCH 01/39] MDEV-40168 [wip] Add multi valued index over fulltext TODOs on top of those in the patch diff: - EXPLAIN output should not say fulltext - check type match to avoid false negative / positive bugs in mysql - transcode the value into the index charset in mvi_encode_key --- libmysqld/CMakeLists.txt | 1 + mysql-test/main/multi_valued_index.opt | 1 + mysql-test/main/multi_valued_index.result | 94 ++++ mysql-test/main/multi_valued_index.test | 55 +++ .../suite/perfschema/r/digest_view.result | 50 +- .../start_server_low_digest_sql_length.result | 4 +- sql/CMakeLists.txt | 3 +- sql/item.h | 5 + sql/item_func.h | 3 +- sql/item_jsonfunc.h | 1 + sql/item_strfunc.h | 24 + sql/lex.h | 1 + sql/opt_multi_valued_index.cc | 461 ++++++++++++++++++ sql/opt_multi_valued_index.h | 20 + sql/sql_class.cc | 3 +- sql/sql_select.cc | 6 + sql/sql_select.h | 1 + sql/sql_table.cc | 4 +- sql/sql_table.h | 2 + sql/sql_yacc.yy | 43 +- 20 files changed, 748 insertions(+), 34 deletions(-) create mode 100644 mysql-test/main/multi_valued_index.opt create mode 100644 mysql-test/main/multi_valued_index.result create mode 100644 mysql-test/main/multi_valued_index.test create mode 100644 sql/opt_multi_valued_index.cc create mode 100644 sql/opt_multi_valued_index.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index d38f2a48c737b..6852fbb19a650 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -72,6 +72,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/mf_iocache.cc ../sql/my_decimal.cc ../sql/net_serv.cc ../sql/opt_range.cc ../sql/opt_group_by_cardinality.cc + ../sql/opt_multi_valued_index.cc ../sql/opt_rewrite_date_cmp.cc ../sql/opt_rewrite_remove_casefold.cc ../sql/opt_sargable_left.cc diff --git a/mysql-test/main/multi_valued_index.opt b/mysql-test/main/multi_valued_index.opt new file mode 100644 index 0000000000000..a076db4e5b886 --- /dev/null +++ b/mysql-test/main/multi_valued_index.opt @@ -0,0 +1 @@ +--innodb_ft_index_cache diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result new file mode 100644 index 0000000000000..369464b0f5a85 --- /dev/null +++ b/mysql-test/main/multi_valued_index.result @@ -0,0 +1,94 @@ +# basic test +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t1 1 invisible1 1 invisible1 A 0 NULL NULL YES BTREE NO +t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; +insert into t1 values (1, '{"tags": [1, 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +insert into t1 values (2, '{"tags": ["1", "abcde", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +31xx 2 2 1 2 0 +6162636465 2 2 1 2 5 +xxxx 2 2 1 2 16 +insert into t1 values (3, '{"tags": ["1.0", "34567", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +312e30 3 3 1 3 0 +31xx 2 2 1 2 0 +3334353637 3 3 1 3 7 +6162636465 2 2 1 2 5 +xxxx 2 3 2 2 16 +xxxx 2 3 2 3 18 +insert into t1 values (4, '{}'); +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext idx idx 0 1 Using where +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +select * from t1 where json_contains(j->'$.tags', '1.0'); +c j +1 {"tags": [1, 34567]} +select * from t1 where json_contains(j->'$.tags', '1'); +c j +1 {"tags": [1, 34567]} +select * from t1 where json_contains(j->'$.tags', '"1"'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +explain +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext idx idx 0 1 Using where +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +explain +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext idx idx 0 1 Using where +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +c j +select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +DROP TABLE t1; +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; +# top level or +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 1 Using where +DROP TABLE t1; +# direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +mvi_encode('[1, 42, "3"]', int) +8000000000000001 800000000000002a +select mvi_encode('[1, 42, "3"]', unsigned); +mvi_encode('[1, 42, "3"]', unsigned) +0000000000000001 000000000000002a +select mvi_encode('[1, 42, "3 "]', char(6)); +mvi_encode('[1, 42, "3 "]', char(6)) +33xx +select mvi_encode('[1, 42, " "]', char(6)); +mvi_encode('[1, 42, " "]', char(6)) +xxxx +select mvi_encode('[1, 42, "3 "]', binary(6)); +mvi_encode('[1, 42, "3 "]', binary(6)) +33xx +select mvi_encode('[1, 42]', char(6)); +mvi_encode('[1, 42]', char(6)) + diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test new file mode 100644 index 0000000000000..8c9d4d8ec3788 --- /dev/null +++ b/mysql-test/main/multi_valued_index.test @@ -0,0 +1,55 @@ +--source include/have_debug.inc +--source include/have_innodb.inc + +--echo # basic test +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +show index from t1; + +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; + +insert into t1 values (1, '{"tags": [1, 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (2, '{"tags": ["1", "abcde", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (3, '{"tags": ["1.0", "34567", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (4, '{}'); + +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +select * from t1 where json_contains(j->'$.tags', '1.0'); +select * from t1 where json_contains(j->'$.tags', '1'); +select * from t1 where json_contains(j->'$.tags', '"1"'); +explain +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +explain +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); + +DROP TABLE t1; + +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; + +--echo # top level or + +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; + +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); + +DROP TABLE t1; + +--echo # direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +select mvi_encode('[1, 42, "3"]', unsigned); +select mvi_encode('[1, 42, "3 "]', char(6)); +select mvi_encode('[1, 42, " "]', char(6)); +select mvi_encode('[1, 42, "3 "]', binary(6)); +select mvi_encode('[1, 42]', char(6)); diff --git a/mysql-test/suite/perfschema/r/digest_view.result b/mysql-test/suite/perfschema/r/digest_view.result index 346b5e0c44e60..b7954b50a12ba 100644 --- a/mysql-test/suite/perfschema/r/digest_view.result +++ b/mysql-test/suite/perfschema/r/digest_view.result @@ -191,17 +191,17 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 1 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 1 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 1 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 1 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 1 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 1 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 1 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 1 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP TABLE test.v1; CREATE VIEW test.v1 AS SELECT * FROM test.t1; EXPLAIN SELECT * from test.v1; @@ -248,19 +248,19 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test ef8b10f452e117fa96af49687a93a10f CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 -test 6b99f4d2ad410f9fa4ee1d501b0db571 DROP TABLE `test` . `v1` 1 -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 2 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 2 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test 187a846fafe04b746eddaaab80b6a766 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 2 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 2 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test bd4716657502ce7922d2ed7e20043b76 CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 +test 2d14a4845ee825e3a1a21bd5799cba3c DROP TABLE `test` . `v1` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 2 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 2 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test aa329bcb29afc39d9793e4fd6e108722 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 2 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 2 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP VIEW test.v1; DROP TABLE test.t1; diff --git a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result index e4e2c8b294a3f..e78e21e19a9ca 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result +++ b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result @@ -8,5 +8,5 @@ SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 #################################### SELECT event_name, digest, digest_text, sql_text FROM events_statements_history_long; event_name digest digest_text sql_text -statement/sql/select c9e142fe40c43498607ca5310e11d2ce SELECT ? + ? + SELECT ... -statement/sql/truncate 506e3496d92689cd2367329dc7725165 TRUNCATE TABLE truncat... +statement/sql/select 3e19ece73d286977122b52fb027c180c SELECT ? + ? + SELECT ... +statement/sql/truncate 7ab36a94986dd98b93370b025166a946 TRUNCATE TABLE truncat... diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 429a85f9cac30..bd01fba5a41dd 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -122,7 +122,6 @@ SET (SQL_SOURCE opt_rewrite_remove_casefold.cc opt_sargable_left.cc opt_sum.cc - opt_vcol_substitution.cc ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc session_tracker.cc @@ -199,6 +198,8 @@ SET (SQL_SOURCE json_table.cc proxy_protocol.cc backup.cc xa.cc socketpair.c socketpair.h + opt_multi_valued_index.h + opt_multi_valued_index.cc opt_vcol_substitution.h opt_vcol_substitution.cc opt_hints_parser.cc opt_hints_parser.h scan_char.h diff --git a/sql/item.h b/sql/item.h index dabe4ca1c8801..5d588a58d6911 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2859,6 +2859,11 @@ class Item :public Value_source, DBUG_ASSERT(fixed()); return false; } + virtual Item *create_ft_for_mvi(THD *thd, List *vcol_fields) + { + return NULL; + } + protected: /* diff --git a/sql/item_func.h b/sql/item_func.h index cc0985c8d53b7..59d02828ff8c5 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -109,7 +109,8 @@ class Item_func :public Item_func_or_sum JSON_EXTRACT_FUNC, JSON_VALID_FUNC, ROWNUM_FUNC, CASE_SEARCHED_FUNC, // Used by ColumnStore/Spider CASE_SIMPLE_FUNC, // Used by ColumnStore/spider, - DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC + DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC, + MVI_ENCODE_FUNC }; /* diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 3ec85d8c1a980..95e8938a1557d 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -376,6 +376,7 @@ class Item_func_json_contains: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; + Item *create_ft_for_mvi(THD *thd, List *vcol_fields) override; protected: Item *shallow_copy(THD *thd) const override diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index a35749fc4dc1b..6c3f67ce9c670 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2643,6 +2643,30 @@ class Item_temptable_rowid :public Item_str_func { return get_item_copy(thd, this); } }; +class Item_func_mvi_encode : public Item_str_ascii_func +{ + Lex_cast_type_st m_cast_type; + String tmp_js; + json_engine_t je; +public: + void print(String *str, enum_query_type query_type) override; + Item_func_mvi_encode(THD* thd, Item *expr, const Lex_cast_type_st &cast_type): + Item_str_ascii_func(thd, expr), m_cast_type(cast_type) {} + String *val_str_ascii(String *buf) override; + enum Functype functype() const override { return MVI_ENCODE_FUNC; } + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("mvi_encode")}; + return name; + } + bool fix_length_and_dec(THD *thd) override; + Item *shallow_copy(THD *thd) const override + { + return get_item_copy(thd, this); + } + Lex_cast_type_st &cast_type() { return m_cast_type; } +}; + class Item_func_format_pico_time : public Item_str_ascii_func { diff --git a/sql/lex.h b/sql/lex.h index 61774df019f0d..d14a4dd7c636b 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -419,6 +419,7 @@ SYMBOL symbols[] = { { "MONITOR", SYM(MONITOR_SYM)}, { "MONTH", SYM(MONTH_SYM)}, { "MUTEX", SYM(MUTEX_SYM)}, + { "MVI_ENCODE", SYM(MVI_ENCODE_SYM)}, { "MYSQL", SYM(MYSQL_SYM)}, { "MYSQL_ERRNO", SYM(MYSQL_ERRNO_SYM)}, { "NAME", SYM(NAME_SYM)}, diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc new file mode 100644 index 0000000000000..1c2912e5578f7 --- /dev/null +++ b/sql/opt_multi_valued_index.cc @@ -0,0 +1,461 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "mariadb.h" +#include "sql_select.h" +#include "item_func.h" + +void Item_func_mvi_encode::print(String *str, enum_query_type query_type) +{ + char buf[32]; + size_t length; + str->append(func_name_cstring()); + str->append('('); + args[0]->print(str, query_type); + str->append(','); + const Name name= m_cast_type.type_handler()->name(); + switch (m_cast_type.type_handler()->field_type()) + { + case MYSQL_TYPE_LONG_BLOB: + str->append(STRING_WITH_LEN("char")); + str->append('('); + length= (size_t) (longlong10_to_str(m_cast_type.length(), buf, -10) - buf); + str->append(buf, length); + str->append(')'); + break; + default: + str->append(name.ptr(), name.length()); + break; + } + /* TODO: this is copied from another print() implementation */ + if (decimals && decimals != NOT_FIXED_DEC) + { + str->append('('); + length= (size_t) (longlong10_to_str(decimals, buf, -10) - buf); + str->append(buf, length); + str->append(')'); + } + str->append(')'); +} + +/* TODO: this duplicates logic in Item_func_json_extract::val_int */ +static longlong json_value_to_longlong(enum json_value_types type, + CHARSET_INFO *cs, + char* value, int value_len) +{ + switch (type) + { + case JSON_VALUE_NUMBER: + case JSON_VALUE_STRING: + { + char *end; + int err; + return cs->strntoll(value, value_len, 10, &end, &err); + } + case JSON_VALUE_TRUE: + return 1; + default: + return 0; + }; +} + +/* Lifted from Type_handler method of the same name */ +static void store_sort_key_longlong(uchar *to, bool unsigned_flag, + longlong value) +{ + to[7]= (uchar) value; + to[6]= (uchar) (value >> 8); + to[5]= (uchar) (value >> 16); + to[4]= (uchar) (value >> 24); + to[3]= (uchar) (value >> 32); + to[2]= (uchar) (value >> 40); + to[1]= (uchar) (value >> 48); + to[0]= (uchar) (value >> 56) ^ (unsigned_flag ? 0 : 128); +} + +static bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf) +{ + enum_field_types cast_ftype= cast_th->field_type(); + bool is_unsigned= cast_th->is_unsigned(); + StringBuffer<42> sorted; + /* Skip encoding on type incompatibility */ + if (mvi_json_class(cast_ftype) != je->value_type) + return true; + /* 1. sort_string */ + sorted.length(0); + /* TODO: handle temporal types and decimal */ + switch(cast_ftype) + { + case MYSQL_TYPE_LONGLONG: + store_sort_key_longlong( + (uchar *) sorted.c_ptr(), is_unsigned, + json_value_to_longlong(je->value_type, cs, + (char *) je->value, je->value_len)); + sorted.length(8); + break; + /* TODO: unquote? */ + /* CHAR(n) => LONG BLOB */ + case MYSQL_TYPE_LONG_BLOB: + { + /* Trim trailing whitespaces if possible */ + if (!(cs->state & MY_CS_NOPAD)) + je->value_len= (int) cs->lengthsp((const char *) je->value, + je->value_len); + if (my_binary_compare(cs)) + sorted.set((char *) je->value, je->value_len, + &my_charset_latin1_bin); + else + { + my_strnxfrm_ret_t rc= cs->strnxfrm( + (uchar *) sorted.c_ptr(), 42, 42, je->value, je->value_len, 0); + sorted.length(rc.m_result_length); + } + break; + } + default: + return true; + } + + /* 2. hex */ + buf->append_hex(sorted.c_ptr(), sorted.length()); + + /* 3. pad */ + if (sorted.length() == 0) + buf->append(STRING_WITH_LEN("xxxx")); + else if (sorted.length() == 1) + buf->append(STRING_WITH_LEN("xx")); + + /* 4. space */ + buf->append(' '); + return false; +} + +String *Item_func_mvi_encode::val_str_ascii(String *buf) +{ + String *value= args[0]->val_json(&tmp_js); + if ((null_value= !value)) + return nullptr; + CHARSET_INFO *cs= value->charset(); + const Type_handler *cast_th= m_cast_type.type_handler(); + bool end_ok= false, at_least_one= false; + const uchar *start= reinterpret_cast(value->ptr()); + const uchar *end= start + value->length(); + DBUG_ASSERT(fixed()); + buf->length(0); + buf->set_charset(&my_charset_latin1_bin); + + if (json_scan_start(&je, cs, start, end) || + json_read_value(&je)) + goto json_error; + + if (je.value_type != JSON_VALUE_ARRAY) + goto error_format; + + /* TODO: deduplicate, so that ["34567", 34567] yield only one token */ + do { + switch (je.state) + { + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + /* + TODO: do something different when an empty string is + returned, i.e. at_least_one == false to avoid wasting index + space? + */ + if (at_least_one) + buf->length(buf->length() - 1); + end_ok = true; + break; + case JST_VALUE: + { + if (json_read_value(&je)) + goto json_error; + + at_least_one = !encode_mvi_key(&je, cast_th, cs, buf) || at_least_one; + break; + } + default: + goto error_format; + } + } while (json_scan_next(&je) == 0); + + if (end_ok) + return buf; + +error_format: + { + int position= (int) ((const char *) je.s.c_str - value->ptr()); + /* TODO: fix error */ + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_VECTOR_FORMAT_INVALID, ER(ER_VECTOR_FORMAT_INVALID), + position, value->c_ptr_safe()); + null_value= true; + return nullptr; + } + +json_error: + report_json_error_ex(value->ptr(), &je, func_name(), + 0, Sql_condition::WARN_LEVEL_WARN); + null_value= true; + return nullptr; +} + +bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) +{ + /* TODO: validate args[0] is a json array */ + mem_root_dynamic_array_init(thd->mem_root, PSI_INSTRUMENT_MEM, + &je.stack, sizeof(int), NULL, + JSON_DEPTH_DEFAULT, JSON_DEPTH_INC, MYF(0)); + decimals= 0; + fix_length_and_charset(args[0]->max_char_length() * 2, + &my_charset_latin1_bin); + set_maybe_null(); + return false; +} + +static +bool collect_mvi_vcols_for_join(JOIN *join, List *vcol_fields) +{ + List_iterator ti(join->select_lex->leaf_tables); + TABLE_LIST *tl; + TABLE *table; + while ((tl= ti++)) + { + if (!(table= tl->table)) // non-merged semi-join or something like that + continue; + // TODO: Make use of iterator to loop through + // keys_in_use_for_query, instead. + for (uint i=0; i < table->s->keys; i++) + { + // note: we could also support histograms here + if (!table->keys_in_use_for_query.is_set(i)) + continue; + + KEY *key= &table->key_info[i]; + for (uint kp=0; kp < key->user_defined_key_parts; kp++) + { + Field *field= key->key_part[kp].field; + if (field->invisible == INVISIBLE_FULL && + field->vcol_info && + field->vcol_info->expr->type() == Item::FUNC_ITEM && + ((Item_func *) field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC && + vcol_fields->push_back(field)) + return TRUE; // Out of memory + } + } + } + return FALSE; // Ok +} + +class Mvi_context +{ + public: + THD *thd; + /* Virtual columns with fulltext index that we can try substituting */ + List vcol_fields; + + Mvi_context(THD *thd_arg) : thd(thd_arg) {} +}; + +Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, + List *vcol_fields) +{ + List_iterator it(*vcol_fields); + Field *vcol_field; + CHARSET_INFO *cs; + Item_func_mvi_encode *mvitem; + DBUG_ASSERT(fixed()); + if (arg_count > 2 || !a2_constant) + return NULL; + while ((vcol_field= it++)) + { + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(args[0], true)) + { + cs= mvitem->arguments()[0]->collation.collation; + break; + } + } + if (!vcol_field) + return NULL; + + const Type_handler *cast_th= mvitem->cast_type().type_handler(); + StringBuffer<42> sorted; + StringBuffer<256> buf; + const uchar *start, *end; + List ifm_args; + Item_field *ivcol; + Item_string *ift_query; + bool at_least_one= false; + DBUG_ASSERT(fixed()); + buf.length(0); + buf.set_charset(&my_charset_latin1_bin); + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + start= reinterpret_cast(val->ptr()); + end= start + val->length(); + + if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) + return NULL; + + if (je.value_type == JSON_VALUE_UNINITIALIZED || + je.value_type == JSON_VALUE_OBJECT) + return NULL; + if (je.value_type != JSON_VALUE_ARRAY) + { + /* scalar */ + if ((at_least_one = !encode_mvi_key(&je, cast_th, cs, &buf))) + buf.length(buf.length() - 1); + goto ok; + } + + /* TODO: deduplicate? */ + do { + switch (je.state) + { + /* TODO: nested array? */ + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + if (at_least_one) + buf.length(buf.length() - 1); + break; + case JST_VALUE: + { + if (json_read_value(&je)) + return NULL; + + buf.append('+'); + if (encode_mvi_key(&je, cast_th, cs, &buf)) + buf.length(buf.length() - 1); + else + at_least_one = true; + break; + } + default: + return NULL; + } + } while (json_scan_next(&je) == 0); + +ok: + if (!at_least_one) + return NULL; + ift_query= new (thd->mem_root) Item_string(thd, &my_charset_latin1_bin, + buf.c_ptr(), buf.length()); + ifm_args.push_back(ift_query); + ivcol= new (thd->mem_root) Item_field(thd, vcol_field); + ifm_args.push_back(ivcol); + return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); +} + +static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, + List *ftfunc_list) +{ + Item *conds= *conds_ref; + Item *cond, *match; + List matches; + THD *thd= ctx->thd; + if (conds->type() != Item::COND_ITEM) + { + if ((match= conds->create_ft_for_mvi(thd, &ctx->vcol_fields))) + { + matches.push_back(match); + ftfunc_list->push_back((Item_func_match *) match); + } + } + else if (((Item_cond *) conds)->functype() == Item_func::COND_OR_FUNC) + return false; + else + { + List_iterator it(*((Item_cond *) conds)->argument_list()); + while ((cond= it++)) + { + if ((match= cond->create_ft_for_mvi(thd, &ctx->vcol_fields))) + { + matches.push_back(match); + ftfunc_list->push_back((Item_func_match *) match); + } + } + } + if (matches.elements == 1) + cond= matches.pop(); + else + cond= new (thd->mem_root) Item_cond_and(thd, matches); + if (cond && + ((cond->fix_fields(thd, &cond) || + !(conds= and_items(thd, conds, cond)) || + conds->fix_fields(thd, &conds)))) + return true; + *conds_ref= conds; + return false; +} + +bool setup_mvi_for_join(JOIN *join) +{ + Mvi_context ctx(join->thd); + if (collect_mvi_vcols_for_join(join, &ctx.vcol_fields)) + return true; + if (!ctx.vcol_fields.is_empty() && join->conds) + return add_ft_for_mvi(&ctx, &join->conds, join->select_lex->ftfunc_list); + return false; +} + +enum json_value_types mvi_json_class(enum_field_types ftype) +{ + switch (ftype) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_DOUBLE: + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_NEWDECIMAL: + return JSON_VALUE_NUMBER; + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_YEAR: + case MYSQL_TYPE_NEWDATE: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_TIMESTAMP2: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIME2: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + return JSON_VALUE_STRING; + default: + return JSON_VALUE_UNINITIALIZED; + } +} diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h new file mode 100644 index 0000000000000..5a49828af0e91 --- /dev/null +++ b/sql/opt_multi_valued_index.h @@ -0,0 +1,20 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +bool setup_mvi_for_join(JOIN *join); + +/* Return the compatible json type */ +enum json_value_types mvi_json_class(enum_field_types ftype); diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 47de84e48d131..7d488ff341ebf 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -170,7 +170,8 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root) columns(rhs.columns, mem_root), name(rhs.name), option_list(rhs.option_list), - generated(rhs.generated), invisible(false), + generated(rhs.generated), + invisible(rhs.invisible), without_overlaps(rhs.without_overlaps), old(rhs.old), length(rhs.length), period(rhs.period) { diff --git a/sql/sql_select.cc b/sql/sql_select.cc index dc9e7b6113333..8f7c0987c7c7a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -2309,6 +2309,12 @@ JOIN::optimize_inner() optimize_schema_tables_memory_usage(select_lex->leaf_tables)) DBUG_RETURN(1); + if (setup_mvi_for_join(this)) + { + error= 1; + DBUG_RETURN(1); + } + if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */ DBUG_RETURN(-1); diff --git a/sql/sql_select.h b/sql/sql_select.h index 23a927bfcb14e..d1e97994915ad 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -3035,5 +3035,6 @@ void propagate_new_equalities(THD *thd, Item *cond, bool dbug_user_var_equals_str(THD *thd, const char *name, const char *value); #include "opt_vcol_substitution.h" +#include "opt_multi_valued_index.h" #endif /* SQL_SELECT_INCLUDED */ diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 54a604e404d47..1725697c65243 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -116,8 +116,6 @@ static Lex_ident_column make_unique_key_name(THD *, static bool make_unique_constraint_name(THD *, LEX_CSTRING *, const char *, List *, List *, uint *); -static Lex_ident_column make_internal_field_name(THD *, const char *, - List *); static int copy_data_between_tables(THD *, TABLE *,TABLE *, bool, uint, ORDER *, ha_rows *, ha_rows *, @@ -2775,7 +2773,7 @@ static int mysql_add_invisible_field(THD *thd, List * field_list, #define INTERNAL_FIELD_NAME_LENGTH 30 -static Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, List *create_list) { char buf[INTERNAL_FIELD_NAME_LENGTH]= {0}; diff --git a/sql/sql_table.h b/sql/sql_table.h index 3a27204f2c45b..171a7aa075ec9 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -226,4 +226,6 @@ extern MYSQL_PLUGIN_IMPORT const Lex_ident_column primary_key_name; bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, + List *create_list); #endif /* SQL_TABLE_INCLUDED */ diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c2fe9974c42df..78e7905e9d897 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1011,6 +1011,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token MONITOR_SYM /* MariaDB privilege */ %token MONTH_SYM /* SQL-2003-R */ %token MUTEX_SYM +%token MVI_ENCODE_SYM %token MYSQL_SYM %token MYSQL_ERRNO_SYM %token NAMES_SYM /* SQL-2003-N */ @@ -1761,7 +1762,7 @@ rule: using_list opt_use_partition use_partition %type - key_part key_part_simple + key_part key_part_simple multi_valued_key_part %type join_table_list join_table @@ -7636,6 +7637,7 @@ opt_without_overlaps: key_part: key_part_simple + | multi_valued_key_part | ident '(' NUM ')' { int key_part_len= atoi($3.str); @@ -7647,6 +7649,38 @@ key_part: } ; +multi_valued_key_part: + '(' CAST_SYM '(' expr AS cast_type ARRAY_SYM ')' ')' + { + /* TODO: check fts_min_token_size is 4, warn if not */ + /* Create a Create_field */ + Create_field *f= new (thd->mem_root) Create_field(); + LEX_CSTRING fname= make_internal_field_name(thd, "DB_MVI_", &Lex->alter_info.create_list); + Item *vcol_expr= + new (thd->mem_root) Item_func_mvi_encode(thd, $4, $6); + + if (unlikely(!f)) + MYSQL_YYABORT; + + f->invisible= INVISIBLE_FULL; + Lex->last_key->invisible= true; + f->set_handler(&type_handler_blob); + f->charset= &my_charset_latin1_bin; + Lex->last_key->type= Key::FULLTEXT; + Lex->init_last_field(f, &fname); + Lex->alter_info.create_list.push_back(f, thd->mem_root); + + /* Create a vcol */ + Virtual_column_info *v= add_virtual_expression(thd, vcol_expr); + if (unlikely(!v)) + MYSQL_YYABORT; + Lex->last_field->vcol_info= v; + Lex->last_field->vcol_info->set_vcol_type(VCOL_GENERATED_STORED); + + $$= new (thd->mem_root) Key_part_spec(&fname, 0, /*gen=*/true); + } + ; + key_part_simple: ident { @@ -11146,6 +11180,12 @@ function_call_nonkeyword: MYSQL_YYABORT; Lex->safe_to_cache_query= false; } + | MVI_ENCODE_SYM '(' expr ',' cast_type ')' + { + $$= new (thd->mem_root) Item_func_mvi_encode(thd, $3, $5); + if (unlikely($$ == NULL)) + MYSQL_YYABORT; + } | NOW_SYM opt_time_precision { $$= new (thd->mem_root) Item_func_current_timestamp(thd, $2); @@ -16907,6 +16947,7 @@ keyword_sp_var_not_label: | HELP_SYM | HOST_SYM | INSTALL_SYM + | MVI_ENCODE_SYM | OPTION | OPTIONS_SYM | OTHERS_MARIADB_SYM From 2b86847f82c3ed89b7b4e2006e036172b16e306e Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Fri, 4 Sep 2026 17:06:05 +1000 Subject: [PATCH 02/39] MDEV-40168 [wip] mvi quick --- sql/item.h | 5 +- sql/item_jsonfunc.h | 3 +- sql/opt_multi_valued_index.cc | 209 +++++++++++++++++++++++++++++----- sql/opt_multi_valued_index.h | 24 ++++ sql/sql_select.cc | 6 + 5 files changed, 218 insertions(+), 29 deletions(-) diff --git a/sql/item.h b/sql/item.h index 5d588a58d6911..359b0b163bd41 100644 --- a/sql/item.h +++ b/sql/item.h @@ -822,6 +822,8 @@ const item_walk_flags WALK_NO_CACHE_PROCESS= (1<<1); const item_walk_flags WALK_NO_REF= (1<<2); +struct Mv_index; + class Item :public Value_source, public Type_all_attributes { @@ -2307,6 +2309,7 @@ class Item :public Value_source, invoked with this processor */ virtual bool get_context_for_vcol_processor(void *arg) { return 0; } + virtual bool mvi_analyze(void *arg) { return 0; } virtual bool enumerate_field_refs_processor(void *arg) { return 0; } virtual bool mark_as_eliminated_processor(void *arg) { return 0; } virtual bool eliminate_subselect_processor(void *arg) { return 0; } @@ -2859,7 +2862,7 @@ class Item :public Value_source, DBUG_ASSERT(fixed()); return false; } - virtual Item *create_ft_for_mvi(THD *thd, List *vcol_fields) + virtual Item *create_ft_for_mvi(THD *thd, List *indexes) { return NULL; } diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 95e8938a1557d..04e5e490f6a97 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -376,7 +376,8 @@ class Item_func_json_contains: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; - Item *create_ft_for_mvi(THD *thd, List *vcol_fields) override; + bool mvi_analyze(void *arg) override; + Item *create_ft_for_mvi(THD *thd, List *indexes) override; protected: Item *shallow_copy(THD *thd) const override diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 1c2912e5578f7..a94817e16ae05 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -139,8 +139,6 @@ static bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, else if (sorted.length() == 1) buf->append(STRING_WITH_LEN("xx")); - /* 4. space */ - buf->append(' '); return false; } @@ -186,7 +184,11 @@ String *Item_func_mvi_encode::val_str_ascii(String *buf) if (json_read_value(&je)) goto json_error; - at_least_one = !encode_mvi_key(&je, cast_th, cs, buf) || at_least_one; + if (!encode_mvi_key(&je, cast_th, cs, buf)) + { + buf->append(' '); + at_least_one= true; + } break; } default: @@ -228,35 +230,39 @@ bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) return false; } +/* Collect all the MVI indexes in `join' */ static -bool collect_mvi_vcols_for_join(JOIN *join, List *vcol_fields) +bool collect_mvi_vcols_for_join(JOIN *join, List *indexes) { List_iterator ti(join->select_lex->leaf_tables); TABLE_LIST *tl; TABLE *table; + THD *thd= join->thd; while ((tl= ti++)) { if (!(table= tl->table)) // non-merged semi-join or something like that continue; - // TODO: Make use of iterator to loop through - // keys_in_use_for_query, instead. for (uint i=0; i < table->s->keys; i++) { - // note: we could also support histograms here if (!table->keys_in_use_for_query.is_set(i)) continue; KEY *key= &table->key_info[i]; for (uint kp=0; kp < key->user_defined_key_parts; kp++) { + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy)) continue; Field *field= key->key_part[kp].field; if (field->invisible == INVISIBLE_FULL && field->vcol_info && field->vcol_info->expr->type() == Item::FUNC_ITEM && ((Item_func *) field->vcol_info->expr)->functype() == - Item_func::MVI_ENCODE_FUNC && - vcol_fields->push_back(field)) - return TRUE; // Out of memory + Item_func::MVI_ENCODE_FUNC) + { + Mv_index *index= new (thd->mem_root) Mv_index(field, i); + if (indexes->push_back(index)) + return TRUE; // Out of memory + } } } } @@ -267,24 +273,136 @@ class Mvi_context { public: THD *thd; - /* Virtual columns with fulltext index that we can try substituting */ - List vcol_fields; + /* All MV indexes in the JOIN */ + List indexes; + /* MVI accesses for all eligible predicates in WHERE */ + List accesses; Mvi_context(THD *thd_arg) : thd(thd_arg) {} }; -Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, - List *vcol_fields) +bool Item_func_json_contains::mvi_analyze(void *arg) { - List_iterator it(*vcol_fields); + Mvi_context *ctx= (Mvi_context *) arg; + List_iterator it(ctx->indexes); Field *vcol_field; - CHARSET_INFO *cs; - Item_func_mvi_encode *mvitem; + Mv_index *index; + CHARSET_INFO *cs= NULL; + Item_func_mvi_encode *mvitem= NULL; + DBUG_ASSERT(fixed()); + if (arg_count > 2 || !a2_constant) + return false; + /* Find the MVI that matches the first argument */ + while ((index= it++)) + { + vcol_field= index->vcol; + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(args[0], true)) + { + cs= mvitem->arguments()[0]->collation.collation; + break; + } + } + if (!index) + return false; + + /* Get ready to construct the ft queries from the second argument */ + const Type_handler *cast_th= mvitem->cast_type().type_handler(); + const uchar *start, *end; + Mvi_access *access= NULL; + StringBuffer<256> buf; + buf.length(0); + buf.set_charset(&my_charset_latin1_bin); + DBUG_ASSERT(fixed()); + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return false; + start= reinterpret_cast(val->ptr()); + end= start + val->length(); + + if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) + return false; + + if (je.value_type == JSON_VALUE_UNINITIALIZED || + je.value_type == JSON_VALUE_OBJECT) + return false; + if (je.value_type != JSON_VALUE_ARRAY) + { + /* scalar */ + if (!encode_mvi_key(&je, cast_th, cs, &buf)) + { + access= new (ctx->thd->mem_root) Mvi_access(index, true); + /* TODO: there gotta be a less verbose way to construct s. */ + String *s= new (ctx->thd->mem_root) String; + s->set_charset(&my_charset_latin1_bin); + if (s->copy(buf.ptr(), buf.length(), &my_charset_latin1_bin)) + return true; + access->encoded.push_back(s); + } + goto ok; + } + + /* TODO: deduplicate? */ + do { + buf.length(0); + switch (je.state) + { + /* TODO: nested array? */ + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + break; + case JST_VALUE: + { + if (json_read_value(&je)) + return false; + + if (!encode_mvi_key(&je, cast_th, cs, &buf)) + { + if (!access) + access= new (ctx->thd->mem_root) Mvi_access(index, true); + /* TODO: there gotta be a less verbose way to construct s. */ + String *s= new (ctx->thd->mem_root) String; + s->set_charset(&my_charset_latin1_bin); + if (s->copy(buf.ptr(), buf.length(), &my_charset_latin1_bin)) + return true; + access->encoded.push_back(s); + } + break; + } + default: + return false; + } + } while (json_scan_next(&je) == 0); + +ok: + if (access) + ctx->accesses.push_back(access); + return false; +} + + +Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, + List *indexes) +{ + List_iterator it(*indexes); + Mv_index *index; + Field *vcol_field= NULL; + CHARSET_INFO *cs= NULL; + Item_func_mvi_encode *mvitem= NULL; DBUG_ASSERT(fixed()); if (arg_count > 2 || !a2_constant) return NULL; - while ((vcol_field= it++)) + while ((index= it++)) { + vcol_field= index->vcol; DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == MVI_ENCODE_FUNC); @@ -295,11 +413,10 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, break; } } - if (!vcol_field) + if (!index) return NULL; const Type_handler *cast_th= mvitem->cast_type().type_handler(); - StringBuffer<42> sorted; StringBuffer<256> buf; const uchar *start, *end; List ifm_args; @@ -328,8 +445,8 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, if (je.value_type != JSON_VALUE_ARRAY) { /* scalar */ - if ((at_least_one = !encode_mvi_key(&je, cast_th, cs, &buf))) - buf.length(buf.length() - 1); + if (!encode_mvi_key(&je, cast_th, cs, &buf)) + at_least_one= true; goto ok; } @@ -353,7 +470,10 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, if (encode_mvi_key(&je, cast_th, cs, &buf)) buf.length(buf.length() - 1); else - at_least_one = true; + { + buf.append(' '); + at_least_one= true; + } break; } default: @@ -381,7 +501,7 @@ static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, THD *thd= ctx->thd; if (conds->type() != Item::COND_ITEM) { - if ((match= conds->create_ft_for_mvi(thd, &ctx->vcol_fields))) + if ((match= conds->create_ft_for_mvi(thd, &ctx->indexes))) { matches.push_back(match); ftfunc_list->push_back((Item_func_match *) match); @@ -394,7 +514,7 @@ static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, List_iterator it(*((Item_cond *) conds)->argument_list()); while ((cond= it++)) { - if ((match= cond->create_ft_for_mvi(thd, &ctx->vcol_fields))) + if ((match= cond->create_ft_for_mvi(thd, &ctx->indexes))) { matches.push_back(match); ftfunc_list->push_back((Item_func_match *) match); @@ -414,12 +534,47 @@ static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, return false; } +static void choose_mvi_access_for_tables(List *accesses, Mvi_access **best) +{ + List_iterator it(*accesses); + /* TODO: cost based */ + /* + TODO: merge + + json_contains(j->'$.tags','"a"') and + json_contains(j->'$.tags','"b"') + + (+ta +tb) + */ + while (Mvi_access *access= it++) + best[access->index->vcol->table->tablenr] = access; +} + +/* Build the scan and install it to join */ +bool setup_mvi_quick(JOIN *join) +{ + Mvi_context ctx(join->thd); + Mvi_access *best[MAX_TABLES]; + bzero(best, sizeof(best)); + if (!join->conds) + return false; + if (collect_mvi_vcols_for_join(join, &ctx.indexes)) + return true; + if (!ctx.indexes.is_empty() && + join->conds->walk(&Item::mvi_analyze, &ctx, WALK_SUBQUERY)) + return true; + choose_mvi_access_for_tables(&ctx.accesses, best); + return false; +} + bool setup_mvi_for_join(JOIN *join) { Mvi_context ctx(join->thd); - if (collect_mvi_vcols_for_join(join, &ctx.vcol_fields)) + if (!join->conds) + return false; + if (collect_mvi_vcols_for_join(join, &ctx.indexes)) return true; - if (!ctx.vcol_fields.is_empty() && join->conds) + if (!ctx.indexes.is_empty()) return add_ft_for_mvi(&ctx, &join->conds, join->select_lex->ftfunc_list); return false; } diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 5a49828af0e91..cc63edf47b75d 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -14,7 +14,31 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ +/* An MVI index */ +struct Mv_index : public Sql_alloc +{ + Field *vcol; /* The hidden vcol of the index */ + uint keyno; /* The keyno of the index */ + Mv_index(Field *vcol_arg, uint keyno_arg) + : vcol(vcol_arg), keyno(keyno_arg) {} +}; + +/* Access descriptor for a predicate */ +struct Mvi_access : public Sql_alloc +{ + Mv_index *index; + List encoded; /* encoded element keys */ + bool conjunctive; /* CONTAINS -> AND, OVERLAPS -> OR */ + Mvi_access(Mv_index *idx, bool conj) : index(idx), conjunctive(conj) {} +}; +/* + bool Item_func_json_contains::mvi_analyze(THD *thd, List *vcol_fields, + Mvi_access *out); + */ + bool setup_mvi_for_join(JOIN *join); /* Return the compatible json type */ enum json_value_types mvi_json_class(enum_field_types ftype); + +bool setup_mvi_quick(JOIN *join); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 8f7c0987c7c7a..002d8c77d77de 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -2309,6 +2309,12 @@ JOIN::optimize_inner() optimize_schema_tables_memory_usage(select_lex->leaf_tables)) DBUG_RETURN(1); + if (setup_mvi_quick(this)) + { + error= 1; + DBUG_RETURN(1); + } + if (setup_mvi_for_join(this)) { error= 1; From 99b2b449f4612b5a885ac14bcefc1fe37d28f98f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 4 Sep 2026 14:06:51 +0300 Subject: [PATCH 03/39] Add comments --- sql/item_strfunc.h | 24 ++++++++++++++++++++++++ sql/opt_multi_valued_index.cc | 26 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 6c3f67ce9c670..3cb20823a36b3 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2643,6 +2643,30 @@ class Item_temptable_rowid :public Item_str_func { return get_item_copy(thd, this); } }; + +/* + A function to support ARRAY indexes. When the user specifies an ARRAY index: + + CREATE INDEX idx1 ON + t1 ((CAST(JSON_EXTRACT(json_col, '$.arr') AS $datatype ARRAY))); + + We create a virtual column and a fulltext index over it: + + mvi_col_1 BLOB AS (MVI_ENCODE(JSON_EXTRACT(json_col, '$.arr'), $datatype)), + FULLTEXT INDEX idx (mvi_col_1) + + So, MVI_ENCODE has this signature: + + MVI_ENCODE(json_array, datatype) + + and it returns the JSON array elements represented in a form suitable for + putting into the fulltext index (without any custom fulltext parser atm) + + @seealso "multi_valued_key_part:" rule in sql_yacc.yy + + (TODO: move this item to opt_multi_valued_index, too) +*/ + class Item_func_mvi_encode : public Item_str_ascii_func { Lex_cast_type_st m_cast_type; diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index a94817e16ae05..834a6d4eeb8dd 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -389,6 +389,21 @@ bool Item_func_json_contains::mvi_analyze(void *arg) } + +/* + @brief + Create a fulltext search item that matches this JSON_CONTAINS(...) predicate. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + If yes, create and return an Item for searching for matches in the index: + + MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) +*/ + Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, List *indexes) { @@ -492,6 +507,17 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); } +/* + @brief + Examine the WHERE clause in (*conds_ref) and add conditions for multi-value + index predicates. + + Since we add fulltext predicates, also add them into *ftfunc_list. + + @detail + Currently we only walk down into top-level's WHERE clause. +*/ + static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, List *ftfunc_list) { From f5896445cee6dadabb8e27e79552388621a2421d Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 4 Sep 2026 14:23:18 +0300 Subject: [PATCH 04/39] Factor out common code into get_mvi_index() --- sql/opt_multi_valued_index.cc | 64 ++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 834a6d4eeb8dd..1ba04d961cb47 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -281,11 +281,34 @@ class Mvi_context Mvi_context(THD *thd_arg) : thd(thd_arg) {} }; + +/* + Find Multi-Value Index created over array_indexed_expr. +*/ +static Mv_index *get_mvi_index(List *indexes, + Item *array_indexed_expr) +{ + Mv_index *index; + List_iterator it(*indexes); + Item_func_mvi_encode *mvitem; + while ((index= it++)) + { + Field *vcol_field= index->vcol; + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == Item::FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(array_indexed_expr, true)) + { + return index; + } + } + return NULL; +} + bool Item_func_json_contains::mvi_analyze(void *arg) { Mvi_context *ctx= (Mvi_context *) arg; - List_iterator it(ctx->indexes); - Field *vcol_field; Mv_index *index; CHARSET_INFO *cs= NULL; Item_func_mvi_encode *mvitem= NULL; @@ -293,22 +316,12 @@ bool Item_func_json_contains::mvi_analyze(void *arg) if (arg_count > 2 || !a2_constant) return false; /* Find the MVI that matches the first argument */ - while ((index= it++)) - { - vcol_field= index->vcol; - DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); - DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == - MVI_ENCODE_FUNC); - mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; - if (mvitem->arguments()[0]->eq(args[0], true)) - { - cs= mvitem->arguments()[0]->collation.collation; - break; - } - } - if (!index) + if (!(index= get_mvi_index(&ctx->indexes, args[0]))) return false; + cs= args[0]->collation.collation; + mvitem= (Item_func_mvi_encode *)index->vcol->vcol_info->expr; + /* Get ready to construct the ft queries from the second argument */ const Type_handler *cast_th= mvitem->cast_type().type_handler(); const uchar *start, *end; @@ -316,7 +329,6 @@ bool Item_func_json_contains::mvi_analyze(void *arg) StringBuffer<256> buf; buf.length(0); buf.set_charset(&my_charset_latin1_bin); - DBUG_ASSERT(fixed()); if (!a2_parsed) { val= args[1]->val_json(&tmp_val); @@ -415,21 +427,11 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, DBUG_ASSERT(fixed()); if (arg_count > 2 || !a2_constant) return NULL; - while ((index= it++)) - { - vcol_field= index->vcol; - DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); - DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == - MVI_ENCODE_FUNC); - mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; - if (mvitem->arguments()[0]->eq(args[0], true)) - { - cs= mvitem->arguments()[0]->collation.collation; - break; - } - } - if (!index) + if (!(index= get_mvi_index(indexes, args[0]))) return NULL; + vcol_field= index->vcol; + cs= args[0]->collation.collation; + mvitem= (Item_func_mvi_encode *)index->vcol->vcol_info->expr; const Type_handler *cast_th= mvitem->cast_type().type_handler(); StringBuffer<256> buf; From a33437a38a2be8e48db3bd613b4665255ef7b2dd Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 4 Sep 2026 15:07:46 +0300 Subject: [PATCH 05/39] Factor out common code into Item_func_json_contains::get_mvi_access() Item_func_json_contains::mvi_analyze() and ::create_ft_for_mvi() were near-identical: both checked the arguments, looked up the matching MVI, parsed the constant second argument and ran the same scan loop calling encode_mvi_key(). They differed only in what they did with each encoded key. Move all of that into get_mvi_access(), which returns an Mvi_access, and give Mvi_access two methods: - add_key(), to collect one encoded element key, - create_ft_item(), to build the MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) item. It honors Mvi_access::conjunctive, so JSON_OVERLAPS will get the OR form for free. mvi_analyze() and create_ft_for_mvi() are now thin wrappers around get_mvi_access(). This also fixes a memory leak: the encoded keys were copied with String::copy(), giving each String in Mvi_access::encoded a heap buffer that is never freed (the Strings live on the MEM_ROOT, so their destructors never run). Copy the keys onto the MEM_ROOT instead. Co-Authored-By: Claude Opus 5 (1M context) --- sql/item.h | 1 + sql/item_jsonfunc.h | 1 + sql/opt_multi_valued_index.cc | 248 +++++++++++++++++----------------- sql/opt_multi_valued_index.h | 10 +- 4 files changed, 130 insertions(+), 130 deletions(-) diff --git a/sql/item.h b/sql/item.h index 359b0b163bd41..c2701d0000d6b 100644 --- a/sql/item.h +++ b/sql/item.h @@ -823,6 +823,7 @@ const item_walk_flags WALK_NO_REF= (1<<2); struct Mv_index; +struct Mvi_access; class Item :public Value_source, public Type_all_attributes diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 04e5e490f6a97..4704c3ca65d8a 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -378,6 +378,7 @@ class Item_func_json_contains: public Item_bool_func bool val_bool() override; bool mvi_analyze(void *arg) override; Item *create_ft_for_mvi(THD *thd, List *indexes) override; + Mvi_access *get_mvi_access(THD *thd, List *indexes); protected: Item *shallow_copy(THD *thd) const override diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 1ba04d961cb47..c544a96a745e5 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -306,141 +306,110 @@ static Mv_index *get_mvi_index(List *indexes, return NULL; } -bool Item_func_json_contains::mvi_analyze(void *arg) +/* + Add one encoded element key to the access. + + TODO: String object live on MEM_ROOT and their destructor is never called + (fix that or switch to something like LEX_STRINGs) +*/ + +bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) { - Mvi_context *ctx= (Mvi_context *) arg; - Mv_index *index; - CHARSET_INFO *cs= NULL; - Item_func_mvi_encode *mvitem= NULL; - DBUG_ASSERT(fixed()); - if (arg_count > 2 || !a2_constant) - return false; - /* Find the MVI that matches the first argument */ - if (!(index= get_mvi_index(&ctx->indexes, args[0]))) - return false; + String *s= new (mem_root) String; + const char *copy= (const char *) memdup_root(mem_root, key->ptr(), + key->length()); + if (!s || !copy) + return true; + s->set(copy, key->length(), &my_charset_latin1_bin); + return encoded.push_back(s, mem_root); +} - cs= args[0]->collation.collation; - mvitem= (Item_func_mvi_encode *)index->vcol->vcol_info->expr; - /* Get ready to construct the ft queries from the second argument */ - const Type_handler *cast_th= mvitem->cast_type().type_handler(); - const uchar *start, *end; - Mvi_access *access= NULL; - StringBuffer<256> buf; - buf.length(0); - buf.set_charset(&my_charset_latin1_bin); - if (!a2_parsed) - { - val= args[1]->val_json(&tmp_val); - a2_parsed= true; - } - if (!val) - return false; - start= reinterpret_cast(val->ptr()); - end= start + val->length(); +/* + @brief + Construct the fulltext predicate that implements this access: - if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) - return false; + MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) - if (je.value_type == JSON_VALUE_UNINITIALIZED || - je.value_type == JSON_VALUE_OBJECT) - return false; - if (je.value_type != JSON_VALUE_ARRAY) + @detail + A conjunctive access (JSON_CONTAINS) requires every element key to be + present, so each key gets a '+' prefix. A disjunctive one (JSON_OVERLAPS) + leaves the keys optional. +*/ + +Item *Mvi_access::create_ft_item(THD *thd) +{ + StringBuffer<256> query; + List_iterator it(encoded); + String *key; + List ifm_args; + query.length(0); + query.set_charset(&my_charset_latin1_bin); + while ((key= it++)) { - /* scalar */ - if (!encode_mvi_key(&je, cast_th, cs, &buf)) - { - access= new (ctx->thd->mem_root) Mvi_access(index, true); - /* TODO: there gotta be a less verbose way to construct s. */ - String *s= new (ctx->thd->mem_root) String; - s->set_charset(&my_charset_latin1_bin); - if (s->copy(buf.ptr(), buf.length(), &my_charset_latin1_bin)) - return true; - access->encoded.push_back(s); - } - goto ok; + if (query.length()) + query.append(' '); + if (conjunctive) + query.append('+'); + query.append(key->ptr(), key->length()); } + if (!query.length()) + return NULL; - /* TODO: deduplicate? */ - do { - buf.length(0); - switch (je.state) - { - /* TODO: nested array? */ - case JST_ARRAY_START: - continue; - case JST_ARRAY_END: - break; - case JST_VALUE: - { - if (json_read_value(&je)) - return false; - - if (!encode_mvi_key(&je, cast_th, cs, &buf)) - { - if (!access) - access= new (ctx->thd->mem_root) Mvi_access(index, true); - /* TODO: there gotta be a less verbose way to construct s. */ - String *s= new (ctx->thd->mem_root) String; - s->set_charset(&my_charset_latin1_bin); - if (s->copy(buf.ptr(), buf.length(), &my_charset_latin1_bin)) - return true; - access->encoded.push_back(s); - } - break; - } - default: - return false; - } - } while (json_scan_next(&je) == 0); - -ok: - if (access) - ctx->accesses.push_back(access); - return false; + Item *ift_query= new (thd->mem_root) Item_string(thd, &my_charset_latin1_bin, + query.c_ptr(), + query.length()); + Item *ivcol= new (thd->mem_root) Item_field(thd, index->vcol); + /* Item_func_match takes the query string first: it is its key_item() */ + if (!ift_query || !ivcol || + ifm_args.push_back(ift_query, thd->mem_root) || + ifm_args.push_back(ivcol, thd->mem_root)) + return NULL; + return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); } - /* @brief - Create a fulltext search item that matches this JSON_CONTAINS(...) predicate. + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. @detail Check if this item is a JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') - If yes, create and return an Item for searching for matches in the index: + If yes, collect the encoded element keys to search the index for. - MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) + Elements that cannot be encoded for that index (e.g. because of a type + mismatch) are skipped: the resulting access is a necessary, not a + sufficient condition, and is only ever ANDed with this predicate. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. */ -Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, - List *indexes) +Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, + List *indexes) { - List_iterator it(*indexes); Mv_index *index; - Field *vcol_field= NULL; - CHARSET_INFO *cs= NULL; - Item_func_mvi_encode *mvitem= NULL; + Mvi_access *access= NULL; + StringBuffer<256> buf; + const uchar *start, *end; DBUG_ASSERT(fixed()); + if (arg_count > 2 || !a2_constant) return NULL; + /* Find the MVI that matches the first argument */ if (!(index= get_mvi_index(indexes, args[0]))) return NULL; - vcol_field= index->vcol; - cs= args[0]->collation.collation; - mvitem= (Item_func_mvi_encode *)index->vcol->vcol_info->expr; + CHARSET_INFO *cs= args[0]->collation.collation; + Item_func_mvi_encode *mvitem= + (Item_func_mvi_encode *) index->vcol->vcol_info->expr; + /* Get ready to encode the element keys from the second argument */ const Type_handler *cast_th= mvitem->cast_type().type_handler(); - StringBuffer<256> buf; - const uchar *start, *end; - List ifm_args; - Item_field *ivcol; - Item_string *ift_query; - bool at_least_one= false; - DBUG_ASSERT(fixed()); + buf.length(0); buf.set_charset(&my_charset_latin1_bin); if (!a2_parsed) @@ -459,38 +428,41 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, if (je.value_type == JSON_VALUE_UNINITIALIZED || je.value_type == JSON_VALUE_OBJECT) return NULL; + if (je.value_type != JSON_VALUE_ARRAY) { - /* scalar */ - if (!encode_mvi_key(&je, cast_th, cs, &buf)) - at_least_one= true; - goto ok; + /* A scalar: JSON_CONTAINS(expr, '123') */ + if (encode_mvi_key(&je, cast_th, cs, &buf)) + return NULL; + if (!(access= new (thd->mem_root) Mvi_access(index, true)) || + access->add_key(thd->mem_root, &buf)) + return NULL; + return access; } + // JSON_VALUE_ARRAY /* TODO: deduplicate? */ do { + buf.length(0); switch (je.state) { /* TODO: nested array? */ case JST_ARRAY_START: continue; case JST_ARRAY_END: - if (at_least_one) - buf.length(buf.length() - 1); break; case JST_VALUE: { if (json_read_value(&je)) return NULL; - buf.append('+'); if (encode_mvi_key(&je, cast_th, cs, &buf)) - buf.length(buf.length() - 1); - else - { - buf.append(' '); - at_least_one= true; - } + break; /* Skip: cannot be encoded */ + if (!access && + !(access= new (thd->mem_root) Mvi_access(index, true))) + return NULL; + if (access->add_key(thd->mem_root, &buf)) + return NULL; break; } default: @@ -498,15 +470,39 @@ Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, } } while (json_scan_next(&je) == 0); -ok: - if (!at_least_one) - return NULL; - ift_query= new (thd->mem_root) Item_string(thd, &my_charset_latin1_bin, - buf.c_ptr(), buf.length()); - ifm_args.push_back(ift_query); - ivcol= new (thd->mem_root) Item_field(thd, vcol_field); - ifm_args.push_back(ivcol); - return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); + return access; +} + + +bool Item_func_json_contains::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + Mvi_access *access= get_mvi_access(ctx->thd, &ctx->indexes); + if (access && ctx->accesses.push_back(access, ctx->thd->mem_root)) + return true; /* Out of memory */ + return false; +} + + +/* + @brief + Create a fulltext search item that matches this JSON_CONTAINS(...) predicate. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + If yes, create and return an Item for searching for matches in the index: + + MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) +*/ + +Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, + List *indexes) +{ + Mvi_access *access= get_mvi_access(thd, indexes); + return access ? access->create_ft_item(thd) : NULL; } /* diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index cc63edf47b75d..7789b19321afd 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -30,11 +30,13 @@ struct Mvi_access : public Sql_alloc List encoded; /* encoded element keys */ bool conjunctive; /* CONTAINS -> AND, OVERLAPS -> OR */ Mvi_access(Mv_index *idx, bool conj) : index(idx), conjunctive(conj) {} + + /* Build: Add one encoded element key */ + bool add_key(MEM_ROOT *mem_root, const String *key); + + /* Usage: Construct the fulltext predicate implementing this access */ + Item *create_ft_item(THD *thd); }; -/* - bool Item_func_json_contains::mvi_analyze(THD *thd, List *vcol_fields, - Mvi_access *out); - */ bool setup_mvi_for_join(JOIN *join); From dae9007403d0a012de0585974ff57aa7792bff15 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 4 Sep 2026 20:52:50 +0300 Subject: [PATCH 06/39] Make the MVI scan a real access method: QUICK_MVI_SELECT JSON_CONTAINS() over a multi-valued index used to be optimized by rewriting the WHERE clause: setup_mvi_for_join() injected a synthetic MATCH vcol AGAINST ('+k1 +k2' IN BOOLEAN MODE) into join->conds and into select_lex->ftfunc_list, and the normal fulltext machinery then picked it up as JT_FT access. The injected item showed up in the plan and in the condition even though the user never wrote it, and because it became ordinary ref access the scan was never costed against the alternatives - it won by being in the WHERE clause. Introduce QUICK_MVI_SELECT (QS_TYPE_MVI), a QUICK_SELECT_I that drives the fulltext index directly through the handler API. Unlike FT_SELECT there is no Item_func_match to have created the FT_INFO, so the quick select creates it in reset() with ft_init_ext() and frees it with close_search() in its destructor. Mvi_access::create_ft_item() is replaced by build_ft_query(), which builds just the query string. The analysis in setup_mvi_quick() is now kept: Mvi_context moves to the header, is allocated on the mem_root and stored as JOIN::mvi_ctx, where JOIN::get_mvi_access_for_table() looks it up. get_quick_record_count() builds the quick select before test_quick_select() and keeps whichever of the two is cheaper; test_quick_select() itself is untouched, so the MVI quick is held in a local across the call (it deletes select->quick on entry). The same save/compare is done around the second test_quick_select() call in make_join_select(), which a LIMIT can reach. A fulltext key never gets a bit in const_keys or keys, so mark the MVI key of every table that has an access: the const_keys bit is what lets the range analysis run for that table at all, the keys bit puts the index into EXPLAIN's possible_keys. Collect the accesses from the top-level AND-parts of the WHERE clause only, instead of walking the whole condition. An MVI scan reads just the rows the index matches, so it is only valid for a predicate that must hold for every row of the result: for json_contains(j1->'$.tags','"a"') OR json_contains(j2->'$.tags','"a"') scanning either index would drop the rows that only match the other branch. The deleted add_ft_for_mvi() refused COND_OR_FUNC for the same reason; walking the condition tree lost that, which only became visible once the accesses were actually used. Costs are placeholders (records=10, read_time=0.001) until the engine can estimate a fulltext search. Note that while there is no estimate, an MVI access is also taken when test_quick_select() produced no quick select at all, without comparing it to the cost of a table scan. TODO: This doesn't handle UPDATE/DELETE! Should it be put into check_quick() call? Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 96 ++++++- mysql-test/main/multi_valued_index.test | 63 +++++ sql/item.h | 4 - sql/item_jsonfunc.h | 1 - sql/opt_multi_valued_index.cc | 317 +++++++++++++--------- sql/opt_multi_valued_index.h | 26 +- sql/opt_range.h | 47 +++- sql/sql_explain.cc | 12 +- sql/sql_explain.h | 7 +- sql/sql_select.cc | 99 ++++++- sql/sql_select.h | 14 +- 11 files changed, 527 insertions(+), 159 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 369464b0f5a85..9c17000105cef 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -35,7 +35,7 @@ insert into t1 values (4, '{}'); explain select * from t1 where json_contains(j->'$.tags', '"abcde"'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 fulltext idx idx 0 1 Using where +1 SIMPLE t1 range idx idx 0 NULL 10 Using where select * from t1 where json_contains(j->'$.tags', '"abcde"'); c j 2 {"tags": ["1", "abcde", "", 34567]} @@ -51,14 +51,14 @@ c j explain select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 fulltext idx idx 0 1 Using where +1 SIMPLE t1 range idx idx 0 NULL 10 Using where select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); c j 2 {"tags": ["1", "abcde", "", 34567]} explain select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 fulltext idx idx 0 1 Using where +1 SIMPLE t1 range idx idx 0 NULL 10 Using where select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); c j select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); @@ -73,6 +73,96 @@ select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 1 Using where DROP TABLE t1; +# MVI access is chosen on cost, and survives the re-optimization +# that a LIMIT triggers in make_join_select() +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); +explain select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +c j +1 {"tags": ["aaa"]} +# No MATCH() is added to the WHERE clause +# ("filtered" depends on the engine's row estimate, so hide it) +explain extended select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range idx idx 0 NULL # # Using where +Warnings: +Note 1003 select `test`.`t1`.`c` AS `c`,`test`.`t1`.`j` AS `j` from `test`.`t1` where json_contains(json_extract(`test`.`t1`.`j`,'$.tags'),'"aaa"') +# A predicate under a top-level OR cannot use the index: the scan +# would drop the rows that only match the other branch +explain select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +# The scan is re-initialized for each execution: MVI table on the +# inner side of a join, and in a correlated subquery +create table t2 (a int) engine=innodb; +insert into t2 values (1),(2),(3); +explain select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL # +1 SIMPLE t1 range idx idx 0 NULL # Using where; Using join buffer (flat, BNL join) +select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +a c j +1 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +set @@join_cache_level=0; +select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +a c j +1 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +set @@join_cache_level=default; +select a, (select count(*) from t1 +where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) from t2; +a (select count(*) from t1 +where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) +1 1 +2 0 +3 1 +drop table t1,t2; +# Repeated optimization: prepared statements and stored procedures. +# '"aaa"' can use the index; an object argument cannot, but still +# matches row 2, which the '"aaa"' index scan would not return. +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [{"a":1}]}'); +Warnings: +Warning 4204 Invalid vector format at offset: 3 for '[{"a": 1}]'. Must be a valid JSON array of numbers. +prepare s from 'select * from t1 where json_contains(j->''$.tags'', ?)'; +set @p='"aaa"'; +execute s using @p; +c j +1 {"tags": ["aaa"]} +set @p='{"a":1}'; +execute s using @p; +c j +2 {"tags": [{"a":1}]} +set @p='"aaa"'; +execute s using @p; +c j +1 {"tags": ["aaa"]} +deallocate prepare s; +create procedure p1(x json) select * from t1 where json_contains(j->'$.tags', x); +call p1('"aaa"'); +c j +1 {"tags": ["aaa"]} +call p1('{"a":1}'); +c j +2 {"tags": [{"a":1}]} +call p1('"aaa"'); +c j +1 {"tags": ["aaa"]} +drop procedure p1; +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 8c9d4d8ec3788..aac6702c9dcae 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -46,6 +46,69 @@ select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j DROP TABLE t1; +--echo # MVI access is chosen on cost, and survives the re-optimization +--echo # that a LIMIT triggers in make_join_select() + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); + +explain select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; + +--echo # No MATCH() is added to the WHERE clause +--echo # ("filtered" depends on the engine's row estimate, so hide it) +--replace_column 9 # 10 # +explain extended select * from t1 where json_contains(j->'$.tags','"aaa"'); + +--echo # A predicate under a top-level OR cannot use the index: the scan +--echo # would drop the rows that only match the other branch +--replace_column 9 # +explain select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; + +--echo # The scan is re-initialized for each execution: MVI table on the +--echo # inner side of a join, and in a correlated subquery +create table t2 (a int) engine=innodb; +insert into t2 values (1),(2),(3); +--replace_column 9 # +explain select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +set @@join_cache_level=0; +select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +set @@join_cache_level=default; +select a, (select count(*) from t1 + where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) from t2; + +drop table t1,t2; + +--echo # Repeated optimization: prepared statements and stored procedures. +--echo # '"aaa"' can use the index; an object argument cannot, but still +--echo # matches row 2, which the '"aaa"' index scan would not return. + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [{"a":1}]}'); + +prepare s from 'select * from t1 where json_contains(j->''$.tags'', ?)'; +set @p='"aaa"'; +execute s using @p; +set @p='{"a":1}'; +execute s using @p; +set @p='"aaa"'; +execute s using @p; +deallocate prepare s; + +create procedure p1(x json) select * from t1 where json_contains(j->'$.tags', x); +call p1('"aaa"'); +call p1('{"a":1}'); +call p1('"aaa"'); +drop procedure p1; + +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/sql/item.h b/sql/item.h index c2701d0000d6b..e64a8fe689262 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2863,10 +2863,6 @@ class Item :public Value_source, DBUG_ASSERT(fixed()); return false; } - virtual Item *create_ft_for_mvi(THD *thd, List *indexes) - { - return NULL; - } protected: diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 4704c3ca65d8a..d02a4c83a494b 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -377,7 +377,6 @@ class Item_func_json_contains: public Item_bool_func bool fix_length_and_dec(THD *thd) override; bool val_bool() override; bool mvi_analyze(void *arg) override; - Item *create_ft_for_mvi(THD *thd, List *indexes) override; Mvi_access *get_mvi_access(THD *thd, List *indexes); protected: diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index c544a96a745e5..f56848bf5b594 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -269,19 +269,6 @@ bool collect_mvi_vcols_for_join(JOIN *join, List *indexes) return FALSE; // Ok } -class Mvi_context -{ - public: - THD *thd; - /* All MV indexes in the JOIN */ - List indexes; - /* MVI accesses for all eligible predicates in WHERE */ - List accesses; - - Mvi_context(THD *thd_arg) : thd(thd_arg) {} -}; - - /* Find Multi-Value Index created over array_indexed_expr. */ @@ -327,45 +314,30 @@ bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) /* @brief - Construct the fulltext predicate that implements this access: + Build the boolean-mode fulltext query to find rows of interest. + For conjunctive access it is - MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) + '+encoded_foo +encoded_bar ...' - @detail - A conjunctive access (JSON_CONTAINS) requires every element key to be - present, so each key gets a '+' prefix. A disjunctive one (JSON_OVERLAPS) - leaves the keys optional. + For disjunctive access, it is + + 'encoded_foo encoded_bar' */ -Item *Mvi_access::create_ft_item(THD *thd) +bool Mvi_access::build_ft_query(String *out) { - StringBuffer<256> query; List_iterator it(encoded); String *key; - List ifm_args; - query.length(0); - query.set_charset(&my_charset_latin1_bin); + out->length(0); + out->set_charset(&my_charset_latin1_bin); while ((key= it++)) { - if (query.length()) - query.append(' '); - if (conjunctive) - query.append('+'); - query.append(key->ptr(), key->length()); + if ((out->length() && out->append(' ')) || + (conjunctive && out->append('+')) || + out->append(key->ptr(), key->length())) + return true; } - if (!query.length()) - return NULL; - - Item *ift_query= new (thd->mem_root) Item_string(thd, &my_charset_latin1_bin, - query.c_ptr(), - query.length()); - Item *ivcol= new (thd->mem_root) Item_field(thd, index->vcol); - /* Item_func_match takes the query string first: it is its key_item() */ - if (!ift_query || !ivcol || - ifm_args.push_back(ift_query, thd->mem_root) || - ifm_args.push_back(ivcol, thd->mem_root)) - return NULL; - return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); + return !out->length(); } @@ -484,124 +456,219 @@ bool Item_func_json_contains::mvi_analyze(void *arg) } +static void choose_mvi_access_for_tables(List *accesses, Mvi_access **best) +{ + List_iterator it(*accesses); + /* TODO: cost based */ + /* + TODO: merge + + json_contains(j->'$.tags','"a"') and + json_contains(j->'$.tags','"b"') + + (+ta +tb) + */ + while (Mvi_access *access= it++) + { + DBUG_ASSERT(access->index->vcol->table->tablenr < MAX_TABLES); + best[access->index->vcol->table->tablenr] = access; + } +} + + /* @brief - Create a fulltext search item that matches this JSON_CONTAINS(...) predicate. + Collect the MVI accesses allowed by the top-level AND-parts of `conds'. @detail - Check if this item is a + An MVI access only reads the rows the index scan matches, so we can only + use it for a predicate that has to be true for every row of the result. + That means the top-level conjuncts and nothing else: for - JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') - - If yes, create and return an Item for searching for matches in the index: + json_contains(j1->'$.tags', '"a"') OR json_contains(j2->'$.tags', '"a"') - MATCH vcol AGAINST ('+encoded_foo +encoded_bar ...' IN BOOLEAN MODE) + a scan of either index would drop the rows that only match the other + branch. */ -Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, - List *indexes) +static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) { - Mvi_access *access= get_mvi_access(thd, indexes); - return access ? access->create_ft_item(thd) : NULL; + Item *cond; + if (conds->type() != Item::COND_ITEM) + return conds->mvi_analyze(ctx); + if (((Item_cond *) conds)->functype() != Item_func::COND_AND_FUNC) + return false; + List_iterator it(*((Item_cond *) conds)->argument_list()); + while ((cond= it++)) + { + /* + No recursion: a nested Item_cond is either an already-flattened AND or + an OR, and Item::mvi_analyze() ignores both. + */ + if (cond->mvi_analyze(ctx)) + return true; + } + return false; } + /* @brief - Examine the WHERE clause in (*conds_ref) and add conditions for multi-value - index predicates. + Analyze the WHERE clause and find the MVI accesses it allows. + + @detail + The accesses are saved in join->mvi_ctx, where get_best_mvi_access() picks + them up during the range analysis of each table. +*/ - Since we add fulltext predicates, also add them into *ftfunc_list. +bool setup_mvi_quick(JOIN *join) +{ + THD *thd= join->thd; + Mvi_context *ctx; + /* mvi_ctx must describe this analysis only, including on the early exits */ + join->mvi_ctx= NULL; + if (!join->conds) + return false; + if (!(ctx= new (thd->mem_root) Mvi_context(thd))) + return true; + if (collect_mvi_vcols_for_join(join, &ctx->indexes)) + return true; + if (ctx->indexes.is_empty()) + return false; + if (collect_mvi_accesses(ctx, join->conds)) + return true; + if (ctx->accesses.is_empty()) + return false; + choose_mvi_access_for_tables(&ctx->accesses, ctx->best); + join->mvi_ctx= ctx; + return false; +} + + +Mvi_access *JOIN::get_mvi_access_for_table(TABLE *table) +{ + if (!mvi_ctx) + return NULL; + DBUG_ASSERT(table->tablenr < MAX_TABLES); + return mvi_ctx->best[table->tablenr]; +} + + +/* + @brief + Create a quick select for the best MVI access to `table', if there is one. @detail - Currently we only walk down into top-level's WHERE clause. + The range optimizer cannot produce this access (it skips fulltext keys), + so the caller creates it here and compares its cost with whatever + test_quick_select() came up with. */ -static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, - List *ftfunc_list) +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) { - Item *conds= *conds_ref; - Item *cond, *match; - List matches; - THD *thd= ctx->thd; - if (conds->type() != Item::COND_ITEM) + Mvi_access *access= join->get_mvi_access_for_table(table); + if (!access) + return NULL; + return new QUICK_MVI_SELECT(thd, table, access); +} + + +/**************************************************************************** + QUICK_MVI_SELECT - reading a multi-valued index +****************************************************************************/ + +QUICK_MVI_SELECT::QUICK_MVI_SELECT(THD *thd, TABLE *table, + Mvi_access *access_arg) + : access(access_arg), ft_handler(NULL) +{ + head= table; + index= access->index->keyno; + record= head->record[0]; + /* + TODO: get a real estimate from the engine (see fulltext_estimate()). + Until then, use numbers low enough that the MVI scan is preferred over a + table scan. + */ + records= 10; + read_time= 0.001; +} + + +QUICK_MVI_SELECT::~QUICK_MVI_SELECT() +{ + handler *file= head->file; + if (ft_handler) { - if ((match= conds->create_ft_for_mvi(thd, &ctx->indexes))) - { - matches.push_back(match); - ftfunc_list->push_back((Item_func_match *) match); - } + file->ha_ft_end(); /* ft_end() + file->ft_handler= NULL */ + /* + We created the FT_INFO, so we free it. For an Item_func_match this is + done by Item_func_match::cleanup(). + */ + ft_handler->please->close_search(ft_handler); + ft_handler= NULL; } - else if (((Item_cond *) conds)->functype() == Item_func::COND_OR_FUNC) - return false; - else + if (file->inited != handler::NONE) + file->ha_index_or_rnd_end(); +} + + +int QUICK_MVI_SELECT::reset() +{ + handler *file= head->file; + int error; + + if (!ft_handler) { - List_iterator it(*((Item_cond *) conds)->argument_list()); - while ((cond= it++)) - { - if ((match= cond->create_ft_for_mvi(thd, &ctx->indexes))) - { - matches.push_back(match); - ftfunc_list->push_back((Item_func_match *) match); - } - } + if (access->build_ft_query(&query)) + return HA_ERR_OUT_OF_MEM; + if (!(ft_handler= file->ft_init_ext(FT_BOOL, index, &query))) + return HA_ERR_WRONG_COMMAND; /* the error is already reported */ + /* + ft_init() and ha_ft_read() both work off handler::ft_handler (and + ha_innobase::ft_init() dereferences it without checking), so it has to + be set before we go any further. + */ + file->ft_handler= ft_handler; + head->fulltext_searched= 1; } - if (matches.elements == 1) - cond= matches.pop(); - else - cond= new (thd->mem_root) Item_cond_and(thd, matches); - if (cond && - ((cond->fix_fields(thd, &cond) || - !(conds= and_items(thd, conds, cond)) || - conds->fix_fields(thd, &conds)))) - return true; - *conds_ref= conds; - return false; + if (!file->inited && (error= file->ha_index_init(index, 1))) + return error; + /* This rewinds the search, so it is also right for a repeated reset() */ + return file->ft_init(); } -static void choose_mvi_access_for_tables(List *accesses, Mvi_access **best) + +int QUICK_MVI_SELECT::get_next() { - List_iterator it(*accesses); - /* TODO: cost based */ - /* - TODO: merge + return head->file->ha_ft_read(record); +} - json_contains(j->'$.tags','"a"') and - json_contains(j->'$.tags','"b"') - (+ta +tb) - */ - while (Mvi_access *access= it++) - best[access->index->vcol->table->tablenr] = access; +void QUICK_MVI_SELECT::add_keys_and_lengths(String *key_names, + String *used_lengths) +{ + bool first= TRUE; + + add_key_and_length(key_names, used_lengths, &first); } -/* Build the scan and install it to join */ -bool setup_mvi_quick(JOIN *join) + +Explain_quick_select *QUICK_MVI_SELECT::get_explain(MEM_ROOT *local_alloc) { - Mvi_context ctx(join->thd); - Mvi_access *best[MAX_TABLES]; - bzero(best, sizeof(best)); - if (!join->conds) - return false; - if (collect_mvi_vcols_for_join(join, &ctx.indexes)) - return true; - if (!ctx.indexes.is_empty() && - join->conds->walk(&Item::mvi_analyze, &ctx, WALK_SUBQUERY)) - return true; - choose_mvi_access_for_tables(&ctx.accesses, best); - return false; + Explain_quick_select *res; + if ((res= new (local_alloc) Explain_quick_select(QS_TYPE_MVI))) + res->range.set(local_alloc, &head->key_info[index], max_used_key_length); + return res; } -bool setup_mvi_for_join(JOIN *join) + +#ifndef DBUG_OFF +void QUICK_MVI_SELECT::dbug_dump(int indent, bool verbose) { - Mvi_context ctx(join->thd); - if (!join->conds) - return false; - if (collect_mvi_vcols_for_join(join, &ctx.indexes)) - return true; - if (!ctx.indexes.is_empty()) - return add_ft_for_mvi(&ctx, &join->conds, join->select_lex->ftfunc_list); - return false; + fprintf(DBUG_FILE, "%*squick_mvi_select: index %s (%d)\n", + indent, "", head->key_info[index].name.str, index); } +#endif enum json_value_types mvi_json_class(enum_field_types ftype) { diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 7789b19321afd..be7162b2a8865 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -34,13 +34,33 @@ struct Mvi_access : public Sql_alloc /* Build: Add one encoded element key */ bool add_key(MEM_ROOT *mem_root, const String *key); - /* Usage: Construct the fulltext predicate implementing this access */ - Item *create_ft_item(THD *thd); + /* Usage: Build the fulltext query searching for the element keys */ + bool build_ft_query(String *out); }; -bool setup_mvi_for_join(JOIN *join); + +/* The result of the MVI analysis of one JOIN */ +class Mvi_context : public Sql_alloc +{ + public: + THD *thd; + /* All MV indexes in the JOIN */ + List indexes; + /* MVI accesses for all eligible predicates in WHERE */ + List accesses; + /* The access we've chosen for each table, indexed by table->tablenr */ + Mvi_access *best[MAX_TABLES]; + + Mvi_context(THD *thd_arg) : thd(thd_arg) + { + bzero(best, sizeof(best)); + } +}; /* Return the compatible json type */ enum json_value_types mvi_json_class(enum_field_types ftype); bool setup_mvi_quick(JOIN *join); + +/* Create a quick select for the best MVI access to `table', if there is one */ +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table); diff --git a/sql/opt_range.h b/sql/opt_range.h index bdac62099607a..0e6790da6fe26 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -1216,7 +1216,8 @@ class QUICK_SELECT_I QS_TYPE_FULLTEXT = 4, QS_TYPE_ROR_INTERSECT = 5, QS_TYPE_ROR_UNION = 6, - QS_TYPE_GROUP_MIN_MAX = 7 + QS_TYPE_GROUP_MIN_MAX = 7, + QS_TYPE_MVI = 8 }; /* Get type of this quick select - one of the QS_TYPE_* values */ @@ -2040,6 +2041,50 @@ class FT_SELECT: public QUICK_RANGE_SELECT int get_type() override { return QS_TYPE_FULLTEXT; } }; + +struct Mvi_access; + +/* + Quick select that reads a multi-valued index. + + It runs a boolean-mode fulltext search over the index's hidden vcol, looking + for the encoded element keys of the JSON predicate this access was built + from. The scan is a necessary, not a sufficient condition: the JSON + predicate stays in the WHERE clause and does the exact filtering. + + Unlike FT_SELECT, there is no Item_func_match to have created the FT_INFO + for us, so we create it ourselves in reset() and own it. + + The methods are implemented in opt_multi_valued_index.cc. +*/ + +class QUICK_MVI_SELECT: public QUICK_SELECT_I +{ + Mvi_access *access; + FT_INFO *ft_handler; + StringBuffer<256> query; /* the boolean-mode ft query */ +public: + QUICK_MVI_SELECT(THD *thd, TABLE *table, Mvi_access *access_arg); + ~QUICK_MVI_SELECT(); + int init() override { return 0; } + int reset() override; + int get_next() override; + bool reverse_sorted() override { return false; } + /* + Fulltext results come back ordered by relevance, not by key, so there is + no sorted output to offer. QS_TYPE_MVI is not one of the types the + ORDER BY-by-index code paths consider, so they never ask. + */ + void need_sorted_output() override {} + int get_type() override { return QS_TYPE_MVI; } + void add_keys_and_lengths(String *key_names, String *used_lengths) override; + void add_used_key_part_to_set() override {} + Explain_quick_select *get_explain(MEM_ROOT *alloc) override; +#ifndef DBUG_OFF + void dbug_dump(int indent, bool verbose) override; +#endif +}; + FT_SELECT *get_ft_select(THD *thd, TABLE *table, uint key); QUICK_RANGE_SELECT *get_quick_select_for_ref(THD *thd, TABLE *table, struct st_table_ref *ref, diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index 2e894deae5fce..eb8184bd5b9b9 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -2524,9 +2524,7 @@ void Explain_table_access::append_tag_name(String *str, enum explain_extra_tag t void Explain_quick_select::print_extra(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { /* print nothing */ } @@ -2614,9 +2612,7 @@ const char * Explain_quick_select::get_name_by_type() void Explain_quick_select::print_key(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { if (str->length() > 0) str->append(','); @@ -2640,9 +2636,7 @@ void Explain_quick_select::print_key(String *str) void Explain_quick_select::print_key_len(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { char buf[64]; size_t length; diff --git a/sql/sql_explain.h b/sql/sql_explain.h index 9b07721973b45..a5a092009ce42 100644 --- a/sql/sql_explain.h +++ b/sql/sql_explain.h @@ -705,11 +705,12 @@ class Explain_quick_select : public Sql_alloc const int quick_type; - bool is_basic() + bool is_basic() { - return (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + return (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX); + quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX || + quick_type == QUICK_SELECT_I::QS_TYPE_MVI); } /* This is used when quick_type == QUICK_SELECT_I::QS_TYPE_RANGE */ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 002d8c77d77de..3335619fc844b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -140,7 +140,7 @@ static int sort_keyuse(const void *a, const void *b); static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables); static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables); -static bool get_quick_record_count(THD *thd, SQL_SELECT *select, +static bool get_quick_record_count(THD *thd, JOIN *join, SQL_SELECT *select, TABLE *table, const key_map *keys,ha_rows limit, ha_rows *quick_count); @@ -510,6 +510,7 @@ void JOIN::init(THD *thd_arg, List &fields_arg, result= result_arg; lock= thd_arg->lock; select_lex= 0; //for safety + mvi_ctx= 0; select_distinct= MY_TEST(select_options & SELECT_DISTINCT); no_order= 0; simple_order= 0; @@ -2315,12 +2316,6 @@ JOIN::optimize_inner() DBUG_RETURN(1); } - if (setup_mvi_for_join(this)) - { - error= 1; - DBUG_RETURN(1); - } - if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */ DBUG_RETURN(-1); @@ -5475,11 +5470,50 @@ mysql_select(THD *thd, TABLE_LIST *tables, List &fields, COND *conds, } +/* + @brief + Keep the cheaper of *quick_ref and mvi_quick in *quick_ref, delete the + other one. + + @detail + The range optimizer skips fulltext keys, so it can never produce an MVI + access itself. Instead the caller creates one, hands it to us and we keep + it if test_quick_select() did not come up with anything better. + + mvi_quick may be NULL, which means "there is no MVI access". + + TODO: when the MVI access gets a real cost estimate, also compare it with + the cost of a table scan. Right now, if test_quick_select() produced no + quick select at all (because a table scan was cheaper than any range), we + take the MVI access without asking how much it costs. +*/ + +static void keep_cheaper_quick(TABLE *table, QUICK_SELECT_I **quick_ref, + QUICK_SELECT_I *mvi_quick) +{ + if (!mvi_quick) + return; + if (*quick_ref && (*quick_ref)->read_time <= mvi_quick->read_time) + { + delete mvi_quick; + return; + } + delete *quick_ref; + *quick_ref= mvi_quick; + /* + Callers assume (*quick_ref)->records >= opt_range_condition_rows. This is + a min-setter, so it can only lower the value. + */ + table->set_opt_range_condition_rows(mvi_quick->records); +} + + /** Approximate how many records are going to be returned by this table in this select with this key. @param thd Thread handle + @param join The join the table belongs to @param select Select to be examined @param table The table of interest @param keys The keys of interest @@ -5491,7 +5525,7 @@ mysql_select(THD *thd, TABLE_LIST *tables, List &fields, COND *conds, @retval true Error */ -static bool get_quick_record_count(THD *thd, SQL_SELECT *select, +static bool get_quick_record_count(THD *thd, JOIN *join, SQL_SELECT *select, TABLE *table, const key_map *keys,ha_rows limit, ha_rows *quick_count) @@ -5508,6 +5542,12 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, { select->head=table; table->reginfo.impossible_range=0; + /* + An MVI access is not something test_quick_select() can find. Create it + here and keep it across the call: test_quick_select() deletes + select->quick on entry. + */ + QUICK_SELECT_I *mvi_quick= get_best_mvi_access(thd, join, table); /* EQ_FUNC and EQUAL_FUNC already sent unusable key notes (if any) during update_ref_and_keys(). Have only other functions raise notes @@ -5521,6 +5561,7 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, if (error == SQL_SELECT::OK) { + keep_cheaper_quick(table, &select->quick, mvi_quick); if (select->quick) { /* @@ -5535,6 +5576,8 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, } DBUG_RETURN(false); } + /* Impossible range or an error: the MVI access is of no use */ + delete mvi_quick; if (error == SQL_SELECT::IMPOSSIBLE_RANGE) { table->reginfo.impossible_range=1; @@ -5882,6 +5925,23 @@ make_join_statistics(JOIN *join, List &tables_list, print_keyuse_array_for_trace(thd, keyuse_array); } + /* + A fulltext key never gets a bit in const_keys or keys, so mark the MVI key + of every table that has an MVI access. The const_keys bit is what makes + the range analysis below run for that table, where get_best_mvi_access() + picks the access up; the keys bit puts the index into EXPLAIN's + possible_keys. + */ + for (JOIN_TAB *s= stat ; s < stat_end ; s++) + { + Mvi_access *acc= join->get_mvi_access_for_table(s->table); + if (acc) + { + s->const_keys.set_bit(acc->index->keyno); + s->keys.set_bit(acc->index->keyno); + } + } + join->const_table_map= no_rows_const_tables; join->const_tables= const_count; eliminate_tables(join); @@ -6253,7 +6313,7 @@ make_join_statistics(JOIN *join, List &tables_list, (SORT_INFO*) 0, 1, &error); if (!select) goto error; - if (get_quick_record_count(join->thd, select, s->table, + if (get_quick_record_count(join->thd, join, select, s->table, &s->const_keys, join->row_limit, &records)) { /* There was an error in test_quick_select */ @@ -14873,6 +14933,19 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) sel->cond->quick_fix_field(); quick_select_return res; + /* + Take an MVI quick select out of sel->quick before the call: + test_quick_select() deletes it on entry and cannot produce + another one, so it would be lost for good. + */ + QUICK_SELECT_I *mvi_quick= NULL; + if (sel->quick && + sel->quick->get_type() == QUICK_SELECT_I::QS_TYPE_MVI) + { + mvi_quick= sel->quick; + sel->quick= 0; + } + if ((res= sel->test_quick_select(thd, tab->keys, ((used_tables & ~ current_map) | OUTER_REF_TABLE_BIT), @@ -14900,13 +14973,21 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) 0, FALSE, FALSE, FALSE, Item_func::BITMAP_NONE)) == SQL_SELECT::IMPOSSIBLE_RANGE) + { + delete mvi_quick; DBUG_RETURN(1); // Impossible WHERE + } } else sel->cond=orig_cond; if (res == SQL_SELECT::ERROR) + { + delete mvi_quick; DBUG_RETURN(1); /* Some error in one of test_quick_select calls */ + } + + keep_cheaper_quick(sel->head, &sel->quick, mvi_quick); /* Fix for EXPLAIN */ if (sel->quick) diff --git a/sql/sql_select.h b/sql/sql_select.h index d1e97994915ad..0b517076580d5 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1453,6 +1453,9 @@ class AGGR_OP :public Sql_alloc }; +class Mvi_context; +struct Mvi_access; + class JOIN :public Sql_alloc { private: @@ -1788,7 +1791,13 @@ class JOIN :public Sql_alloc SELECT_LEX_UNIT *unit; /// select that processed SELECT_LEX *select_lex; - /** + /* + The result of the multi-valued index analysis, or NULL if there is no + usable MVI access. Produced by setup_mvi_quick(), used by + get_best_mvi_access() during range analysis. + */ + Mvi_context *mvi_ctx; + /** TRUE <=> optimizer must not mark any table as a constant table. This is needed for subqueries in form "a IN (SELECT .. UNION SELECT ..): when we optimize the select that reads the results of the union from a @@ -2002,6 +2011,9 @@ class JOIN :public Sql_alloc void init(THD *thd_arg, List &fields_arg, ulonglong select_options_arg, select_result *result_arg); + /* Return the MVI access chosen for `table', or NULL if there is none */ + Mvi_access *get_mvi_access_for_table(TABLE *table); + /* True if the plan guarantees that it will be returned zero or one row */ bool only_const_tables() { return const_tables == table_count; } /* Number of tables actually joined at the top level */ From 39ceedfe72ad4a9d863cc3d4ffb0779a897f3548 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 5 Sep 2026 12:17:46 +0300 Subject: [PATCH 07/39] Make optimizer trace print "range", not "index_merge" for MVI quick selects. --- sql/sql_select.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 3335619fc844b..4a5835c7e0438 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -9947,7 +9947,13 @@ best_access_path(JOIN *join, } else { - type= JT_INDEX_MERGE; + if (s->quick->get_type() == QUICK_SELECT_I::QS_TYPE_MVI) + type= JT_RANGE; + else + { + type= JT_INDEX_MERGE; + force_plan= s->quick->force_index_merge; + } /* We don't know exactly from where the costs comes from. Let's store it in copy_cost. @@ -9956,7 +9962,6 @@ best_access_path(JOIN *join, */ cost.reset(); cost.copy_cost= s->quick->read_time; - force_plan= s->quick->force_index_merge; } loose_scan_opt.check_range_access(join, idx, s->quick); } From 287527fee4537244f9ba1a7e271ae820a46d7d1e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 5 Sep 2026 12:45:22 +0300 Subject: [PATCH 08/39] Add optimizer trace for the multi-valued index access get_best_mvi_access() picked an Mvi_access and wrapped it in a QUICK_MVI_SELECT without recording anything, so there was no way to see which index was chosen or what it would search that index for: the rows_estimation trace showed a range_analysis that found nothing, and then a plan using a key the trace never mentioned. Print a "multi_value_index_use" object: { "table": "t1", "index": "idx", "ranges": ["616161"] } Mvi_access::print_json() fills in the index and the element keys, following TRP_RANGE::trace_basic_info(): same "index" / "ranges" member names, so an MVI entry reads like a range scan's. The keys are printed in their encoded form, which is not readable. That is what is stored in the index and what we search for, so it is still the useful thing to print; making it readable can come later. It is plain ASCII (hex plus the xx/xxxx padding from encode_mvi_key()), so it needs no JSON escaping. get_best_mvi_access() runs inside the "rows_estimation" array, so the named object needs an object of its own around it, the same way make_join_statistics() and the sel_arg_alloc_limit_hit trace do it. Without it the writer hits an assertion in Single_line_formatting_helper::on_add_member(). The new test is a separate file because optimizer trace tests need not_embedded.inc, and putting that in multi_valued_index.test would skip the whole feature test on embedded builds. It cross-checks the printed keys against mvi_encode() over the indexed column, which produces the tokens the index is actually built from. Co-Authored-By: Claude Opus 5 (1M context) --- .../multi_valued_index_notembedded.result | 89 +++++++++++++++++++ .../main/multi_valued_index_notembedded.test | 64 +++++++++++++ sql/opt_multi_valued_index.cc | 35 ++++++++ sql/opt_multi_valued_index.h | 5 ++ 4 files changed, 193 insertions(+) create mode 100644 mysql-test/main/multi_valued_index_notembedded.result create mode 100644 mysql-test/main/multi_valued_index_notembedded.test diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result new file mode 100644 index 0000000000000..b712ad8832411 --- /dev/null +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -0,0 +1,89 @@ +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb","ccc"]}'); +# The trace prints the element keys in their encoded form, which is not +# readable. These are the tokens the index is built from, so the values +# the trace prints below must be found here: +select c, mvi_encode(j->'$.tags', char(6)) from t1; +c mvi_encode(j->'$.tags', char(6)) +1 616161 +2 626262 636363 +set optimizer_trace=1; +# +# One element key +# +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "ranges": + ["616161"] + } +] +# +# Several element keys +# +explain select * from t1 where json_contains(j->'$.tags','["bbb","ccc"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "ranges": + [ + "626262", + "636363" + ] + } +] +# +# Two tables: each entry names the table it belongs to +# +create table t2 (c int, j json, +key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t2 values (1,'{"tags": ["zzz"]}'); +explain select * from t1,t2 where json_contains(t1.j->'$.tags','"aaa"') +and json_contains(t2.j->'$.tags','"zzz"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx2 idx2 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 10 Using where; Using join buffer (flat, BNL join) +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "ranges": + ["616161"] + }, + { + "table": "t2", + "index": "idx2", + "ranges": + ["7a7a7a"] + } +] +# +# A predicate under a top-level OR gives no access, so nothing is printed +# +explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +NULL +set optimizer_trace=default; +drop table t1,t2; diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test new file mode 100644 index 0000000000000..3fcc56c4b941e --- /dev/null +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -0,0 +1,64 @@ +--source include/have_debug.inc +--source include/have_innodb.inc +# The test uses the optimizer trace: +--source include/not_embedded.inc + +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb","ccc"]}'); + +--echo # The trace prints the element keys in their encoded form, which is not +--echo # readable. These are the tokens the index is built from, so the values +--echo # the trace prints below must be found here: +select c, mvi_encode(j->'$.tags', char(6)) from t1; + +set optimizer_trace=1; + +--echo # +--echo # One element key +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # Several element keys +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_contains(j->'$.tags','["bbb","ccc"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # Two tables: each entry names the table it belongs to +--echo # +create table t2 (c int, j json, + key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t2 values (1,'{"tags": ["zzz"]}'); +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1,t2 where json_contains(t1.j->'$.tags','"aaa"') + and json_contains(t2.j->'$.tags','"zzz"'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # A predicate under a top-level OR gives no access, so nothing is printed +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +set optimizer_trace=default; +drop table t1,t2; diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index f56848bf5b594..77014ed1d5a2d 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -17,6 +17,7 @@ #include "mariadb.h" #include "sql_select.h" #include "item_func.h" +#include "my_json_writer.h" void Item_func_mvi_encode::print(String *str, enum_query_type query_type) { @@ -341,6 +342,28 @@ bool Mvi_access::build_ft_query(String *out) } +/* + @brief + Print the index this access uses and the element keys it will search that + index for into the optimizer trace. + + @detail + The keys are printed in their encoded form. That is what is stored in the + index and what we search for, but it is not readable. +*/ + +void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) +{ + KEY *key_info= index->vcol->table->key_info + index->keyno; + List_iterator it(encoded); + String *key; + trace_object->add("index", key_info->name); + Json_writer_array trace_ranges(thd, "ranges"); + while ((key= it++)) + trace_ranges.add(key->ptr(), key->length()); +} + + /* @brief Check if we can use Multi-Value Index access to read rows for this @@ -569,6 +592,18 @@ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) Mvi_access *access= join->get_mvi_access_for_table(table); if (!access) return NULL; + if (unlikely(thd->trace_started())) + { + /* + We are inside the "rows_estimation" array, so we need an object of our + own before we can add anything by name. Without it the writer hits an + assertion in Single_line_formatting_helper::on_add_member(). + */ + Json_writer_object trace_wrapper(thd); + Json_writer_object trace_mvi(thd, "multi_value_index_use"); + trace_mvi.add_table_name(table); + access->print_json(thd, &trace_mvi); + } return new QUICK_MVI_SELECT(thd, table, access); } diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index be7162b2a8865..216eb9012354f 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -14,6 +14,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ +class Json_writer_object; + /* An MVI index */ struct Mv_index : public Sql_alloc { @@ -36,6 +38,9 @@ struct Mvi_access : public Sql_alloc /* Usage: Build the fulltext query searching for the element keys */ bool build_ft_query(String *out); + + /* Usage: describe this access in the optimizer trace */ + void print_json(THD *thd, Json_writer_object *trace_object); }; From 1d9319058cd020c6d393109086c98c38d2bba802 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 5 Sep 2026 13:06:33 +0300 Subject: [PATCH 09/39] Make JSON_OVERLAPS sargable for multi-valued indexes Both argument orders are handled, since JSON_OVERLAPS is symmetric: JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ...]') JSON_OVERLAPS('[foo, bar, ...]', array_indexed_expr) JSON_CONTAINS is true when ALL of the elements have a match, JSON_OVERLAPS when ANY of them does, so the access it produces has conjunctive=false and build_ft_query() leaves the keys optional instead of prefixing them with '+'. The two get_mvi_access() implementations share collect_mvi_keys(), which is the scan of the JSON literal that used to sit inside Item_func_json_contains::get_mvi_access(). The two differ in one way beyond the flag. An element that cannot be encoded for the index (a number against a CHAR array, say) is skipped for JSON_CONTAINS: dropping a key from an AND makes the index scan less selective, so it still returns a superset of the rows the predicate matches and the predicate does the exact filtering afterwards. That reasoning does not hold for an OR. A row can satisfy the predicate through the very element we failed to encode, and MVI_ENCODE skips such elements as well, so that row has no key in the index for the scan to find it by - dropping the key would lose it. So for a disjunctive access we give up instead of skipping. With a row {"tags": [123]} in the table, select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]') must not use the index, and the test checks its result against the same query with IGNORE INDEX. Also print "match": "all"/"any" in the optimizer trace. Now that an access can be either, the printed ranges alone did not say whether a row has to have all of the keys or just one. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 79 +++++++ mysql-test/main/multi_valued_index.test | 50 +++++ .../multi_valued_index_notembedded.result | 25 +++ .../main/multi_valued_index_notembedded.test | 10 + sql/item_jsonfunc.h | 2 + sql/opt_multi_valued_index.cc | 202 +++++++++++++----- 6 files changed, 318 insertions(+), 50 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 9c17000105cef..9cd704c81ed47 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -163,6 +163,85 @@ c j 1 {"tags": ["aaa"]} drop procedure p1; drop table t1; +# JSON_OVERLAPS is sargable too. It is true when ANY of the elements +# has a match, so the keys are ORed, and it is symmetric: the indexed +# expression can be either argument. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), +(5,'{"tags": [123]}'),(6,'{}'); +explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# the indexed expression as the second argument +explain select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# a scalar instead of an array +select * from t1 where json_overlaps(j->'$.tags','"ccc"') order by c; +c j +3 {"tags": ["ccc"]} +# 123 cannot be encoded for a CHAR array, so it is not in the index +# either. Row 5 matches the predicate through it, so an index scan +# for the remaining key would lose that row: don't use the index. +explain select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +4 {"tags": ["aaa","bbb"]} +5 {"tags": [123]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +4 {"tags": ["aaa","bbb"]} +5 {"tags": [123]} +# JSON_CONTAINS may skip such an element: dropping a key from an AND +# only makes the scan less selective, so it stays a superset. +explain select * from t1 where json_contains(j->'$.tags','[123,"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select * from t1 where json_contains(j->'$.tags','[123,"aaa"]') order by c; +c j +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[123,"aaa"]') order by c; +c j +# Forms that cannot use the index. None of these may change the result. +select * from t1 where json_overlaps(j->'$.tags', NULL) order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','[]') order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','{"a":1}') order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +# neither argument is a constant +select * from t1 a, t1 b +where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; +c j c j +1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} +1 {"tags": ["aaa"]} 4 {"tags": ["aaa","bbb"]} +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index aac6702c9dcae..85cb9bd674701 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -109,6 +109,56 @@ drop procedure p1; drop table t1; +--echo # JSON_OVERLAPS is sargable too. It is true when ANY of the elements +--echo # has a match, so the keys are ORed, and it is symmetric: the indexed +--echo # expression can be either argument. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), + (5,'{"tags": [123]}'),(6,'{}'); + +explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); +select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; + +--echo # the indexed expression as the second argument +explain select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags'); +select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags') order by c; + +--echo # a scalar instead of an array +select * from t1 where json_overlaps(j->'$.tags','"ccc"') order by c; + +--echo # 123 cannot be encoded for a CHAR array, so it is not in the index +--echo # either. Row 5 matches the predicate through it, so an index scan +--echo # for the remaining key would lose that row: don't use the index. +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; + +--echo # JSON_CONTAINS may skip such an element: dropping a key from an AND +--echo # only makes the scan less selective, so it stays a superset. +explain select * from t1 where json_contains(j->'$.tags','[123,"aaa"]'); +select * from t1 where json_contains(j->'$.tags','[123,"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[123,"aaa"]') order by c; + +--echo # Forms that cannot use the index. None of these may change the result. +select * from t1 where json_overlaps(j->'$.tags', NULL) order by c; +select * from t1 where json_overlaps(j->'$.tags','[]') order by c; +select * from t1 where json_overlaps(j->'$.tags','{"a":1}') order by c; +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +--echo # neither argument is a constant +select * from t1 a, t1 b +where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; + +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result index b712ad8832411..3092da52ead76 100644 --- a/mysql-test/main/multi_valued_index_notembedded.result +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -23,6 +23,7 @@ jd { "table": "t1", "index": "idx", + "match": "all", "ranges": ["616161"] } @@ -40,6 +41,7 @@ jd { "table": "t1", "index": "idx", + "match": "all", "ranges": [ "626262", @@ -65,17 +67,40 @@ jd { "table": "t1", "index": "idx", + "match": "all", "ranges": ["616161"] }, { "table": "t2", "index": "idx2", + "match": "all", "ranges": ["7a7a7a"] } ] # +# JSON_OVERLAPS: the keys are ORed, so "match" is "any" +# +explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 10 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "match": "any", + "ranges": + [ + "626262", + "636363" + ] + } +] +# # A predicate under a top-level OR gives no access, so nothing is printed # explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test index 3fcc56c4b941e..529ad180171f3 100644 --- a/mysql-test/main/multi_valued_index_notembedded.test +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -50,6 +50,16 @@ select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; --enable_view_protocol +--echo # +--echo # JSON_OVERLAPS: the keys are ORed, so "match" is "any" +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + --echo # --echo # A predicate under a top-level OR gives no access, so nothing is printed --echo # diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index d02a4c83a494b..e7dc491f8e1b1 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -948,6 +948,8 @@ class Item_func_json_overlaps: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; + bool mvi_analyze(void *arg) override; + Mvi_access *get_mvi_access(THD *thd, List *indexes); Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } }; diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 77014ed1d5a2d..4d736e7a824e3 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -350,6 +350,9 @@ bool Mvi_access::build_ft_query(String *out) @detail The keys are printed in their encoded form. That is what is stored in the index and what we search for, but it is not readable. + + "match" tells whether a row has to have all of the keys (JSON_CONTAINS) + or just one of them (JSON_OVERLAPS). */ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) @@ -357,7 +360,8 @@ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) KEY *key_info= index->vcol->table->key_info + index->keyno; List_iterator it(encoded); String *key; - trace_object->add("index", key_info->name); + trace_object->add("index", key_info->name). + add("match", conjunctive ? "all" : "any"); Json_writer_array trace_ranges(thd, "ranges"); while ((key= it++)) trace_ranges.add(key->ptr(), key->length()); @@ -366,70 +370,60 @@ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) /* @brief - Check if we can use Multi-Value Index access to read rows for this - predicate, if yes create an access descriptor. + Collect the element keys to search `index' for from a JSON literal. - @detail - Check if this item is a + @param cs Collation of the indexed expression + TODO why does that matter? - JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + @param json The JSON literal: an array, or a single scalar + @param conjunctive true when the keys are ANDed (JSON_CONTAINS), + false when they are ORed (JSON_OVERLAPS) + @param je A json_engine_t to scan with - If yes, collect the encoded element keys to search the index for. + @detail + An element that cannot be encoded for this index (a type mismatch, say) + can only be skipped when the keys are ANDed. Dropping a key from an AND + makes the index scan less selective, so it still returns a superset of + the rows the predicate matches, and the predicate itself does the exact + filtering afterwards. - Elements that cannot be encoded for that index (e.g. because of a type - mismatch) are skipped: the resulting access is a necessary, not a - sufficient condition, and is only ever ANDed with this predicate. + For an OR we cannot do that. A row can satisfy the predicate through the + very element we failed to encode, and MVI_ENCODE skips such elements too, + so that row has no key in the index for us to find it by. Dropping the + key would lose it. Give up on the access instead. @return - The access descriptor, or NULL if the predicate cannot use an MVI. + The access descriptor, or NULL if the predicate cannot use this MVI. */ -Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, - List *indexes) +static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, + CHARSET_INFO *cs, String *json, + bool conjunctive, json_engine_t *je) { - Mv_index *index; Mvi_access *access= NULL; StringBuffer<256> buf; - const uchar *start, *end; - DBUG_ASSERT(fixed()); - - if (arg_count > 2 || !a2_constant) - return NULL; - /* Find the MVI that matches the first argument */ - if (!(index= get_mvi_index(indexes, args[0]))) - return NULL; - - CHARSET_INFO *cs= args[0]->collation.collation; + const uchar *start= reinterpret_cast(json->ptr()); + const uchar *end= start + json->length(); Item_func_mvi_encode *mvitem= (Item_func_mvi_encode *) index->vcol->vcol_info->expr; - /* Get ready to encode the element keys from the second argument */ const Type_handler *cast_th= mvitem->cast_type().type_handler(); buf.length(0); buf.set_charset(&my_charset_latin1_bin); - if (!a2_parsed) - { - val= args[1]->val_json(&tmp_val); - a2_parsed= true; - } - if (!val) - return NULL; - start= reinterpret_cast(val->ptr()); - end= start + val->length(); - if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) + if (json_scan_start(je, cs, start, end) || json_read_value(je)) return NULL; - if (je.value_type == JSON_VALUE_UNINITIALIZED || - je.value_type == JSON_VALUE_OBJECT) + if (je->value_type == JSON_VALUE_UNINITIALIZED || + je->value_type == JSON_VALUE_OBJECT) return NULL; - if (je.value_type != JSON_VALUE_ARRAY) + if (je->value_type != JSON_VALUE_ARRAY) { /* A scalar: JSON_CONTAINS(expr, '123') */ - if (encode_mvi_key(&je, cast_th, cs, &buf)) + if (encode_mvi_key(je, cast_th, cs, &buf)) return NULL; - if (!(access= new (thd->mem_root) Mvi_access(index, true)) || + if (!(access= new (thd->mem_root) Mvi_access(index, conjunctive)) || access->add_key(thd->mem_root, &buf)) return NULL; return access; @@ -439,7 +433,7 @@ Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, /* TODO: deduplicate? */ do { buf.length(0); - switch (je.state) + switch (je->state) { /* TODO: nested array? */ case JST_ARRAY_START: @@ -448,13 +442,18 @@ Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, break; case JST_VALUE: { - if (json_read_value(&je)) + if (json_read_value(je)) return NULL; - if (encode_mvi_key(&je, cast_th, cs, &buf)) - break; /* Skip: cannot be encoded */ + if (encode_mvi_key(je, cast_th, cs, &buf)) + { + /* See above: only an AND of the keys tolerates a missing one */ + if (!conjunctive) + return NULL; + break; + } if (!access && - !(access= new (thd->mem_root) Mvi_access(index, true))) + !(access= new (thd->mem_root) Mvi_access(index, conjunctive))) return NULL; if (access->add_key(thd->mem_root, &buf)) return NULL; @@ -463,19 +462,122 @@ Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, default: return NULL; } - } while (json_scan_next(&je) == 0); + } while (json_scan_next(je) == 0); return access; } +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + which is true when ALL of the elements have a match, so the keys are + ANDed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + DBUG_ASSERT(fixed()); + + if (arg_count > 2 || !a2_constant) + return NULL; + /* Find the MVI that matches the first argument */ + if (!(index= get_mvi_index(indexes, args[0]))) + return NULL; + + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + + return collect_mvi_keys(thd, index, args[0]->collation.collation, val, + true, &je); +} + + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + We can use MVI index when the predicate has either of the forms: + + JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') + JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) + + JSON_OVERLAPS is true when ANY of the elements has a match, so the keys + are ORed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + uint literal_arg; + String *json; + StringBuffer<256> tmp; + DBUG_ASSERT(fixed()); + + if ((index= get_mvi_index(indexes, args[0]))) + literal_arg= 1; + else if ((index= get_mvi_index(indexes, args[1]))) + literal_arg= 0; + else + return NULL; + + if (!args[literal_arg]->const_item()) + return NULL; + if (!(json= args[literal_arg]->val_json(&tmp))) + return NULL; + + /* + encode_mvi_key() must see the collation of the indexed expression: that + is what decides how MVI_ENCODE built the keys that are in the index. + */ + return collect_mvi_keys(thd, index, + args[1 - literal_arg]->collation.collation, json, + false, &je); +} + + +/* Add `access' to the context, if there is one. Returns true on error */ + +static bool add_mvi_access(Mvi_context *ctx, Mvi_access *access) +{ + return access && ctx->accesses.push_back(access, ctx->thd->mem_root); +} + + bool Item_func_json_contains::mvi_analyze(void *arg) { Mvi_context *ctx= (Mvi_context *) arg; - Mvi_access *access= get_mvi_access(ctx->thd, &ctx->indexes); - if (access && ctx->accesses.push_back(access, ctx->thd->mem_root)) - return true; /* Out of memory */ - return false; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); +} + + +bool Item_func_json_overlaps::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); } From 84f2cbf59c20c9b743d2111aa430f0a102128e9f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 08:07:31 +0300 Subject: [PATCH 10/39] Move the JSON function MVI code into opt_mvi_jsonfuncs.cc opt_multi_valued_index.cc held two separate concerns: the index side (how a value is encoded into the index, the access descriptor, the quick select) and the predicate side (which JSON functions can be computed from an MVI, which of their arguments holds the indexed expression, and what to search the index for). Split the second one out. Moved verbatim: get_mvi_index() collect_mvi_keys() Item_func_json_contains::get_mvi_access() and ::mvi_analyze() Item_func_json_overlaps::get_mvi_access() and ::mvi_analyze() add_mvi_access() The only code change is that encode_mvi_key() is no longer static: it is used both by Item_func_mvi_encode::val_str_ascii(), which stays, and by collect_mvi_keys(), which moves. It is declared in opt_multi_valued_index.h now. The other three moved helpers had no callers outside the moved code and stay static. Item_func_mvi_encode is not a JSON function and stays put: it is how the values get into the index in the first place. Co-Authored-By: Claude Opus 5 (1M context) --- sql/CMakeLists.txt | 1 + sql/opt_multi_valued_index.cc | 240 +----------------------------- sql/opt_multi_valued_index.h | 8 + sql/opt_mvi_jsonfuncs.cc | 265 ++++++++++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 238 deletions(-) create mode 100644 sql/opt_mvi_jsonfuncs.cc diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index bd01fba5a41dd..48eef3ca4bfac 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -200,6 +200,7 @@ SET (SQL_SOURCE socketpair.c socketpair.h opt_multi_valued_index.h opt_multi_valued_index.cc + opt_mvi_jsonfuncs.cc opt_vcol_substitution.h opt_vcol_substitution.cc opt_hints_parser.cc opt_hints_parser.h scan_char.h diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 4d736e7a824e3..e3cd33e182ab2 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -87,8 +87,8 @@ static void store_sort_key_longlong(uchar *to, bool unsigned_flag, to[0]= (uchar) (value >> 56) ^ (unsigned_flag ? 0 : 128); } -static bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, - CHARSET_INFO *cs, String *buf) +bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf) { enum_field_types cast_ftype= cast_th->field_type(); bool is_unsigned= cast_th->is_unsigned(); @@ -270,29 +270,6 @@ bool collect_mvi_vcols_for_join(JOIN *join, List *indexes) return FALSE; // Ok } -/* - Find Multi-Value Index created over array_indexed_expr. -*/ -static Mv_index *get_mvi_index(List *indexes, - Item *array_indexed_expr) -{ - Mv_index *index; - List_iterator it(*indexes); - Item_func_mvi_encode *mvitem; - while ((index= it++)) - { - Field *vcol_field= index->vcol; - DBUG_ASSERT(vcol_field->vcol_info->expr->type() == Item::FUNC_ITEM); - DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == - Item_func::MVI_ENCODE_FUNC); - mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; - if (mvitem->arguments()[0]->eq(array_indexed_expr, true)) - { - return index; - } - } - return NULL; -} /* Add one encoded element key to the access. @@ -368,219 +345,6 @@ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) } -/* - @brief - Collect the element keys to search `index' for from a JSON literal. - - @param cs Collation of the indexed expression - TODO why does that matter? - - @param json The JSON literal: an array, or a single scalar - @param conjunctive true when the keys are ANDed (JSON_CONTAINS), - false when they are ORed (JSON_OVERLAPS) - @param je A json_engine_t to scan with - - @detail - An element that cannot be encoded for this index (a type mismatch, say) - can only be skipped when the keys are ANDed. Dropping a key from an AND - makes the index scan less selective, so it still returns a superset of - the rows the predicate matches, and the predicate itself does the exact - filtering afterwards. - - For an OR we cannot do that. A row can satisfy the predicate through the - very element we failed to encode, and MVI_ENCODE skips such elements too, - so that row has no key in the index for us to find it by. Dropping the - key would lose it. Give up on the access instead. - - @return - The access descriptor, or NULL if the predicate cannot use this MVI. -*/ - -static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, - CHARSET_INFO *cs, String *json, - bool conjunctive, json_engine_t *je) -{ - Mvi_access *access= NULL; - StringBuffer<256> buf; - const uchar *start= reinterpret_cast(json->ptr()); - const uchar *end= start + json->length(); - Item_func_mvi_encode *mvitem= - (Item_func_mvi_encode *) index->vcol->vcol_info->expr; - const Type_handler *cast_th= mvitem->cast_type().type_handler(); - - buf.length(0); - buf.set_charset(&my_charset_latin1_bin); - - if (json_scan_start(je, cs, start, end) || json_read_value(je)) - return NULL; - - if (je->value_type == JSON_VALUE_UNINITIALIZED || - je->value_type == JSON_VALUE_OBJECT) - return NULL; - - if (je->value_type != JSON_VALUE_ARRAY) - { - /* A scalar: JSON_CONTAINS(expr, '123') */ - if (encode_mvi_key(je, cast_th, cs, &buf)) - return NULL; - if (!(access= new (thd->mem_root) Mvi_access(index, conjunctive)) || - access->add_key(thd->mem_root, &buf)) - return NULL; - return access; - } - // JSON_VALUE_ARRAY - - /* TODO: deduplicate? */ - do { - buf.length(0); - switch (je->state) - { - /* TODO: nested array? */ - case JST_ARRAY_START: - continue; - case JST_ARRAY_END: - break; - case JST_VALUE: - { - if (json_read_value(je)) - return NULL; - - if (encode_mvi_key(je, cast_th, cs, &buf)) - { - /* See above: only an AND of the keys tolerates a missing one */ - if (!conjunctive) - return NULL; - break; - } - if (!access && - !(access= new (thd->mem_root) Mvi_access(index, conjunctive))) - return NULL; - if (access->add_key(thd->mem_root, &buf)) - return NULL; - break; - } - default: - return NULL; - } - } while (json_scan_next(je) == 0); - - return access; -} - - -/* - @brief - Check if we can use Multi-Value Index access to read rows for this - predicate, if yes create an access descriptor. - - @detail - Check if this item is a - - JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') - - which is true when ALL of the elements have a match, so the keys are - ANDed. - - @return - The access descriptor, or NULL if the predicate cannot use an MVI. -*/ - -Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, - List *indexes) -{ - Mv_index *index; - DBUG_ASSERT(fixed()); - - if (arg_count > 2 || !a2_constant) - return NULL; - /* Find the MVI that matches the first argument */ - if (!(index= get_mvi_index(indexes, args[0]))) - return NULL; - - if (!a2_parsed) - { - val= args[1]->val_json(&tmp_val); - a2_parsed= true; - } - if (!val) - return NULL; - - return collect_mvi_keys(thd, index, args[0]->collation.collation, val, - true, &je); -} - - -/* - @brief - Check if we can use Multi-Value Index access to read rows for this - predicate, if yes create an access descriptor. - - @detail - We can use MVI index when the predicate has either of the forms: - - JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') - JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) - - JSON_OVERLAPS is true when ANY of the elements has a match, so the keys - are ORed. - - @return - The access descriptor, or NULL if the predicate cannot use an MVI. -*/ - -Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, - List *indexes) -{ - Mv_index *index; - uint literal_arg; - String *json; - StringBuffer<256> tmp; - DBUG_ASSERT(fixed()); - - if ((index= get_mvi_index(indexes, args[0]))) - literal_arg= 1; - else if ((index= get_mvi_index(indexes, args[1]))) - literal_arg= 0; - else - return NULL; - - if (!args[literal_arg]->const_item()) - return NULL; - if (!(json= args[literal_arg]->val_json(&tmp))) - return NULL; - - /* - encode_mvi_key() must see the collation of the indexed expression: that - is what decides how MVI_ENCODE built the keys that are in the index. - */ - return collect_mvi_keys(thd, index, - args[1 - literal_arg]->collation.collation, json, - false, &je); -} - - -/* Add `access' to the context, if there is one. Returns true on error */ - -static bool add_mvi_access(Mvi_context *ctx, Mvi_access *access) -{ - return access && ctx->accesses.push_back(access, ctx->thd->mem_root); -} - - -bool Item_func_json_contains::mvi_analyze(void *arg) -{ - Mvi_context *ctx= (Mvi_context *) arg; - return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); -} - - -bool Item_func_json_overlaps::mvi_analyze(void *arg) -{ - Mvi_context *ctx= (Mvi_context *) arg; - return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); -} - - static void choose_mvi_access_for_tables(List *accesses, Mvi_access **best) { List_iterator it(*accesses); diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 216eb9012354f..9076f392aedc2 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -65,6 +65,14 @@ class Mvi_context : public Sql_alloc /* Return the compatible json type */ enum json_value_types mvi_json_class(enum_field_types ftype); +/* + Encode one JSON value into the form it has in the index. Returns true if + the value cannot be encoded for this index and has to be skipped. + Shared with opt_mvi_jsonfuncs.cc. +*/ +bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf); + bool setup_mvi_quick(JOIN *join); /* Create a quick select for the best MVI access to `table', if there is one */ diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc new file mode 100644 index 0000000000000..d03e202e89a3a --- /dev/null +++ b/sql/opt_mvi_jsonfuncs.cc @@ -0,0 +1,265 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Making the JSON functions use a Multi-Value Index. + + This is the part of the MVI support that knows about the JSON predicates: + which of them can be computed from an index, which argument holds the + indexed expression, and what to search the index for. The index side of it + lives in opt_multi_valued_index.cc. +*/ + +#include "mariadb.h" +#include "sql_select.h" +#include "item_func.h" + +/* + Find Multi-Value Index created over array_indexed_expr. +*/ +static Mv_index *get_mvi_index(List *indexes, + Item *array_indexed_expr) +{ + Mv_index *index; + List_iterator it(*indexes); + Item_func_mvi_encode *mvitem; + while ((index= it++)) + { + Field *vcol_field= index->vcol; + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == Item::FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(array_indexed_expr, true)) + { + return index; + } + } + return NULL; +} + + +/* + @brief + Collect the element keys to search `index' for from a JSON literal. + + @param cs Collation of the indexed expression + TODO why does that matter? + + @param json The JSON literal: an array, or a single scalar + @param conjunctive true when the keys are ANDed (JSON_CONTAINS), + false when they are ORed (JSON_OVERLAPS) + @param je A json_engine_t to scan with + + @detail + An element that cannot be encoded for this index (a type mismatch, say) + can only be skipped when the keys are ANDed. Dropping a key from an AND + makes the index scan less selective, so it still returns a superset of + the rows the predicate matches, and the predicate itself does the exact + filtering afterwards. + + For an OR we cannot do that. A row can satisfy the predicate through the + very element we failed to encode, and MVI_ENCODE skips such elements too, + so that row has no key in the index for us to find it by. Dropping the + key would lose it. Give up on the access instead. + + @return + The access descriptor, or NULL if the predicate cannot use this MVI. +*/ + +static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, + CHARSET_INFO *cs, String *json, + bool conjunctive, json_engine_t *je) +{ + Mvi_access *access= NULL; + StringBuffer<256> buf; + const uchar *start= reinterpret_cast(json->ptr()); + const uchar *end= start + json->length(); + Item_func_mvi_encode *mvitem= + (Item_func_mvi_encode *) index->vcol->vcol_info->expr; + const Type_handler *cast_th= mvitem->cast_type().type_handler(); + + buf.length(0); + buf.set_charset(&my_charset_latin1_bin); + + if (json_scan_start(je, cs, start, end) || json_read_value(je)) + return NULL; + + if (je->value_type == JSON_VALUE_UNINITIALIZED || + je->value_type == JSON_VALUE_OBJECT) + return NULL; + + if (je->value_type != JSON_VALUE_ARRAY) + { + /* A scalar: JSON_CONTAINS(expr, '123') */ + if (encode_mvi_key(je, cast_th, cs, &buf)) + return NULL; + if (!(access= new (thd->mem_root) Mvi_access(index, conjunctive)) || + access->add_key(thd->mem_root, &buf)) + return NULL; + return access; + } + // JSON_VALUE_ARRAY + + /* TODO: deduplicate? */ + do { + buf.length(0); + switch (je->state) + { + /* TODO: nested array? */ + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + break; + case JST_VALUE: + { + if (json_read_value(je)) + return NULL; + + if (encode_mvi_key(je, cast_th, cs, &buf)) + { + /* See above: only an AND of the keys tolerates a missing one */ + if (!conjunctive) + return NULL; + break; + } + if (!access && + !(access= new (thd->mem_root) Mvi_access(index, conjunctive))) + return NULL; + if (access->add_key(thd->mem_root, &buf)) + return NULL; + break; + } + default: + return NULL; + } + } while (json_scan_next(je) == 0); + + return access; +} + + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + which is true when ALL of the elements have a match, so the keys are + ANDed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + DBUG_ASSERT(fixed()); + + if (arg_count > 2 || !a2_constant) + return NULL; + /* Find the MVI that matches the first argument */ + if (!(index= get_mvi_index(indexes, args[0]))) + return NULL; + + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + + return collect_mvi_keys(thd, index, args[0]->collation.collation, val, + true, &je); +} + + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + We can use MVI index when the predicate has either of the forms: + + JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') + JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) + + JSON_OVERLAPS is true when ANY of the elements has a match, so the keys + are ORed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + uint literal_arg; + String *json; + StringBuffer<256> tmp; + DBUG_ASSERT(fixed()); + + if ((index= get_mvi_index(indexes, args[0]))) + literal_arg= 1; + else if ((index= get_mvi_index(indexes, args[1]))) + literal_arg= 0; + else + return NULL; + + if (!args[literal_arg]->const_item()) + return NULL; + if (!(json= args[literal_arg]->val_json(&tmp))) + return NULL; + + /* + encode_mvi_key() must see the collation of the indexed expression: that + is what decides how MVI_ENCODE built the keys that are in the index. + */ + return collect_mvi_keys(thd, index, + args[1 - literal_arg]->collation.collation, json, + false, &je); +} + + +/* Add `access' to the context, if there is one. Returns true on error */ + +static bool add_mvi_access(Mvi_context *ctx, Mvi_access *access) +{ + return access && ctx->accesses.push_back(access, ctx->thd->mem_root); +} + + +bool Item_func_json_contains::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); +} + + +bool Item_func_json_overlaps::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); +} From 8d84dfdd61efa46c891cabf7f6843f08df7035eb Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 2 Sep 2026 22:03:14 +0300 Subject: [PATCH 11/39] MDEV-40168: JSON-over-fulltext: add estimates. Add records_in_range-like estimates for fulltext index --- mysql-test/suite/innodb_fts/r/estimate.result | 108 +++++ .../suite/innodb_fts/r/estimate_big.result | 46 ++ mysql-test/suite/innodb_fts/t/estimate.opt | 1 + mysql-test/suite/innodb_fts/t/estimate.test | 91 ++++ .../suite/innodb_fts/t/estimate_big.opt | 1 + .../suite/innodb_fts/t/estimate_big.test | 88 ++++ sql/handler.h | 32 ++ sql/item_func.cc | 13 + storage/innobase/fts/fts0fts.cc | 416 ++++++++++++++++++ storage/innobase/handler/ha_innodb.cc | 126 ++++++ storage/innobase/handler/ha_innodb.h | 5 + storage/innobase/include/fts0fts.h | 35 ++ 12 files changed, 962 insertions(+) create mode 100644 mysql-test/suite/innodb_fts/r/estimate.result create mode 100644 mysql-test/suite/innodb_fts/r/estimate_big.result create mode 100644 mysql-test/suite/innodb_fts/t/estimate.opt create mode 100644 mysql-test/suite/innodb_fts/t/estimate.test create mode 100644 mysql-test/suite/innodb_fts/t/estimate_big.opt create mode 100644 mysql-test/suite/innodb_fts/t/estimate_big.test diff --git a/mysql-test/suite/innodb_fts/r/estimate.result b/mysql-test/suite/innodb_fts/r/estimate.result new file mode 100644 index 0000000000000..598dd13442f89 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate.result @@ -0,0 +1,108 @@ +SET @optimize= @@GLOBAL.innodb_optimize_fulltext_only; +SET GLOBAL innodb_optimize_fulltext_only= 1; +CREATE TABLE t1 ( +id INT PRIMARY KEY, +a TEXT, +FULLTEXT(a) +) ENGINE=InnoDB; +INSERT INTO t1 VALUES +(1,'alpha beta gamma'), (2,'beta gamma'), (3,'beta gamma'), +(4,'gamma'), (5,'gamma'), (6,'gamma'), (7,'gamma'), (8,'gamma'); +# +# Nothing has been SYNCed yet, so the words live only in the in-memory +# FTS cache, which the estimate deliberately does not consult. The +# auxiliary table is still empty, which means we have no information at +# all rather than "no matching rows", so the answer is "unknown". +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('gamma')= unknown +SET debug_dbug=''; +# +# Flush the cache into the auxiliary tables. +# +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +# +# Each word now occupies a single auxiliary record, so the walk covers +# the whole of its key range and the estimate is exact. +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('beta')= 3 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('gamma')= 8 +# +# A word that is not indexed at all: floored to 1, not 0. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('zzzzz')= 1 +# +# The word is folded with the collation of the fulltext index, so case +# does not matter. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('GAMMA'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('GAMMA')= 8 +SET debug_dbug=''; +# +# Cross-check against the authoritative auxiliary table contents. +# +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +# One auxiliary record per word here, so DOC_COUNT is the exact answer. +SELECT DISTINCT WORD, DOC_COUNT FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE +ORDER BY WORD; +WORD DOC_COUNT +alpha 1 +beta 3 +gamma 8 +# +# Deleted rows are deliberately still counted: their entries survive in +# the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 +# is then clamped to the number of rows left in the table, so 4 is +# reported -- the clamp, not the deletions, is what moved the number. +# +DELETE FROM t1 WHERE id > 4; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +4 +Warnings: +Note 1105 fulltext_estimate('gamma')= 4 +SET debug_dbug=''; +# +# An engine that does not implement fulltext_estimate() reports +# "unknown" via the handler default. +# +SET GLOBAL innodb_ft_aux_table= @aux; +ALTER TABLE t1 ENGINE=MyISAM; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('gamma')= unknown +SET debug_dbug=''; +DROP TABLE t1; +SET GLOBAL innodb_optimize_fulltext_only= @optimize; diff --git a/mysql-test/suite/innodb_fts/r/estimate_big.result b/mysql-test/suite/innodb_fts/r/estimate_big.result new file mode 100644 index 0000000000000..16cc65e19579a --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate_big.result @@ -0,0 +1,46 @@ +# +# Case 1: the word's documents are evenly spread over its doc id range, +# which is what the extrapolation assumes. +# +CREATE TABLE t1 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET debug_dbug=''; +# One auxiliary record per insert, well past the FTS_EST_MAX_RECS +# budget of 64, so the answer below is extrapolated rather than counted. +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +SELECT COUNT(*) AS n_records FROM +(SELECT DISTINCT WORD, FIRST_DOC_ID, DOC_COUNT +FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE WHERE WORD='gamma') x; +n_records +80 +SET GLOBAL innodb_ft_aux_table= @aux; +# Evenly spread, so the extrapolation lands on the true count. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +80 +Warnings: +Note 1105 fulltext_estimate('gamma')= 80 +SET debug_dbug=''; +DROP TABLE t1; +# +# Case 2: the documented limitation. When a word occurs only at both +# ends of the doc id range, the density measured over the sampled prefix +# does not describe the gap in the middle, and the estimate is far too +# high. Two probes cannot tell this apart from a word that really does +# occur throughout, so the estimator errs on the high side. +# +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET debug_dbug=''; +# 71 rows really match, but the estimate is pulled up towards the size +# of the doc id span, and then clamped to the number of rows. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +71 +Warnings: +Note 1105 fulltext_estimate('gamma')= 471 +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/t/estimate.opt b/mysql-test/suite/innodb_fts/t/estimate.opt new file mode 100644 index 0000000000000..444dfb0dcf530 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate.opt @@ -0,0 +1 @@ +--innodb-ft-index-table diff --git a/mysql-test/suite/innodb_fts/t/estimate.test b/mysql-test/suite/innodb_fts/t/estimate.test new file mode 100644 index 0000000000000..db1a102fcc33a --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate.test @@ -0,0 +1,91 @@ +# +# handler::fulltext_estimate() -- the fulltext analogue of records_in_range(). +# +# The estimate is read back through a debug-only hook in +# Item_func_match::init_search(), which pushes it as a note. +# +--source include/have_innodb.inc +--source include/have_debug.inc + +SET @optimize= @@GLOBAL.innodb_optimize_fulltext_only; +SET GLOBAL innodb_optimize_fulltext_only= 1; + +CREATE TABLE t1 ( + id INT PRIMARY KEY, + a TEXT, + FULLTEXT(a) +) ENGINE=InnoDB; + +# gamma appears in all 8 rows, beta in 3, alpha in 1. +INSERT INTO t1 VALUES + (1,'alpha beta gamma'), (2,'beta gamma'), (3,'beta gamma'), + (4,'gamma'), (5,'gamma'), (6,'gamma'), (7,'gamma'), (8,'gamma'); + +--echo # +--echo # Nothing has been SYNCed yet, so the words live only in the in-memory +--echo # FTS cache, which the estimate deliberately does not consult. The +--echo # auxiliary table is still empty, which means we have no information at +--echo # all rather than "no matching rows", so the answer is "unknown". +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +--echo # +--echo # Flush the cache into the auxiliary tables. +--echo # +OPTIMIZE TABLE t1; + +--echo # +--echo # Each word now occupies a single auxiliary record, so the walk covers +--echo # the whole of its key range and the estimate is exact. +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); + +--echo # +--echo # A word that is not indexed at all: floored to 1, not 0. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); + +--echo # +--echo # The word is folded with the collation of the fulltext index, so case +--echo # does not matter. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('GAMMA'); +SET debug_dbug=''; + +--echo # +--echo # Cross-check against the authoritative auxiliary table contents. +--echo # +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +--echo # One auxiliary record per word here, so DOC_COUNT is the exact answer. +SELECT DISTINCT WORD, DOC_COUNT FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE + ORDER BY WORD; + +--echo # +--echo # Deleted rows are deliberately still counted: their entries survive in +--echo # the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 +--echo # is then clamped to the number of rows left in the table, so 4 is +--echo # reported -- the clamp, not the deletions, is what moved the number. +--echo # +DELETE FROM t1 WHERE id > 4; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +--echo # +--echo # An engine that does not implement fulltext_estimate() reports +--echo # "unknown" via the handler default. +--echo # +SET GLOBAL innodb_ft_aux_table= @aux; +ALTER TABLE t1 ENGINE=MyISAM; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t1; +SET GLOBAL innodb_optimize_fulltext_only= @optimize; diff --git a/mysql-test/suite/innodb_fts/t/estimate_big.opt b/mysql-test/suite/innodb_fts/t/estimate_big.opt new file mode 100644 index 0000000000000..444dfb0dcf530 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_big.opt @@ -0,0 +1 @@ +--innodb-ft-index-table diff --git a/mysql-test/suite/innodb_fts/t/estimate_big.test b/mysql-test/suite/innodb_fts/t/estimate_big.test new file mode 100644 index 0000000000000..bc60069edbba9 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_big.test @@ -0,0 +1,88 @@ +# +# handler::fulltext_estimate() when a word owns more auxiliary records than +# the estimator is willing to read, so it extrapolates instead of counting. +# +# Note: the numbers below depend on how many auxiliary records each word ends +# up occupying, so they have to be re-recorded if the FTS node layout or the +# FTS_EST_MAX_RECS / FTS_EST_MAX_PAGES budgets change. +# +--source include/have_innodb.inc +--source include/have_debug.inc + +--echo # +--echo # Case 1: the word's documents are evenly spread over its doc id range, +--echo # which is what the extrapolation assumes. +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; + +# Force a synchronous cache flush on every commit, so each single-row +# transaction leaves a node of its own in the auxiliary table. This is what a +# table that takes incremental inserts and is never OPTIMIZEd looks like. +SET debug_dbug='+d,fts_instrument_sync_debug'; +--disable_query_log +let $i= 0; +while ($i < 80) +{ + inc $i; + eval INSERT INTO t1 VALUES ($i, 'gamma'); +} +# 40 more rows without the word, so the table has clearly more rows than the +# word has documents. That keeps the row count clamp out of the picture and +# leaves the extrapolated value itself visible below. +while ($i < 120) +{ + inc $i; + eval INSERT INTO t1 VALUES ($i, 'delta'); +} +--enable_query_log +SET debug_dbug=''; + +--echo # One auxiliary record per insert, well past the FTS_EST_MAX_RECS +--echo # budget of 64, so the answer below is extrapolated rather than counted. +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +SELECT COUNT(*) AS n_records FROM + (SELECT DISTINCT WORD, FIRST_DOC_ID, DOC_COUNT + FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE WHERE WORD='gamma') x; +SET GLOBAL innodb_ft_aux_table= @aux; + +--echo # Evenly spread, so the extrapolation lands on the true count. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t1; + +--echo # +--echo # Case 2: the documented limitation. When a word occurs only at both +--echo # ends of the doc id range, the density measured over the sampled prefix +--echo # does not describe the gap in the middle, and the estimate is far too +--echo # high. Two probes cannot tell this apart from a word that really does +--echo # occur throughout, so the estimator errs on the high side. +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; + +SET debug_dbug='+d,fts_instrument_sync_debug'; +--disable_query_log +let $i= 0; +while ($i < 70) +{ + inc $i; + eval INSERT INTO t2 VALUES ($i, 'gamma'); +} +while ($i < 470) +{ + inc $i; + eval INSERT INTO t2 VALUES ($i, 'delta'); +} +INSERT INTO t2 VALUES (471, 'gamma'); +--enable_query_log +SET debug_dbug=''; + +--echo # 71 rows really match, but the estimate is pulled up towards the size +--echo # of the doc id span, and then clamped to the number of rows. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t2; diff --git a/sql/handler.h b/sql/handler.h index fa9196f189a8f..276bf091e1e63 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -4557,6 +4557,38 @@ class handler :public Sql_alloc const key_range *max_key, page_range *res) { return (ha_rows) 10; } + + /** + Estimate how many records a fulltext search for a single word will match. + + The fulltext analogue of records_in_range(): it probes fulltext index + @a index_nr cheaply, without performing the search. + + @param index_nr number of a fulltext index of this table + @param word a single literal word, in the character set of the + fulltext index. NOT a query: no boolean mode operators, + no wildcards, no phrases, no query expansion. + @param word_len length of @a word in bytes + + The result is an estimate. An engine may ignore index entries of rows + that were deleted but not yet purged, may ignore not yet flushed in-memory + index buffers, and may ignore stopword and token length rules. It is + therefore neither an upper nor a lower bound. + + @retval HA_POS_ERROR no estimate available: not a fulltext index, the + engine cannot estimate, an I/O or consistency + problem. The caller must fall back to its own + guess. This is called during optimization, so no + error is raised and no warning is pushed. + @return estimated number of matching records, >= 1. Never + 0: like records_in_range(), callers may treat 0 as + "provably empty", and this estimate may not make + that claim. + */ + virtual ha_rows fulltext_estimate(uint index_nr, const char *word, + uint word_len) + { return HA_POS_ERROR; } + /* If HA_PRIMARY_KEY_REQUIRED_FOR_POSITION is set, then it sets ref (reference to the row, aka position, with the primary key given in diff --git a/sql/item_func.cc b/sql/item_func.cc index 2b0bbbba70991..aa0df4f9df3d9 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -6343,6 +6343,19 @@ bool Item_func_match::init_search(THD *thd, bool no_order) if (join_key && !no_order) match_flags|=FT_SORTED; + DBUG_EXECUTE_IF("fulltext_estimate", + if (key != NO_SUCH_KEY) + { + char buff[22]; + ha_rows rows= table->file->fulltext_estimate(key, ft_tmp->ptr(), + (uint) ft_tmp->length()); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_UNKNOWN_ERROR, "fulltext_estimate('%.*s')= %s", + (int) ft_tmp->length(), ft_tmp->ptr(), + rows == HA_POS_ERROR ? "unknown" + : llstr((longlong) rows, buff)); + }); + if (key != NO_SUCH_KEY) THD_STAGE_INFO(table->in_use, stage_fulltext_initialization); diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 402f8b6e61553..5c20b4486d796 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -888,6 +888,422 @@ fts_index_get_charset( return fts_get_charset(prtype); } + +/** Maximum number of auxiliary records fts_estimate_word_docs() samples +before it gives up on an exact answer and extrapolates instead. */ +static constexpr uint32_t FTS_EST_MAX_RECS = 64; +/** Maximum number of auxiliary leaf pages fts_estimate_word_docs() reads. */ +static constexpr uint32_t FTS_EST_MAX_PAGES = 4; + +/* Physical field numbers of an FTS auxiliary INDEX_[1..6] record. The +clustered index is UNIQUE(word, first_doc_id), so the record is +(word, first_doc_id, DB_TRX_ID, DB_ROLL_PTR, last_doc_id, doc_count, ilist); +see fts_create_one_index_table(). */ +static constexpr ulint FTS_AUX_FLD_WORD = 0; +static constexpr ulint FTS_AUX_FLD_FIRST_DOC_ID = 1; +static constexpr ulint FTS_AUX_FLD_LAST_DOC_ID = 4; +static constexpr ulint FTS_AUX_FLD_DOC_COUNT = 5; +/** Number of offsets the estimator needs. This deliberately stops short of +the ilist (field 6), which may be stored off-page: the estimator must never +read a BLOB. */ +static constexpr ulint FTS_AUX_EST_N_FIELDS = FTS_AUX_FLD_DOC_COUNT + 1; + +/** Build the search tuple (word) for an FTS auxiliary clustered index. +The index key is (word, first_doc_id), and because the tuple compares on its +first field only it compares equal to every record of the word. So +PAGE_CUR_GE positions on the word's first record and PAGE_CUR_LE on its last. +@param[in] heap heap to allocate the tuple from +@param[in] aux_index auxiliary clustered index +@param[in] word word to search for +@return the search tuple */ +static +dtuple_t* +fts_est_word_tuple( + mem_heap_t* heap, + dict_index_t* aux_index, + const fts_string_t* word) noexcept +{ + dtuple_t* tuple = dtuple_create(heap, 1); + + dict_index_copy_types(tuple, aux_index, 1); + dfield_set_data(dtuple_get_nth_field(tuple, 0), + word->f_str, word->f_len); + dtuple_set_n_fields_cmp(tuple, 1); + + return(tuple); +} + +/** Read the fields the estimator needs out of an FTS auxiliary INDEX_[1..6] +leaf record. Never touches the ilist. +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in] rec auxiliary table record +@param[in] aux_index auxiliary clustered index +@param[in] offsets rec_get_offsets(rec, aux_index, ...) +@param[out] first_doc_id first doc id in this record's ilist +@param[out] last_doc_id last doc id in this record's ilist +@param[out] doc_count number of doc ids in this record's ilist +@return whether the record belongs to the word tuple was built for */ +static +bool +fts_est_read_rec( + const dtuple_t* tuple, + const rec_t* rec, + const dict_index_t* aux_index, + const rec_offs* offsets, + doc_id_t* first_doc_id, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + /* tuple has n_fields_cmp == 1, so this compares the word only, in the + collation of the auxiliary table's word column. */ + if (cmp_dtuple_rec(tuple, rec, aux_index, offsets)) { + return(false); + } + + ulint len; + const byte* data; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_FIRST_DOC_ID, &len); + *first_doc_id = (data && len == sizeof *first_doc_id) + ? fts_read_doc_id(data) : 0; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_LAST_DOC_ID, &len); + *last_doc_id = (data && len == sizeof *last_doc_id) + ? fts_read_doc_id(data) : *first_doc_id; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_DOC_COUNT, &len); + *doc_count = (data && len == 4) ? mach_read_from_4(data) : 0; + + return(true); +} + +/** What fts_est_sample_word() found. */ +struct fts_est_sample_t +{ + /** Sum of doc_count over the records sampled. */ + uint64_t docs; + /** Number of records sampled. */ + uint32_t n_recs; + /** First doc id of the word's first record. */ + doc_id_t first_doc_id; + /** Last doc id of the last record sampled. */ + doc_id_t last_doc_id; + /** Whether sampling stopped because a budget ran out rather than + because the word's key range ended. When this is false, docs is the + exact number of documents that contain the word. */ + bool truncated; +}; + +/** Dive to a word's first auxiliary record and walk forward from it, over at +most FTS_EST_MAX_RECS records and FTS_EST_MAX_PAGES leaf pages. The walk is +nearly free because the dive has already latched the leaf page, and if it +reaches a different word before a budget runs out the result is exact. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[out] out what the walk found +@return DB_SUCCESS, or DB_RECORD_NOT_FOUND if the auxiliary table is empty +(nothing has been SYNCed yet, so there is no information at all) */ +static +dberr_t +fts_est_sample_word( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + fts_est_sample_t* out) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + memset(out, 0, sizeof *out); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err != DB_SUCCESS) { + goto func_exit; + } + + if (!btr_pcur_is_on_user_rec(&pcur)) { + /* Nothing at or after the word. Tell an empty auxiliary + table, where we have no information at all, apart from a word + that simply sorts after everything that is indexed. */ + const page_t* page = btr_pcur_get_page(&pcur); + + if (!page_has_prev(page) && !page_get_n_recs(page)) { + err = DB_RECORD_NOT_FOUND; + } + + goto func_exit; + } + + { + uint32_t n_pages = 1; + page_id_t last_page + = btr_pcur_get_block(&pcur)->page.id(); + + do { + const rec_t* rec = btr_pcur_get_rec(&pcur); + const buf_block_t* block + = btr_pcur_get_block(&pcur); + const ulint offs + = ulint(rec - block->page.frame); + + if (page_rec_is_infimum_low(offs) + || page_rec_is_supremum_low(offs)) { + continue; + } + + if (block->page.id() != last_page) { + last_page = block->page.id(); + if (++n_pages > FTS_EST_MAX_PAGES) { + out->truncated = true; + break; + } + } + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + doc_id_t first; + doc_id_t last; + uint32_t doc_count; + + if (!fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &doc_count)) { + /* Walked past the word's key range, so we + have seen all of it. */ + break; + } + + if (!out->n_recs) { + out->first_doc_id = first; + } + + out->last_doc_id = last; + out->docs += doc_count; + + if (++out->n_recs >= FTS_EST_MAX_RECS) { + out->truncated = true; + break; + } + } while (btr_pcur_move_to_next(&pcur, &mtr)); + } + +func_exit: + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Dive to a word's last auxiliary record, to learn how far its doc ids +reach. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in,out] last_doc_id last doc id of the word; left alone if the + record cannot be read +@param[in,out] doc_count doc_count of that record; left alone if the + record cannot be read +@return DB_SUCCESS or error code */ +static +dberr_t +fts_est_last_rec( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err == DB_SUCCESS) { + /* PAGE_CUR_LE may leave the cursor on the page infimum, in + which case the record we want is the last one of the previous + page. */ + const bool positioned + = !btr_pcur_is_before_first_on_page(&pcur) + || btr_pcur_move_to_prev(&pcur, &mtr); + + if (positioned && btr_pcur_is_on_user_rec(&pcur)) { + const rec_t* rec = btr_pcur_get_rec(&pcur); + doc_id_t first; + doc_id_t last; + uint32_t count; + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + if (fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &count)) { + *last_doc_id = last; + *doc_count = count; + } + } + } + + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Estimate how many documents contain a word, by probing an already opened +FTS auxiliary INDEX_[1..6] table. See fts_estimate_word_docs(). +@param[in] trx transaction, used only for the mtr +@param[in] aux auxiliary table that holds the word +@param[in] word word to look up +@param[out] n_docs estimated number of matching documents +@return DB_SUCCESS, DB_RECORD_NOT_FOUND or DB_CORRUPTION */ +static +dberr_t +fts_est_probe_aux( + trx_t* trx, + dict_table_t* aux, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + dict_index_t* aux_index = dict_table_get_first_index(aux); + + if (!aux->space || !aux->is_readable() || !aux_index + || aux_index->page == FIL_NULL || aux_index->is_corrupted()) { + return(DB_CORRUPTION); + } + + mem_heap_t* heap = mem_heap_create(256); + const dtuple_t* tuple = fts_est_word_tuple(heap, aux_index, + word); + fts_est_sample_t s; + dberr_t err = fts_est_sample_word(trx, aux_index, + tuple, &s); + + if (err != DB_SUCCESS) { + mem_heap_free(heap); + return(err); + } + + if (!s.n_recs || !s.truncated) { + /* Either the word is absent, or the walk covered its whole + key range, in which case the count is exact. */ + *n_docs = s.docs; + mem_heap_free(heap); + return(DB_SUCCESS); + } + + /* The word has more records than we are willing to read. Dive once + more, to its last record, and extrapolate the density we measured + across the word's whole doc id span. Doc ids only ever increase, so + the records we sampled are a prefix of that span. */ + doc_id_t last_doc_id = s.last_doc_id; + uint32_t last_count = 0; + + err = fts_est_last_rec(trx, aux_index, tuple, &last_doc_id, + &last_count); + + if (err == DB_SUCCESS) { + if (last_doc_id < s.last_doc_id) { + last_doc_id = s.last_doc_id; + } + + const uint64_t sampled_span + = s.last_doc_id > s.first_doc_id + ? s.last_doc_id - s.first_doc_id + 1 : 1; + const uint64_t total_span + = last_doc_id - s.first_doc_id + 1; + + /* In double, because docs * total_span overflows 64 bits for + large inputs. The caller clamps the result to the number of + rows in the table. */ + uint64_t est = uint64_t(double(s.docs) + * double(total_span) + / double(sampled_span)); + + /* Never below what we actually counted. */ + if (est < s.docs + last_count) { + est = s.docs + last_count; + } + + *n_docs = est; + } + + mem_heap_free(heap); + + return(err); +} + +dberr_t +fts_estimate_word_docs( + trx_t* trx, + dict_index_t* index, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + ut_ad(index->type & DICT_FTS); + ut_ad(!dict_sys.locked()); + ut_ad(word->f_len); + + *n_docs = 0; + + /* A word lives in exactly one of INDEX_1..INDEX_6, so only that one + auxiliary table is ever opened -- unlike the query path, which opens + all six. */ + CHARSET_INFO* cs = fts_index_get_charset(index); + const uint8_t selected = fts_select_index(cs, word->f_str, + word->f_len); + fts_table_t fts_table; + + FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), + FTS_INDEX_TABLE, index); + + char aux_name[MAX_FULL_NAME_LEN]; + + fts_get_table_name(&fts_table, aux_name, false); + + dict_table_t* aux = dict_table_open_on_name( + aux_name, false, DICT_ERR_IGNORE_TABLESPACE); + + if (!aux) { + return(DB_TABLE_NOT_FOUND); + } + + const dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); + + aux->release(); + + return(err); +} /****************************************************************//** Create an FTS index cache. @return Index Cache */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b60155f995683..cc31ba8f9c28a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14741,6 +14741,132 @@ ha_innobase::records_in_range( goto cleanup; } +/*********************************************************************//** +Estimates the number of records a fulltext search for a single word will +match. The fulltext analogue of records_in_range(). + +Unlike records_in_range() this leaves all handler state alone (in particular +active_index and trx->op_info): the optimizer may well discard this access +path. And unlike ft_init_ext() it does not lazily run fts_init_index(): an +estimate must have no side effects. +@return estimated number of rows, or HA_POS_ERROR if not known */ + +ha_rows +ha_innobase::fulltext_estimate( +/*===========================*/ + uint index_nr, /*!< in: fulltext index number */ + const char* word, /*!< in: word to search for, in the + character set of the fulltext index */ + uint word_len) /*!< in: length of word in bytes */ +{ + DBUG_ENTER("ha_innobase::fulltext_estimate"); + + if (!word || !word_len || word_len > FTS_MAX_WORD_LEN) { + DBUG_RETURN(HA_POS_ERROR); + } + + ut_ad(m_prebuilt->trx == thd_to_trx(ha_thd())); + + dict_table_t* ft_table = m_prebuilt->table; + + /* Mirrors the validation of ft_init_ext(), but reports nothing: this + runs during optimization, where raising an error would corrupt the + statement. */ + if (!ft_table->fts + || ib_vector_is_empty(ft_table->fts->indexes) + || !ft_table->space) { /* tablespace discarded */ + DBUG_RETURN(HA_POS_ERROR); + } + + dict_index_t* index = innobase_get_index(index_nr); + + if (!index + || index->type != DICT_FTS + || index->is_corrupted() + || !row_merge_is_index_usable(m_prebuilt->trx, index)) { + DBUG_RETURN(HA_POS_ERROR); + } + + /* The auxiliary tables store words folded to lower case, so fold the + search word the same way fts_query() folds the query string. */ + CHARSET_INFO* cs = fts_index_get_charset(index); + + /* A utf16 or utf32 fulltext index would need the conversion that + ft_init_ext() does; not worth it for an estimate. */ + if (cs->mbminlen != 1) { + DBUG_RETURN(HA_POS_ERROR); + } + + byte buf[FTS_MAX_WORD_LEN * 2 + 1]; + fts_string_t w; + + w.f_n_char = 0; + + if (my_binary_compare(cs)) { + /* Binary collations are searched case sensitively. */ + w.f_str = reinterpret_cast(const_cast(word)); + w.f_len = word_len; + } else { + const size_t buf_len + = word_len * cs->casedn_multiply() + 1; + + if (buf_len > sizeof buf) { + DBUG_RETURN(HA_POS_ERROR); + } + + w.f_len = cs->casedn_z(word, word_len, + reinterpret_cast(buf), buf_len); + w.f_str = buf; + } + + if (!w.f_len) { + DBUG_RETURN(HA_POS_ERROR); + } + + uint64_t n_docs; + dberr_t err; + + { + /* Attribute the pages read to this handler, the way + records_in_range() does. No private transaction is needed: + fts_estimate_word_docs() only probes the B-tree, so it takes + no locks, opens no read view and never commits anything. */ + mariadb_set_stats temp(m_prebuilt->trx, handler_stats); + + err = fts_estimate_word_docs(m_prebuilt->trx, index, &w, + &n_docs); + } + + if (err != DB_SUCCESS) { + /* Includes DB_RECORD_NOT_FOUND, which fts_estimate_word_docs() + returns for an empty auxiliary table: nothing has been SYNCed + yet, so we know nothing at all. */ + DBUG_RETURN(HA_POS_ERROR); + } + + /* The auxiliary tables still hold entries for rows that were deleted + or updated since the last OPTIMIZE TABLE, so the count can exceed the + number of rows in the table. */ + if (ft_table->stat_initialized()) { + const uint64_t n_rows = dict_table_get_n_rows(ft_table); + + if (n_docs > n_rows) { + n_docs = n_rows; + } + } + + /* Never report 0. The in-memory FTS cache is deliberately not + consulted, so "absent from the auxiliary table" does not mean "no + matching rows", and callers may treat 0 as provably empty. */ + if (!n_docs) { + n_docs = 1; + } + + /* HA_ROWS_MAX is HA_POS_ERROR, the "not known" value, so stay below + it. */ + DBUG_RETURN((ha_rows) std::min(n_docs, HA_ROWS_MAX - 1)); +} + /*********************************************************************//** Gives an UPPER BOUND to the number of rows in a table. This is used in filesort.cc. diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index d47a52e5061ea..08a61651adacd 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -196,6 +196,11 @@ class ha_innobase final : public handler const key_range* max_key, page_range* pages) override; + ha_rows fulltext_estimate( + uint index_nr, + const char* word, + uint word_len) override; + ha_rows estimate_rows_upper_bound() override; void update_create_info(HA_CREATE_INFO* create_info) override; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 32e4d2cb1e887..20ddc5d4bfd14 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -537,6 +537,41 @@ fts_query( fts_result_t** result) MY_ATTRIBUTE((warn_unused_result)); +/** Estimate the number of documents that contain a single word, by probing +the FTS auxiliary INDEX_[1..6] table that holds it. + +This is cheap, in the spirit of records_in_range(): one B-tree dive plus a +bounded walk over the leaf pages that dive already latched. It takes no +record locks, opens no read view and creates no transaction. It is therefore +deliberately approximate: + + - the in-memory FTS cache is NOT consulted, so a word that has been inserted + but not yet SYNCed is reported as absent even though rows do match; + - FTS_..._DELETED and DELETED_CACHE are NOT consulted, so documents that + were deleted or updated since the last OPTIMIZE TABLE are still counted; + - delete-marked auxiliary records are counted, exactly the way + records_in_range() counts delete-marked index records; + - the word is matched literally: no wildcards, no stemming, and no stopword + or token length filtering. + +@param[in] trx transaction to attribute buffer pool statistics to; it + is neither started, modified nor committed +@param[in] index fulltext index (index->type & DICT_FTS) +@param[in] word word to look up, in index's charset, already folded to + lower case unless my_binary_compare(charset) +@param[out] n_docs estimated number of matching documents; 0 means the + word is not present in the auxiliary table +@return DB_SUCCESS, DB_TABLE_NOT_FOUND if the auxiliary table cannot be +opened, DB_RECORD_NOT_FOUND if it is empty (nothing has been SYNCed yet, so +there is no information at all), or DB_CORRUPTION */ +dberr_t +fts_estimate_word_docs( + trx_t* trx, + dict_index_t* index, + const fts_string_t* word, + uint64_t* n_docs) + noexcept MY_ATTRIBUTE((nonnull, warn_unused_result)); + /******************************************************************//** Retrieve the FTS Relevance Ranking result for doc with doc_id @return the relevance ranking value. */ From b41e91073bc365eb22c26801524172a14b4afff7 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 08:49:15 +0300 Subject: [PATCH 12/39] Estimate the number of records an MVI access will read QUICK_MVI_SELECT carried records=10 and read_time=0.001, numbers picked low enough that the access always won over a table scan. Ask the engine instead, through the fulltext_estimate() added by the previous commit. Mvi_access::estimate_records() estimates one element key at a time and combines the answers the way the query combines the keys: - A disjunctive access (JSON_OVERLAPS) reads the rows of every key, so the estimates add up. - A conjunctive access (JSON_CONTAINS) reads the rows that have all of the keys, so the rarest key alone bounds the result. We use its estimate and drop the other keys from the query: reading the rarest key and letting the WHERE clause discard the rest is not worse than having the engine intersect the terms. This is the trade-off collect_mvi_keys() already makes for the keys it cannot encode - a shorter AND matches a superset of the rows, and the JSON predicate does the exact filtering. The engine may be unable to estimate a key: ha_innobase only looks at the fulltext auxiliary tables, so until the words are flushed out of the FTS cache the answer is "unknown" for everything. Such a key takes no part in the choice of the rarest one, and if not a single key could be estimated the old guess stands and the query is left as it is. For an OR we cannot do that: we have to read that key and have no idea what it costs. Give the access a DBL_MAX read_time and do not use it. That has to be acted on in get_best_mvi_access() rather than left to the cost comparison, because best_access_path() takes a quick select to be cheaper than a table scan without checking - true of anything the range optimizer proposes, but this access does not come from there. read_time is now the cost of reading the estimated rows plus evaluating the WHERE clause on them, which is what the join optimizer expects of a quick select's read_time. It does not account for the fulltext search that produces the rowids in the first place. The trace prints the estimate, or says the access is unusable and why. The two existing tests ran on tables whose words were still in the FTS cache, so the JSON_OVERLAPS sections would have stopped using the index entirely. They now flush the cache with OPTIMIZE TABLE under innodb_optimize_fulltext_only, the way innodb_fts.estimate does. The trace test gets a table with a skewed distribution ("bbb" in every row, "aaa" in one) to show the conjunctive access keeping only the rarest key and still producing the same rows as the same query with IGNORE INDEX. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 15 ++- mysql-test/main/multi_valued_index.test | 8 ++ .../multi_valued_index_notembedded.result | 100 ++++++++++++++- .../main/multi_valued_index_notembedded.test | 50 ++++++++ sql/opt_multi_valued_index.cc | 120 +++++++++++++++++- sql/opt_multi_valued_index.h | 18 ++- 6 files changed, 299 insertions(+), 12 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 9cd704c81ed47..661b43d3e361b 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -171,9 +171,18 @@ key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), (5,'{"tags": [123]}'),(6,'{}'); +# An ORed access has to read every one of its keys, so we only use it +# when the engine can estimate all of them. The estimate is taken from +# the fulltext auxiliary tables, so flush the FTS cache into them. +set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; +set global innodb_optimize_fulltext_only=1; +optimize table t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +set global innodb_optimize_fulltext_only=@old_fulltext_only; explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 4 Using where select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; c j 1 {"tags": ["aaa"]} @@ -189,7 +198,7 @@ c j # the indexed expression as the second argument explain select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 4 Using where select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags') order by c; c j 1 {"tags": ["aaa"]} @@ -220,7 +229,7 @@ c j # only makes the scan less selective, so it stays a superset. explain select * from t1 where json_contains(j->'$.tags','[123,"aaa"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 2 Using where select * from t1 where json_contains(j->'$.tags','[123,"aaa"]') order by c; c j select * from t1 ignore index(idx) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 85cb9bd674701..9dff45e758ac2 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -119,6 +119,14 @@ insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), (5,'{"tags": [123]}'),(6,'{}'); +--echo # An ORed access has to read every one of its keys, so we only use it +--echo # when the engine can estimate all of them. The estimate is taken from +--echo # the fulltext auxiliary tables, so flush the FTS cache into them. +set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; +set global innodb_optimize_fulltext_only=1; +optimize table t1; +set global innodb_optimize_fulltext_only=@old_fulltext_only; + explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; --echo # the same rows without the index: diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result index 3092da52ead76..d1ae73486e60d 100644 --- a/mysql-test/main/multi_valued_index_notembedded.result +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -24,6 +24,8 @@ jd "table": "t1", "index": "idx", "match": "all", + "rows": 10, + "cost": 0.001, "ranges": ["616161"] } @@ -42,6 +44,8 @@ jd "table": "t1", "index": "idx", "match": "all", + "rows": 10, + "cost": 0.001, "ranges": [ "626262", @@ -68,6 +72,8 @@ jd "table": "t1", "index": "idx", "match": "all", + "rows": 10, + "cost": 0.001, "ranges": ["616161"] }, @@ -75,6 +81,8 @@ jd "table": "t2", "index": "idx2", "match": "all", + "rows": 10, + "cost": 0.001, "ranges": ["7a7a7a"] } @@ -84,7 +92,7 @@ jd # explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 ALL idx NULL NULL NULL 2 Using where select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; jd @@ -93,6 +101,8 @@ jd "table": "t1", "index": "idx", "match": "any", + "usable": false, + "cause": "the engine cannot estimate one of the keys", "ranges": [ "626262", @@ -101,6 +111,94 @@ jd } ] # +# Everything above ran on a table whose words are still in the FTS +# cache, where the engine cannot estimate them: the conjunctive +# accesses fell back to a guess, and the disjunctive one was dropped. +# Flush the words into the auxiliary tables and the estimates are real. +# +create table t3 (c int, j json, +key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; +set global innodb_optimize_fulltext_only=1; +optimize table t3; +Table Op Msg_type Msg_text +test.t3 optimize status OK +set global innodb_optimize_fulltext_only=@old_fulltext_only; +select c, mvi_encode(j->'$.tags', char(6)) from t3; +c mvi_encode(j->'$.tags', char(6)) +1 616161 626262 +2 626262 +3 626262 +4 626262 636363 +# +# "bbb" is in every row and "aaa" in one. A row has to have both, so +# the search for "aaa" alone already reads every row we may return: +# only that key is left in "ranges", and the JSON predicate in the +# WHERE clause discards whatever else the shorter search finds. +# +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t3 range idx3 idx3 0 NULL 1 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t3", + "index": "idx3", + "match": "all", + "rows": 1, + "cost": 0.00171364, + "ranges": + ["616161"] + } +] +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +# +# A row matches an OR through any of its keys, so the estimates add up +# and all the keys stay in the query. +# +explain select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t3 range idx3 idx3 0 NULL 2 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t3", + "index": "idx3", + "match": "any", + "rows": 2, + "cost": 0.00260808, + "ranges": + [ + "616161", + "636363" + ] + } +] +select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +4 {"tags": ["bbb","ccc"]} +# the same rows without the index: +select * from t3 ignore index(idx3) +where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +4 {"tags": ["bbb","ccc"]} +drop table t3; +# # A predicate under a top-level OR gives no access, so nothing is printed # explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test index 529ad180171f3..304a7f2b80f35 100644 --- a/mysql-test/main/multi_valued_index_notembedded.test +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -60,6 +60,56 @@ select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; --enable_view_protocol +--echo # +--echo # Everything above ran on a table whose words are still in the FTS +--echo # cache, where the engine cannot estimate them: the conjunctive +--echo # accesses fell back to a guess, and the disjunctive one was dropped. +--echo # Flush the words into the auxiliary tables and the estimates are real. +--echo # +create table t3 (c int, j json, + key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; +set global innodb_optimize_fulltext_only=1; +optimize table t3; +set global innodb_optimize_fulltext_only=@old_fulltext_only; +select c, mvi_encode(j->'$.tags', char(6)) from t3; + +--echo # +--echo # "bbb" is in every row and "aaa" in one. A row has to have both, so +--echo # the search for "aaa" alone already reads every row we may return: +--echo # only that key is left in "ranges", and the JSON predicate in the +--echo # WHERE clause discards whatever else the shorter search finds. +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +--echo # the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; + +--echo # +--echo # A row matches an OR through any of its keys, so the estimates add up +--echo # and all the keys stay in the query. +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol +select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; +--echo # the same rows without the index: +select * from t3 ignore index(idx3) +where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; + +drop table t3; + --echo # --echo # A predicate under a top-level OR gives no access, so nothing is printed --echo # diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index e3cd33e182ab2..02cca3fd56939 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -290,6 +290,99 @@ bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) } +/* + @brief + Estimate how many records this access will read, and simplify the access + if that lets us read fewer. + + @detail + The engine gives us an estimate for one element key at a time (the + fulltext analogue of records_in_range()). We combine the estimates the + way the query combines the keys: + + - Disjunctive access (JSON_OVERLAPS) reads the rows of every key, so the + estimates add up. A key the engine cannot estimate leaves us with no + idea of what the scan costs, and we cannot leave that key out: dropping + it from an OR loses the rows that only have that key. Price the access + out of the plan instead. + + - Conjunctive access (JSON_CONTAINS) reads the rows that have all of the + keys, so the rarest key alone bounds the result. Use its estimate, and + drop the other keys from the query: reading the rarest key and letting + the WHERE clause discard the rest is not worse than having the engine + intersect the terms. This is the trade-off collect_mvi_keys() already + makes for the keys it cannot encode - a shorter AND matches a superset + of the rows, and the JSON predicate does the exact filtering. + Keys the engine cannot estimate take no part in the choice. If it could + not estimate a single one of them, we know nothing: keep the query as + it is and fall back to a guess low enough that the access is still + preferred over a table scan. + + TODO: read_time only accounts for reading the rows, not for the fulltext + search that produces their rowids. +*/ + +void Mvi_access::estimate_records() +{ + TABLE *table= index->vcol->table; + handler *file= table->file; + List_iterator it(encoded); + String *key, *rarest= NULL; + ha_rows sum= 0, min_rows= 0; + bool unknown= false; + + while ((key= it++)) + { + ha_rows rows= file->fulltext_estimate(index->keyno, key->ptr(), + (uint) key->length()); + if (rows == HA_POS_ERROR) + { + unknown= true; + continue; + } + sum+= rows; + if (!rarest || rows < min_rows) + { + min_rows= rows; + rarest= key; + } + } + + if (!conjunctive && unknown) + { + /* We have to read this key and have no idea what that costs */ + records= table->stat_records(); + read_time= DBL_MAX; + return; + } + if (conjunctive && !rarest) + { + /* Nothing was estimated. Keep the old guess and the query as it is */ + records= 10; + read_time= 0.001; + return; + } + + if (conjunctive) + { + /* Search for the rarest key only */ + it.rewind(); + while ((key= it++)) + { + if (key != rarest) + it.remove(); + } + records= min_rows; + } + else + records= sum; + + set_if_smaller(records, table->stat_records()); + set_if_bigger(records, (ha_rows) 1); + read_time= file->cost(file->ha_rnd_pos_call_and_compare_time(records)); +} + + /* @brief Build the boolean-mode fulltext query to find rows of interest. @@ -339,6 +432,11 @@ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) String *key; trace_object->add("index", key_info->name). add("match", conjunctive ? "all" : "any"); + if (cost_is_known()) + trace_object->add("rows", records).add("cost", read_time); + else + trace_object->add("usable", false). + add("cause", "the engine cannot estimate one of the keys"); Json_writer_array trace_ranges(thd, "ranges"); while ((key= it++)) trace_ranges.add(key->ptr(), key->length()); @@ -458,6 +556,12 @@ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) Mvi_access *access= join->get_mvi_access_for_table(table); if (!access) return NULL; + /* + We are called once per table for each of the two range analysis passes. + Probe the engine (and drop the keys we don't need) only on the first one. + */ + if (access->records == HA_POS_ERROR) + access->estimate_records(); if (unlikely(thd->trace_started())) { /* @@ -470,6 +574,13 @@ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) trace_mvi.add_table_name(table); access->print_json(thd, &trace_mvi); } + /* + best_access_path() takes a quick select to be cheaper than a table scan + without checking (the range optimizer only proposes a quick when it is), + so an access we could not put a price on has to be dropped here. + */ + if (!access->cost_is_known()) + return NULL; return new QUICK_MVI_SELECT(thd, table, access); } @@ -485,13 +596,8 @@ QUICK_MVI_SELECT::QUICK_MVI_SELECT(THD *thd, TABLE *table, head= table; index= access->index->keyno; record= head->record[0]; - /* - TODO: get a real estimate from the engine (see fulltext_estimate()). - Until then, use numbers low enough that the MVI scan is preferred over a - table scan. - */ - records= 10; - read_time= 0.001; + records= access->records; + read_time= access->read_time; } diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 9076f392aedc2..bacdb0c6fc2af 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -31,11 +31,27 @@ struct Mvi_access : public Sql_alloc Mv_index *index; List encoded; /* encoded element keys */ bool conjunctive; /* CONTAINS -> AND, OVERLAPS -> OR */ - Mvi_access(Mv_index *idx, bool conj) : index(idx), conjunctive(conj) {} + /* + The estimate for this access, produced by estimate_records(). + HA_POS_ERROR means we haven't estimated it yet. + */ + ha_rows records; + double read_time; + Mvi_access(Mv_index *idx, bool conj) + : index(idx), conjunctive(conj), records(HA_POS_ERROR), read_time(0.0) {} /* Build: Add one encoded element key */ bool add_key(MEM_ROOT *mem_root, const String *key); + /* Usage: Estimate how many records this access will read */ + void estimate_records(); + + /* + Usage: false when estimate_records() could not put a price on the access. + Such an access must not be used: we have no idea what it costs. + */ + bool cost_is_known() const { return read_time != DBL_MAX; } + /* Usage: Build the fulltext query searching for the element keys */ bool build_ft_query(String *out); From d6fbb2c4e6ae0f501f12c8907da25b7383b32dc8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 09:06:54 +0300 Subject: [PATCH 13/39] MDEV-40168: JSON-over-fulltext: let the estimate consult the FTS cache. fts_estimate_word_docs() probed only the on-disk auxiliary INDEX_[1..6] table. Documents inserted but not SYNCed yet are only in the in-memory FTS cache, so they were missed entirely, and on a table that has never been SYNCed the auxiliary table is empty and the estimate was simply "unknown". Look in the cache as well. fts_index_cache_t::words is an rb tree of fts_tokenizer_word_t, and fts_node_t::doc_count already holds the number of documents in the node's ilist, so this is one rbt_search plus a walk over a short vector: no ilist decoding and no I/O. Three details: - the cache mutex is taken with trylock. This runs during optimization, where a SYNC holding cache->lock across SQL execution would stall the optimizer; dropping the cache contribution is the better trade. - nodes flagged fts_node_t::synced are skipped. An in-flight SYNC has already written them out, and the estimator reads the B-tree without a read view, so it sees those records; counting the node too would count its documents twice. - index_cache->words is NULL between fts_cache_clear() and fts_cache_init(). The query path never observes that because it runs after fts_init_index(); the estimate does not run it. DB_RECORD_NOT_FOUND now means both sources are empty. The cache on its own cannot prove a word absent, because it is only complete once fts_init_index() has run, and an estimate must have no side effects. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/suite/innodb_fts/r/estimate.result | 76 +++++++++++++++- .../suite/innodb_fts/r/estimate_debug.result | 47 ++++++++++ mysql-test/suite/innodb_fts/t/estimate.test | 48 +++++++++- .../suite/innodb_fts/t/estimate_debug.test | 63 +++++++++++++ storage/innobase/fts/fts0fts.cc | 90 ++++++++++++++++++- storage/innobase/handler/ha_innodb.cc | 7 +- storage/innobase/include/fts0fts.h | 31 ++++--- 7 files changed, 338 insertions(+), 24 deletions(-) create mode 100644 mysql-test/suite/innodb_fts/r/estimate_debug.result create mode 100644 mysql-test/suite/innodb_fts/t/estimate_debug.test diff --git a/mysql-test/suite/innodb_fts/r/estimate.result b/mysql-test/suite/innodb_fts/r/estimate.result index 598dd13442f89..39565fdce9de4 100644 --- a/mysql-test/suite/innodb_fts/r/estimate.result +++ b/mysql-test/suite/innodb_fts/r/estimate.result @@ -10,16 +10,37 @@ INSERT INTO t1 VALUES (4,'gamma'), (5,'gamma'), (6,'gamma'), (7,'gamma'), (8,'gamma'); # # Nothing has been SYNCed yet, so the words live only in the in-memory -# FTS cache, which the estimate deliberately does not consult. The -# auxiliary table is still empty, which means we have no information at -# all rather than "no matching rows", so the answer is "unknown". +# FTS cache. The estimate consults it, and since every cache node +# carries its own doc_count the answer is exact. # SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('beta')= 3 SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); COUNT(*) 8 Warnings: -Note 1105 fulltext_estimate('gamma')= unknown +Note 1105 fulltext_estimate('gamma')= 8 +# +# A word that is in neither the cache nor the auxiliary table, while the +# auxiliary table is still empty: that is no information at all rather +# than "no matching rows", so the answer is "unknown". The cache on its +# own cannot prove a word absent, because it is only complete once +# fts_init_index() has run, and an estimate must not run it. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('zzzzz')= unknown SET debug_dbug=''; # # Flush the cache into the auxiliary tables. @@ -78,6 +99,36 @@ alpha 1 beta 3 gamma 8 # +# Rows inserted after the SYNC are back in the cache, while the older +# ones are in the auxiliary table. The two counts are added, and no +# document is counted twice: gamma is 8 + 2, beta 3 + 1, alpha 1 + 0. +# +INSERT INTO t1 VALUES (9,'beta gamma'), (10,'gamma'); +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +4 +Warnings: +Note 1105 fulltext_estimate('beta')= 4 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +10 +Warnings: +Note 1105 fulltext_estimate('gamma')= 10 +SET debug_dbug=''; +# +# Back to the state the rest of the test expects. +# +DELETE FROM t1 WHERE id > 8; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +# # Deleted rows are deliberately still counted: their entries survive in # the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 # is then clamped to the number of rows left in the table, so 4 is @@ -106,3 +157,20 @@ Note 1105 fulltext_estimate('gamma')= unknown SET debug_dbug=''; DROP TABLE t1; SET GLOBAL innodb_optimize_fulltext_only= @optimize; +# +# A restart empties the cache, and the estimate does not repopulate it: +# fts_init_index() has side effects that an estimate must not have. A +# clean shutdown has SYNCed the words, so the auxiliary table answers on +# its own. +# +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1,'delta'), (2,'delta'), (3,'delta'); +# restart +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('delta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('delta')= 3 +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/r/estimate_debug.result b/mysql-test/suite/innodb_fts/r/estimate_debug.result new file mode 100644 index 0000000000000..315ba583b1df2 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate_debug.result @@ -0,0 +1,47 @@ +CREATE TABLE t1 ( +id INT AUTO_INCREMENT PRIMARY KEY, +a VARCHAR(64), +FULLTEXT(a) +) ENGINE=InnoDB; +INSERT INTO t1(a) SELECT 'alpha' FROM seq_1_to_40; +INSERT INTO t1(a) SELECT 'beta' FROM seq_1_to_60; +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connection con1; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET DEBUG_SYNC='fts_write_node SIGNAL written WAIT_FOR go'; +INSERT INTO t1(a) VALUES('zeta'); +connection con2; +SET DEBUG_SYNC='now WAIT_FOR written'; +# +# alpha is now in the auxiliary table and still in the cache. 40, not +# 80. +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +40 +Warnings: +Note 1105 fulltext_estimate('alpha')= 40 +# +# beta has not been written yet, so it is counted from the cache alone. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +60 +Warnings: +Note 1105 fulltext_estimate('beta')= 60 +SET debug_dbug=''; +connection default; +SET DEBUG_SYNC='now SIGNAL go'; +connection con1; +SET DEBUG_SYNC='RESET'; +SET debug_dbug=''; +connection default; +disconnect con1; +disconnect con2; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/estimate.test b/mysql-test/suite/innodb_fts/t/estimate.test index db1a102fcc33a..2cb489bd1d82e 100644 --- a/mysql-test/suite/innodb_fts/t/estimate.test +++ b/mysql-test/suite/innodb_fts/t/estimate.test @@ -23,12 +23,22 @@ INSERT INTO t1 VALUES --echo # --echo # Nothing has been SYNCed yet, so the words live only in the in-memory ---echo # FTS cache, which the estimate deliberately does not consult. The ---echo # auxiliary table is still empty, which means we have no information at ---echo # all rather than "no matching rows", so the answer is "unknown". +--echo # FTS cache. The estimate consults it, and since every cache node +--echo # carries its own doc_count the answer is exact. --echo # SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); + +--echo # +--echo # A word that is in neither the cache nor the auxiliary table, while the +--echo # auxiliary table is still empty: that is no information at all rather +--echo # than "no matching rows", so the answer is "unknown". The cache on its +--echo # own cannot prove a word absent, because it is only complete once +--echo # fts_init_index() has run, and an estimate must not run it. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); SET debug_dbug=''; --echo # @@ -66,6 +76,24 @@ SET GLOBAL innodb_ft_aux_table='test/t1'; SELECT DISTINCT WORD, DOC_COUNT FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE ORDER BY WORD; +--echo # +--echo # Rows inserted after the SYNC are back in the cache, while the older +--echo # ones are in the auxiliary table. The two counts are added, and no +--echo # document is counted twice: gamma is 8 + 2, beta 3 + 1, alpha 1 + 0. +--echo # +INSERT INTO t1 VALUES (9,'beta gamma'), (10,'gamma'); +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +--echo # +--echo # Back to the state the rest of the test expects. +--echo # +DELETE FROM t1 WHERE id > 8; +OPTIMIZE TABLE t1; + --echo # --echo # Deleted rows are deliberately still counted: their entries survive in --echo # the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 @@ -89,3 +117,17 @@ SET debug_dbug=''; DROP TABLE t1; SET GLOBAL innodb_optimize_fulltext_only= @optimize; + +--echo # +--echo # A restart empties the cache, and the estimate does not repopulate it: +--echo # fts_init_index() has side effects that an estimate must not have. A +--echo # clean shutdown has SYNCed the words, so the auxiliary table answers on +--echo # its own. +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1,'delta'), (2,'delta'), (3,'delta'); +--source include/restart_mysqld.inc +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('delta'); +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/t/estimate_debug.test b/mysql-test/suite/innodb_fts/t/estimate_debug.test new file mode 100644 index 0000000000000..d3db1d0413900 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_debug.test @@ -0,0 +1,63 @@ +# +# handler::fulltext_estimate() while a SYNC is half way through writing the +# cache out to the auxiliary table. +# +# A node that SYNC has already written is flagged fts_node_t::synced but stays +# in the cache until the SYNC ends. The estimate reads the auxiliary B-tree +# without a read view, so it sees the record that was just written; it must +# skip the node as well, or it counts those documents twice. +# +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +CREATE TABLE t1 ( + id INT AUTO_INCREMENT PRIMARY KEY, + a VARCHAR(64), + FULLTEXT(a) +) ENGINE=InnoDB; + +# alpha in 40 rows, beta in the other 60. Neither is in a majority of the +# table, so a doubled count would still be below the row count and would not +# be hidden by the clamp in ha_innobase::fulltext_estimate(). +INSERT INTO t1(a) SELECT 'alpha' FROM seq_1_to_40; +INSERT INTO t1(a) SELECT 'beta' FROM seq_1_to_60; +ANALYZE TABLE t1; + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); + +connection con1; +# The words are visited in rb tree order, so the first node SYNC writes is +# alpha's, and that is where it stops. +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET DEBUG_SYNC='fts_write_node SIGNAL written WAIT_FOR go'; +send INSERT INTO t1(a) VALUES('zeta'); + +connection con2; +SET DEBUG_SYNC='now WAIT_FOR written'; +--echo # +--echo # alpha is now in the auxiliary table and still in the cache. 40, not +--echo # 80. +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +--echo # +--echo # beta has not been written yet, so it is counted from the cache alone. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SET debug_dbug=''; + +connection default; +SET DEBUG_SYNC='now SIGNAL go'; + +connection con1; +--reap +SET DEBUG_SYNC='RESET'; +SET debug_dbug=''; + +connection default; +disconnect con1; +disconnect con2; +DROP TABLE t1; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 5c20b4486d796..6e6b37ecfca30 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -1263,6 +1263,70 @@ fts_est_probe_aux( return(err); } +/** Count the documents that contain a word and are still only in the +in-memory FTS cache, that is, have not been SYNCed to the auxiliary table yet. + +fts_node_t::doc_count already holds the number we want, so this decodes no +ilist and reads nothing from disk; it is one rb tree lookup. + +Nodes flagged as synced are skipped. An in-flight SYNC has already written +them to the auxiliary table, and fts_est_probe_aux() reads the B-tree without +a read view, so it sees those records; counting the node as well would count +its documents twice. + +@param[in] index fulltext index +@param[in] word word to look up, folded the same way the caller folds it + for the auxiliary table +@return number of matching documents found in the cache, or 0 if there are +none, if the cache is not initialized, or if another thread holds the cache +mutex (an estimate is not worth waiting for a SYNC to finish) */ +static +uint64_t +fts_est_cache_docs( + const dict_index_t* index, + const fts_string_t* word) noexcept +{ + const fts_t* fts = index->table->fts; + + if (!fts || !fts->cache) { + return(0); + } + + fts_cache_t* cache = fts->cache; + + if (mysql_mutex_trylock(&cache->lock)) { + return(0); + } + + uint64_t n_docs = 0; + + if (const fts_index_cache_t* index_cache + = fts_find_index_cache(cache, index)) { + /* fts_cache_clear() leaves words NULL until the following + fts_cache_init(). The query path never observes that, because + it runs after fts_init_index(); we may. */ + if (index_cache->words) { + const ib_vector_t* nodes + = fts_cache_find_word(index_cache, word); + + for (ulint i = 0; nodes && i < ib_vector_size(nodes); + ++i) { + const fts_node_t* node + = static_cast( + ib_vector_get_const(nodes, i)); + + if (!node->synced) { + n_docs += node->doc_count; + } + } + } + } + + mysql_mutex_unlock(&cache->lock); + + return(n_docs); +} + dberr_t fts_estimate_word_docs( trx_t* trx, @@ -1276,6 +1340,11 @@ fts_estimate_word_docs( *n_docs = 0; + /* Documents that have been inserted but not SYNCed yet are only in the + memory cache. Look there first: it costs no I/O, and on a table that + has never been SYNCed it is the only information there is. */ + const uint64_t cached = fts_est_cache_docs(index, word); + /* A word lives in exactly one of INDEX_1..INDEX_6, so only that one auxiliary table is ever opened -- unlike the query path, which opens all six. */ @@ -1295,13 +1364,32 @@ fts_estimate_word_docs( aux_name, false, DICT_ERR_IGNORE_TABLESPACE); if (!aux) { + if (cached) { + *n_docs = cached; + return(DB_SUCCESS); + } + return(DB_TABLE_NOT_FOUND); } - const dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); + dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); aux->release(); + if (err == DB_SUCCESS) { + /* The two populations are disjoint: fts_est_cache_docs() + skipped every node that the auxiliary table already holds. */ + *n_docs += cached; + } else if (cached && err == DB_RECORD_NOT_FOUND) { + /* The auxiliary table is empty, but the cache is not, so we + are no longer without information. The other way round -- + nothing in either -- stays DB_RECORD_NOT_FOUND: the cache is + only complete once fts_init_index() has run, and an estimate + must not run it. */ + *n_docs = cached; + err = DB_SUCCESS; + } + return(err); } /****************************************************************//** diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index cc31ba8f9c28a..2178c2b0d24d0 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14855,9 +14855,10 @@ ha_innobase::fulltext_estimate( } } - /* Never report 0. The in-memory FTS cache is deliberately not - consulted, so "absent from the auxiliary table" does not mean "no - matching rows", and callers may treat 0 as provably empty. */ + /* Never report 0. The in-memory FTS cache is consulted only on a best + effort basis (see fts_estimate_word_docs()), so "found nowhere" does + not mean "no matching rows", and callers may treat 0 as provably + empty. */ if (!n_docs) { n_docs = 1; } diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 20ddc5d4bfd14..0ed3e25d65b4f 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -538,15 +538,18 @@ fts_query( MY_ATTRIBUTE((warn_unused_result)); /** Estimate the number of documents that contain a single word, by probing -the FTS auxiliary INDEX_[1..6] table that holds it. - -This is cheap, in the spirit of records_in_range(): one B-tree dive plus a -bounded walk over the leaf pages that dive already latched. It takes no -record locks, opens no read view and creates no transaction. It is therefore -deliberately approximate: - - - the in-memory FTS cache is NOT consulted, so a word that has been inserted - but not yet SYNCed is reported as absent even though rows do match; +the FTS auxiliary INDEX_[1..6] table that holds it and the in-memory FTS +cache. + +This is cheap, in the spirit of records_in_range(): one rb tree lookup in the +cache, plus one B-tree dive and a bounded walk over the leaf pages that dive +already latched. It takes no record locks, opens no read view and creates no +transaction. It is therefore deliberately approximate: + + - the cache is consulted only on a best effort basis: it is skipped when + another thread holds cache->lock, and it is incomplete until + fts_init_index() has run, which an estimate must not do. A word can + therefore still be reported as absent even though rows do match; - FTS_..._DELETED and DELETED_CACHE are NOT consulted, so documents that were deleted or updated since the last OPTIMIZE TABLE are still counted; - delete-marked auxiliary records are counted, exactly the way @@ -560,10 +563,12 @@ deliberately approximate: @param[in] word word to look up, in index's charset, already folded to lower case unless my_binary_compare(charset) @param[out] n_docs estimated number of matching documents; 0 means the - word is not present in the auxiliary table -@return DB_SUCCESS, DB_TABLE_NOT_FOUND if the auxiliary table cannot be -opened, DB_RECORD_NOT_FOUND if it is empty (nothing has been SYNCed yet, so -there is no information at all), or DB_CORRUPTION */ + word is present neither in the auxiliary table nor in + the cache +@return DB_SUCCESS, DB_TABLE_NOT_FOUND if the auxiliary table cannot be opened +and the cache knows nothing either, DB_RECORD_NOT_FOUND if both the auxiliary +table and the cache are empty (so there is no information at all), or +DB_CORRUPTION */ dberr_t fts_estimate_word_docs( trx_t* trx, From 6351aaf5183947f19f2b371a11e359911e67ba49 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 09:26:09 +0300 Subject: [PATCH 14/39] MDEV-40168: JSON-over-fulltext: move the estimator into fts0est.cc. Pure code motion, no functional change. Co-Authored-By: Claude Opus 5 (1M context) --- storage/innobase/CMakeLists.txt | 1 + storage/innobase/fts/fts0est.cc | 545 ++++++++++++++++++++++++++++++++ storage/innobase/fts/fts0fts.cc | 504 ----------------------------- 3 files changed, 546 insertions(+), 504 deletions(-) create mode 100644 storage/innobase/fts/fts0est.cc diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index c1b4a5c9559fc..b9465456d0027 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -173,6 +173,7 @@ SET(INNOBASE_SOURCES fts/fts0blex.cc fts/fts0config.cc fts/fts0exec.cc + fts/fts0est.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc diff --git a/storage/innobase/fts/fts0est.cc b/storage/innobase/fts/fts0est.cc new file mode 100644 index 0000000000000..e1c0b334c513b --- /dev/null +++ b/storage/innobase/fts/fts0est.cc @@ -0,0 +1,545 @@ +/***************************************************************************** + +Copyright (c) 2026, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0est.cc +Estimating the size of a fulltext search result. + +fts_estimate_word_docs() is the fulltext analogue of records_in_range(): it +answers "how many documents contain this word?" cheaply enough to be called +while the optimizer is still choosing a plan. It probes the auxiliary +INDEX_[1..6] B-tree that holds the word and the in-memory FTS cache, takes no +record locks, opens no read view, creates no transaction, and never reads an +ilist. + +Created 2026/09/06 +*******************************************************/ + +#include "fts0fts.h" +#include "fts0priv.h" +#include "fts0types.h" +#include "fts0types.inl" +#include "dict0dict.h" +#include "btr0pcur.h" +#include "mtr0mtr.h" +#include "rem0cmp.h" +#include "page0page.h" + +/** Maximum number of auxiliary records fts_estimate_word_docs() samples +before it gives up on an exact answer and extrapolates instead. */ +static constexpr uint32_t FTS_EST_MAX_RECS = 64; +/** Maximum number of auxiliary leaf pages fts_estimate_word_docs() reads. */ +static constexpr uint32_t FTS_EST_MAX_PAGES = 4; + +/* Physical field numbers of an FTS auxiliary INDEX_[1..6] record. The +clustered index is UNIQUE(word, first_doc_id), so the record is +(word, first_doc_id, DB_TRX_ID, DB_ROLL_PTR, last_doc_id, doc_count, ilist); +see fts_create_one_index_table(). */ +static constexpr ulint FTS_AUX_FLD_WORD = 0; +static constexpr ulint FTS_AUX_FLD_FIRST_DOC_ID = 1; +static constexpr ulint FTS_AUX_FLD_LAST_DOC_ID = 4; +static constexpr ulint FTS_AUX_FLD_DOC_COUNT = 5; +/** Number of offsets the estimator needs. This deliberately stops short of +the ilist (field 6), which may be stored off-page: the estimator must never +read a BLOB. */ +static constexpr ulint FTS_AUX_EST_N_FIELDS = FTS_AUX_FLD_DOC_COUNT + 1; + +/** Build the search tuple (word) for an FTS auxiliary clustered index. +The index key is (word, first_doc_id), and because the tuple compares on its +first field only it compares equal to every record of the word. So +PAGE_CUR_GE positions on the word's first record and PAGE_CUR_LE on its last. +@param[in] heap heap to allocate the tuple from +@param[in] aux_index auxiliary clustered index +@param[in] word word to search for +@return the search tuple */ +static +dtuple_t* +fts_est_word_tuple( + mem_heap_t* heap, + dict_index_t* aux_index, + const fts_string_t* word) noexcept +{ + dtuple_t* tuple = dtuple_create(heap, 1); + + dict_index_copy_types(tuple, aux_index, 1); + dfield_set_data(dtuple_get_nth_field(tuple, 0), + word->f_str, word->f_len); + dtuple_set_n_fields_cmp(tuple, 1); + + return(tuple); +} + +/** Read the fields the estimator needs out of an FTS auxiliary INDEX_[1..6] +leaf record. Never touches the ilist. +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in] rec auxiliary table record +@param[in] aux_index auxiliary clustered index +@param[in] offsets rec_get_offsets(rec, aux_index, ...) +@param[out] first_doc_id first doc id in this record's ilist +@param[out] last_doc_id last doc id in this record's ilist +@param[out] doc_count number of doc ids in this record's ilist +@return whether the record belongs to the word tuple was built for */ +static +bool +fts_est_read_rec( + const dtuple_t* tuple, + const rec_t* rec, + const dict_index_t* aux_index, + const rec_offs* offsets, + doc_id_t* first_doc_id, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + /* tuple has n_fields_cmp == 1, so this compares the word only, in the + collation of the auxiliary table's word column. */ + if (cmp_dtuple_rec(tuple, rec, aux_index, offsets)) { + return(false); + } + + ulint len; + const byte* data; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_FIRST_DOC_ID, &len); + *first_doc_id = (data && len == sizeof *first_doc_id) + ? fts_read_doc_id(data) : 0; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_LAST_DOC_ID, &len); + *last_doc_id = (data && len == sizeof *last_doc_id) + ? fts_read_doc_id(data) : *first_doc_id; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_DOC_COUNT, &len); + *doc_count = (data && len == 4) ? mach_read_from_4(data) : 0; + + return(true); +} + +/** What fts_est_sample_word() found. */ +struct fts_est_sample_t +{ + /** Sum of doc_count over the records sampled. */ + uint64_t docs; + /** Number of records sampled. */ + uint32_t n_recs; + /** First doc id of the word's first record. */ + doc_id_t first_doc_id; + /** Last doc id of the last record sampled. */ + doc_id_t last_doc_id; + /** Whether sampling stopped because a budget ran out rather than + because the word's key range ended. When this is false, docs is the + exact number of documents that contain the word. */ + bool truncated; +}; + +/** Dive to a word's first auxiliary record and walk forward from it, over at +most FTS_EST_MAX_RECS records and FTS_EST_MAX_PAGES leaf pages. The walk is +nearly free because the dive has already latched the leaf page, and if it +reaches a different word before a budget runs out the result is exact. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[out] out what the walk found +@return DB_SUCCESS, or DB_RECORD_NOT_FOUND if the auxiliary table is empty +(nothing has been SYNCed yet, so there is no information at all) */ +static +dberr_t +fts_est_sample_word( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + fts_est_sample_t* out) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + memset(out, 0, sizeof *out); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err != DB_SUCCESS) { + goto func_exit; + } + + if (!btr_pcur_is_on_user_rec(&pcur)) { + /* Nothing at or after the word. Tell an empty auxiliary + table, where we have no information at all, apart from a word + that simply sorts after everything that is indexed. */ + const page_t* page = btr_pcur_get_page(&pcur); + + if (!page_has_prev(page) && !page_get_n_recs(page)) { + err = DB_RECORD_NOT_FOUND; + } + + goto func_exit; + } + + { + uint32_t n_pages = 1; + page_id_t last_page + = btr_pcur_get_block(&pcur)->page.id(); + + do { + const rec_t* rec = btr_pcur_get_rec(&pcur); + const buf_block_t* block + = btr_pcur_get_block(&pcur); + const ulint offs + = ulint(rec - block->page.frame); + + if (page_rec_is_infimum_low(offs) + || page_rec_is_supremum_low(offs)) { + continue; + } + + if (block->page.id() != last_page) { + last_page = block->page.id(); + if (++n_pages > FTS_EST_MAX_PAGES) { + out->truncated = true; + break; + } + } + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + doc_id_t first; + doc_id_t last; + uint32_t doc_count; + + if (!fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &doc_count)) { + /* Walked past the word's key range, so we + have seen all of it. */ + break; + } + + if (!out->n_recs) { + out->first_doc_id = first; + } + + out->last_doc_id = last; + out->docs += doc_count; + + if (++out->n_recs >= FTS_EST_MAX_RECS) { + out->truncated = true; + break; + } + } while (btr_pcur_move_to_next(&pcur, &mtr)); + } + +func_exit: + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Dive to a word's last auxiliary record, to learn how far its doc ids +reach. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in,out] last_doc_id last doc id of the word; left alone if the + record cannot be read +@param[in,out] doc_count doc_count of that record; left alone if the + record cannot be read +@return DB_SUCCESS or error code */ +static +dberr_t +fts_est_last_rec( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err == DB_SUCCESS) { + /* PAGE_CUR_LE may leave the cursor on the page infimum, in + which case the record we want is the last one of the previous + page. */ + const bool positioned + = !btr_pcur_is_before_first_on_page(&pcur) + || btr_pcur_move_to_prev(&pcur, &mtr); + + if (positioned && btr_pcur_is_on_user_rec(&pcur)) { + const rec_t* rec = btr_pcur_get_rec(&pcur); + doc_id_t first; + doc_id_t last; + uint32_t count; + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + if (fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &count)) { + *last_doc_id = last; + *doc_count = count; + } + } + } + + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Estimate how many documents contain a word, by probing an already opened +FTS auxiliary INDEX_[1..6] table. See fts_estimate_word_docs(). +@param[in] trx transaction, used only for the mtr +@param[in] aux auxiliary table that holds the word +@param[in] word word to look up +@param[out] n_docs estimated number of matching documents +@return DB_SUCCESS, DB_RECORD_NOT_FOUND or DB_CORRUPTION */ +static +dberr_t +fts_est_probe_aux( + trx_t* trx, + dict_table_t* aux, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + dict_index_t* aux_index = dict_table_get_first_index(aux); + + if (!aux->space || !aux->is_readable() || !aux_index + || aux_index->page == FIL_NULL || aux_index->is_corrupted()) { + return(DB_CORRUPTION); + } + + mem_heap_t* heap = mem_heap_create(256); + const dtuple_t* tuple = fts_est_word_tuple(heap, aux_index, + word); + fts_est_sample_t s; + dberr_t err = fts_est_sample_word(trx, aux_index, + tuple, &s); + + if (err != DB_SUCCESS) { + mem_heap_free(heap); + return(err); + } + + if (!s.n_recs || !s.truncated) { + /* Either the word is absent, or the walk covered its whole + key range, in which case the count is exact. */ + *n_docs = s.docs; + mem_heap_free(heap); + return(DB_SUCCESS); + } + + /* The word has more records than we are willing to read. Dive once + more, to its last record, and extrapolate the density we measured + across the word's whole doc id span. Doc ids only ever increase, so + the records we sampled are a prefix of that span. */ + doc_id_t last_doc_id = s.last_doc_id; + uint32_t last_count = 0; + + err = fts_est_last_rec(trx, aux_index, tuple, &last_doc_id, + &last_count); + + if (err == DB_SUCCESS) { + if (last_doc_id < s.last_doc_id) { + last_doc_id = s.last_doc_id; + } + + const uint64_t sampled_span + = s.last_doc_id > s.first_doc_id + ? s.last_doc_id - s.first_doc_id + 1 : 1; + const uint64_t total_span + = last_doc_id - s.first_doc_id + 1; + + /* In double, because docs * total_span overflows 64 bits for + large inputs. The caller clamps the result to the number of + rows in the table. */ + uint64_t est = uint64_t(double(s.docs) + * double(total_span) + / double(sampled_span)); + + /* Never below what we actually counted. */ + if (est < s.docs + last_count) { + est = s.docs + last_count; + } + + *n_docs = est; + } + + mem_heap_free(heap); + + return(err); +} + +/** Count the documents that contain a word and are still only in the +in-memory FTS cache, that is, have not been SYNCed to the auxiliary table yet. + +fts_node_t::doc_count already holds the number we want, so this decodes no +ilist and reads nothing from disk; it is one rb tree lookup. + +Nodes flagged as synced are skipped. An in-flight SYNC has already written +them to the auxiliary table, and fts_est_probe_aux() reads the B-tree without +a read view, so it sees those records; counting the node as well would count +its documents twice. + +@param[in] index fulltext index +@param[in] word word to look up, folded the same way the caller folds it + for the auxiliary table +@return number of matching documents found in the cache, or 0 if there are +none, if the cache is not initialized, or if another thread holds the cache +mutex (an estimate is not worth waiting for a SYNC to finish) */ +static +uint64_t +fts_est_cache_docs( + const dict_index_t* index, + const fts_string_t* word) noexcept +{ + const fts_t* fts = index->table->fts; + + if (!fts || !fts->cache) { + return(0); + } + + fts_cache_t* cache = fts->cache; + + if (mysql_mutex_trylock(&cache->lock)) { + return(0); + } + + uint64_t n_docs = 0; + + if (const fts_index_cache_t* index_cache + = fts_find_index_cache(cache, index)) { + /* fts_cache_clear() leaves words NULL until the following + fts_cache_init(). The query path never observes that, because + it runs after fts_init_index(); we may. */ + if (index_cache->words) { + const ib_vector_t* nodes + = fts_cache_find_word(index_cache, word); + + for (ulint i = 0; nodes && i < ib_vector_size(nodes); + ++i) { + const fts_node_t* node + = static_cast( + ib_vector_get_const(nodes, i)); + + if (!node->synced) { + n_docs += node->doc_count; + } + } + } + } + + mysql_mutex_unlock(&cache->lock); + + return(n_docs); +} + +dberr_t +fts_estimate_word_docs( + trx_t* trx, + dict_index_t* index, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + ut_ad(index->type & DICT_FTS); + ut_ad(!dict_sys.locked()); + ut_ad(word->f_len); + + *n_docs = 0; + + /* Documents that have been inserted but not SYNCed yet are only in the + memory cache. Look there first: it costs no I/O, and on a table that + has never been SYNCed it is the only information there is. */ + const uint64_t cached = fts_est_cache_docs(index, word); + + /* A word lives in exactly one of INDEX_1..INDEX_6, so only that one + auxiliary table is ever opened -- unlike the query path, which opens + all six. */ + CHARSET_INFO* cs = fts_index_get_charset(index); + const uint8_t selected = fts_select_index(cs, word->f_str, + word->f_len); + fts_table_t fts_table; + + FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), + FTS_INDEX_TABLE, index); + + char aux_name[MAX_FULL_NAME_LEN]; + + fts_get_table_name(&fts_table, aux_name, false); + + dict_table_t* aux = dict_table_open_on_name( + aux_name, false, DICT_ERR_IGNORE_TABLESPACE); + + if (!aux) { + if (cached) { + *n_docs = cached; + return(DB_SUCCESS); + } + + return(DB_TABLE_NOT_FOUND); + } + + dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); + + aux->release(); + + if (err == DB_SUCCESS) { + /* The two populations are disjoint: fts_est_cache_docs() + skipped every node that the auxiliary table already holds. */ + *n_docs += cached; + } else if (cached && err == DB_RECORD_NOT_FOUND) { + /* The auxiliary table is empty, but the cache is not, so we + are no longer without information. The other way round -- + nothing in either -- stays DB_RECORD_NOT_FOUND: the cache is + only complete once fts_init_index() has run, and an estimate + must not run it. */ + *n_docs = cached; + err = DB_SUCCESS; + } + + return(err); +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 6e6b37ecfca30..402f8b6e61553 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -888,510 +888,6 @@ fts_index_get_charset( return fts_get_charset(prtype); } - -/** Maximum number of auxiliary records fts_estimate_word_docs() samples -before it gives up on an exact answer and extrapolates instead. */ -static constexpr uint32_t FTS_EST_MAX_RECS = 64; -/** Maximum number of auxiliary leaf pages fts_estimate_word_docs() reads. */ -static constexpr uint32_t FTS_EST_MAX_PAGES = 4; - -/* Physical field numbers of an FTS auxiliary INDEX_[1..6] record. The -clustered index is UNIQUE(word, first_doc_id), so the record is -(word, first_doc_id, DB_TRX_ID, DB_ROLL_PTR, last_doc_id, doc_count, ilist); -see fts_create_one_index_table(). */ -static constexpr ulint FTS_AUX_FLD_WORD = 0; -static constexpr ulint FTS_AUX_FLD_FIRST_DOC_ID = 1; -static constexpr ulint FTS_AUX_FLD_LAST_DOC_ID = 4; -static constexpr ulint FTS_AUX_FLD_DOC_COUNT = 5; -/** Number of offsets the estimator needs. This deliberately stops short of -the ilist (field 6), which may be stored off-page: the estimator must never -read a BLOB. */ -static constexpr ulint FTS_AUX_EST_N_FIELDS = FTS_AUX_FLD_DOC_COUNT + 1; - -/** Build the search tuple (word) for an FTS auxiliary clustered index. -The index key is (word, first_doc_id), and because the tuple compares on its -first field only it compares equal to every record of the word. So -PAGE_CUR_GE positions on the word's first record and PAGE_CUR_LE on its last. -@param[in] heap heap to allocate the tuple from -@param[in] aux_index auxiliary clustered index -@param[in] word word to search for -@return the search tuple */ -static -dtuple_t* -fts_est_word_tuple( - mem_heap_t* heap, - dict_index_t* aux_index, - const fts_string_t* word) noexcept -{ - dtuple_t* tuple = dtuple_create(heap, 1); - - dict_index_copy_types(tuple, aux_index, 1); - dfield_set_data(dtuple_get_nth_field(tuple, 0), - word->f_str, word->f_len); - dtuple_set_n_fields_cmp(tuple, 1); - - return(tuple); -} - -/** Read the fields the estimator needs out of an FTS auxiliary INDEX_[1..6] -leaf record. Never touches the ilist. -@param[in] tuple search tuple built by fts_est_word_tuple() -@param[in] rec auxiliary table record -@param[in] aux_index auxiliary clustered index -@param[in] offsets rec_get_offsets(rec, aux_index, ...) -@param[out] first_doc_id first doc id in this record's ilist -@param[out] last_doc_id last doc id in this record's ilist -@param[out] doc_count number of doc ids in this record's ilist -@return whether the record belongs to the word tuple was built for */ -static -bool -fts_est_read_rec( - const dtuple_t* tuple, - const rec_t* rec, - const dict_index_t* aux_index, - const rec_offs* offsets, - doc_id_t* first_doc_id, - doc_id_t* last_doc_id, - uint32_t* doc_count) noexcept -{ - /* tuple has n_fields_cmp == 1, so this compares the word only, in the - collation of the auxiliary table's word column. */ - if (cmp_dtuple_rec(tuple, rec, aux_index, offsets)) { - return(false); - } - - ulint len; - const byte* data; - - data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_FIRST_DOC_ID, &len); - *first_doc_id = (data && len == sizeof *first_doc_id) - ? fts_read_doc_id(data) : 0; - - data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_LAST_DOC_ID, &len); - *last_doc_id = (data && len == sizeof *last_doc_id) - ? fts_read_doc_id(data) : *first_doc_id; - - data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_DOC_COUNT, &len); - *doc_count = (data && len == 4) ? mach_read_from_4(data) : 0; - - return(true); -} - -/** What fts_est_sample_word() found. */ -struct fts_est_sample_t -{ - /** Sum of doc_count over the records sampled. */ - uint64_t docs; - /** Number of records sampled. */ - uint32_t n_recs; - /** First doc id of the word's first record. */ - doc_id_t first_doc_id; - /** Last doc id of the last record sampled. */ - doc_id_t last_doc_id; - /** Whether sampling stopped because a budget ran out rather than - because the word's key range ended. When this is false, docs is the - exact number of documents that contain the word. */ - bool truncated; -}; - -/** Dive to a word's first auxiliary record and walk forward from it, over at -most FTS_EST_MAX_RECS records and FTS_EST_MAX_PAGES leaf pages. The walk is -nearly free because the dive has already latched the leaf page, and if it -reaches a different word before a budget runs out the result is exact. -@param[in] trx transaction, used only for the mtr -@param[in] aux_index auxiliary clustered index -@param[in] tuple search tuple built by fts_est_word_tuple() -@param[out] out what the walk found -@return DB_SUCCESS, or DB_RECORD_NOT_FOUND if the auxiliary table is empty -(nothing has been SYNCed yet, so there is no information at all) */ -static -dberr_t -fts_est_sample_word( - trx_t* trx, - dict_index_t* aux_index, - const dtuple_t* tuple, - fts_est_sample_t* out) noexcept -{ - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - mem_heap_t* offs_heap = NULL; - btr_pcur_t pcur; - mtr_t mtr{trx}; - - rec_offs_init(offsets_); - memset(out, 0, sizeof *out); - - mtr.start(); - pcur.btr_cur.page_cur.index = aux_index; - - dberr_t err = btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, - &pcur, &mtr); - - if (err != DB_SUCCESS) { - goto func_exit; - } - - if (!btr_pcur_is_on_user_rec(&pcur)) { - /* Nothing at or after the word. Tell an empty auxiliary - table, where we have no information at all, apart from a word - that simply sorts after everything that is indexed. */ - const page_t* page = btr_pcur_get_page(&pcur); - - if (!page_has_prev(page) && !page_get_n_recs(page)) { - err = DB_RECORD_NOT_FOUND; - } - - goto func_exit; - } - - { - uint32_t n_pages = 1; - page_id_t last_page - = btr_pcur_get_block(&pcur)->page.id(); - - do { - const rec_t* rec = btr_pcur_get_rec(&pcur); - const buf_block_t* block - = btr_pcur_get_block(&pcur); - const ulint offs - = ulint(rec - block->page.frame); - - if (page_rec_is_infimum_low(offs) - || page_rec_is_supremum_low(offs)) { - continue; - } - - if (block->page.id() != last_page) { - last_page = block->page.id(); - if (++n_pages > FTS_EST_MAX_PAGES) { - out->truncated = true; - break; - } - } - - offsets = rec_get_offsets(rec, aux_index, offsets, - aux_index->n_core_fields, - FTS_AUX_EST_N_FIELDS, - &offs_heap); - - doc_id_t first; - doc_id_t last; - uint32_t doc_count; - - if (!fts_est_read_rec(tuple, rec, aux_index, offsets, - &first, &last, &doc_count)) { - /* Walked past the word's key range, so we - have seen all of it. */ - break; - } - - if (!out->n_recs) { - out->first_doc_id = first; - } - - out->last_doc_id = last; - out->docs += doc_count; - - if (++out->n_recs >= FTS_EST_MAX_RECS) { - out->truncated = true; - break; - } - } while (btr_pcur_move_to_next(&pcur, &mtr)); - } - -func_exit: - mtr.commit(); - ut_free(pcur.old_rec_buf); - - if (UNIV_LIKELY_NULL(offs_heap)) { - mem_heap_free(offs_heap); - } - - return(err); -} - -/** Dive to a word's last auxiliary record, to learn how far its doc ids -reach. -@param[in] trx transaction, used only for the mtr -@param[in] aux_index auxiliary clustered index -@param[in] tuple search tuple built by fts_est_word_tuple() -@param[in,out] last_doc_id last doc id of the word; left alone if the - record cannot be read -@param[in,out] doc_count doc_count of that record; left alone if the - record cannot be read -@return DB_SUCCESS or error code */ -static -dberr_t -fts_est_last_rec( - trx_t* trx, - dict_index_t* aux_index, - const dtuple_t* tuple, - doc_id_t* last_doc_id, - uint32_t* doc_count) noexcept -{ - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - mem_heap_t* offs_heap = NULL; - btr_pcur_t pcur; - mtr_t mtr{trx}; - - rec_offs_init(offsets_); - - mtr.start(); - pcur.btr_cur.page_cur.index = aux_index; - - dberr_t err = btr_pcur_open(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, - &pcur, &mtr); - - if (err == DB_SUCCESS) { - /* PAGE_CUR_LE may leave the cursor on the page infimum, in - which case the record we want is the last one of the previous - page. */ - const bool positioned - = !btr_pcur_is_before_first_on_page(&pcur) - || btr_pcur_move_to_prev(&pcur, &mtr); - - if (positioned && btr_pcur_is_on_user_rec(&pcur)) { - const rec_t* rec = btr_pcur_get_rec(&pcur); - doc_id_t first; - doc_id_t last; - uint32_t count; - - offsets = rec_get_offsets(rec, aux_index, offsets, - aux_index->n_core_fields, - FTS_AUX_EST_N_FIELDS, - &offs_heap); - - if (fts_est_read_rec(tuple, rec, aux_index, offsets, - &first, &last, &count)) { - *last_doc_id = last; - *doc_count = count; - } - } - } - - mtr.commit(); - ut_free(pcur.old_rec_buf); - - if (UNIV_LIKELY_NULL(offs_heap)) { - mem_heap_free(offs_heap); - } - - return(err); -} - -/** Estimate how many documents contain a word, by probing an already opened -FTS auxiliary INDEX_[1..6] table. See fts_estimate_word_docs(). -@param[in] trx transaction, used only for the mtr -@param[in] aux auxiliary table that holds the word -@param[in] word word to look up -@param[out] n_docs estimated number of matching documents -@return DB_SUCCESS, DB_RECORD_NOT_FOUND or DB_CORRUPTION */ -static -dberr_t -fts_est_probe_aux( - trx_t* trx, - dict_table_t* aux, - const fts_string_t* word, - uint64_t* n_docs) noexcept -{ - dict_index_t* aux_index = dict_table_get_first_index(aux); - - if (!aux->space || !aux->is_readable() || !aux_index - || aux_index->page == FIL_NULL || aux_index->is_corrupted()) { - return(DB_CORRUPTION); - } - - mem_heap_t* heap = mem_heap_create(256); - const dtuple_t* tuple = fts_est_word_tuple(heap, aux_index, - word); - fts_est_sample_t s; - dberr_t err = fts_est_sample_word(trx, aux_index, - tuple, &s); - - if (err != DB_SUCCESS) { - mem_heap_free(heap); - return(err); - } - - if (!s.n_recs || !s.truncated) { - /* Either the word is absent, or the walk covered its whole - key range, in which case the count is exact. */ - *n_docs = s.docs; - mem_heap_free(heap); - return(DB_SUCCESS); - } - - /* The word has more records than we are willing to read. Dive once - more, to its last record, and extrapolate the density we measured - across the word's whole doc id span. Doc ids only ever increase, so - the records we sampled are a prefix of that span. */ - doc_id_t last_doc_id = s.last_doc_id; - uint32_t last_count = 0; - - err = fts_est_last_rec(trx, aux_index, tuple, &last_doc_id, - &last_count); - - if (err == DB_SUCCESS) { - if (last_doc_id < s.last_doc_id) { - last_doc_id = s.last_doc_id; - } - - const uint64_t sampled_span - = s.last_doc_id > s.first_doc_id - ? s.last_doc_id - s.first_doc_id + 1 : 1; - const uint64_t total_span - = last_doc_id - s.first_doc_id + 1; - - /* In double, because docs * total_span overflows 64 bits for - large inputs. The caller clamps the result to the number of - rows in the table. */ - uint64_t est = uint64_t(double(s.docs) - * double(total_span) - / double(sampled_span)); - - /* Never below what we actually counted. */ - if (est < s.docs + last_count) { - est = s.docs + last_count; - } - - *n_docs = est; - } - - mem_heap_free(heap); - - return(err); -} - -/** Count the documents that contain a word and are still only in the -in-memory FTS cache, that is, have not been SYNCed to the auxiliary table yet. - -fts_node_t::doc_count already holds the number we want, so this decodes no -ilist and reads nothing from disk; it is one rb tree lookup. - -Nodes flagged as synced are skipped. An in-flight SYNC has already written -them to the auxiliary table, and fts_est_probe_aux() reads the B-tree without -a read view, so it sees those records; counting the node as well would count -its documents twice. - -@param[in] index fulltext index -@param[in] word word to look up, folded the same way the caller folds it - for the auxiliary table -@return number of matching documents found in the cache, or 0 if there are -none, if the cache is not initialized, or if another thread holds the cache -mutex (an estimate is not worth waiting for a SYNC to finish) */ -static -uint64_t -fts_est_cache_docs( - const dict_index_t* index, - const fts_string_t* word) noexcept -{ - const fts_t* fts = index->table->fts; - - if (!fts || !fts->cache) { - return(0); - } - - fts_cache_t* cache = fts->cache; - - if (mysql_mutex_trylock(&cache->lock)) { - return(0); - } - - uint64_t n_docs = 0; - - if (const fts_index_cache_t* index_cache - = fts_find_index_cache(cache, index)) { - /* fts_cache_clear() leaves words NULL until the following - fts_cache_init(). The query path never observes that, because - it runs after fts_init_index(); we may. */ - if (index_cache->words) { - const ib_vector_t* nodes - = fts_cache_find_word(index_cache, word); - - for (ulint i = 0; nodes && i < ib_vector_size(nodes); - ++i) { - const fts_node_t* node - = static_cast( - ib_vector_get_const(nodes, i)); - - if (!node->synced) { - n_docs += node->doc_count; - } - } - } - } - - mysql_mutex_unlock(&cache->lock); - - return(n_docs); -} - -dberr_t -fts_estimate_word_docs( - trx_t* trx, - dict_index_t* index, - const fts_string_t* word, - uint64_t* n_docs) noexcept -{ - ut_ad(index->type & DICT_FTS); - ut_ad(!dict_sys.locked()); - ut_ad(word->f_len); - - *n_docs = 0; - - /* Documents that have been inserted but not SYNCed yet are only in the - memory cache. Look there first: it costs no I/O, and on a table that - has never been SYNCed it is the only information there is. */ - const uint64_t cached = fts_est_cache_docs(index, word); - - /* A word lives in exactly one of INDEX_1..INDEX_6, so only that one - auxiliary table is ever opened -- unlike the query path, which opens - all six. */ - CHARSET_INFO* cs = fts_index_get_charset(index); - const uint8_t selected = fts_select_index(cs, word->f_str, - word->f_len); - fts_table_t fts_table; - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - - char aux_name[MAX_FULL_NAME_LEN]; - - fts_get_table_name(&fts_table, aux_name, false); - - dict_table_t* aux = dict_table_open_on_name( - aux_name, false, DICT_ERR_IGNORE_TABLESPACE); - - if (!aux) { - if (cached) { - *n_docs = cached; - return(DB_SUCCESS); - } - - return(DB_TABLE_NOT_FOUND); - } - - dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); - - aux->release(); - - if (err == DB_SUCCESS) { - /* The two populations are disjoint: fts_est_cache_docs() - skipped every node that the auxiliary table already holds. */ - *n_docs += cached; - } else if (cached && err == DB_RECORD_NOT_FOUND) { - /* The auxiliary table is empty, but the cache is not, so we - are no longer without information. The other way round -- - nothing in either -- stays DB_RECORD_NOT_FOUND: the cache is - only complete once fts_init_index() has run, and an estimate - must not run it. */ - *n_docs = cached; - err = DB_SUCCESS; - } - - return(err); -} /****************************************************************//** Create an FTS index cache. @return Index Cache */ From 4308bb6e96664a17b5d90b2de34c2385bfbe67f7 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 09:41:18 +0300 Subject: [PATCH 15/39] Adjust the MVI tests to the estimate that reads the FTS cache fulltext_estimate() used to see only the on-disk auxiliary table, so the multi-valued index tests, which insert and immediately EXPLAIN, got "unknown" for every element key: the conjunctive accesses fell back to a guess of 10 rows and the disjunctive ones were dropped for want of a cost. Both tests worked around that with OPTIMIZE TABLE under innodb_optimize_fulltext_only. The estimate consults the cache now, so the workaround is gone and the row counts in the plans are real. Two sections of the trace test were written before the estimate existed and no longer showed what they said they did: - "Several element keys" printed one range, not several, because a conjunctive access now keeps only the rarest key. It runs on a table with a skewed distribution instead ("bbb" in every row, "aaa" in one), which makes the choice of key visible rather than a tie, and checks the rows against the same query with IGNORE INDEX. - The JSON_OVERLAPS section is where several ranges are printed now, so it says so, along with the estimates adding up. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 17 +- mysql-test/main/multi_valued_index.test | 8 - .../multi_valued_index_notembedded.result | 156 +++++------------- .../main/multi_valued_index_notembedded.test | 74 +++------ 4 files changed, 70 insertions(+), 185 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 661b43d3e361b..8e00696c34feb 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -35,7 +35,7 @@ insert into t1 values (4, '{}'); explain select * from t1 where json_contains(j->'$.tags', '"abcde"'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 1 Using where select * from t1 where json_contains(j->'$.tags', '"abcde"'); c j 2 {"tags": ["1", "abcde", "", 34567]} @@ -51,14 +51,14 @@ c j explain select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 1 Using where select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); c j 2 {"tags": ["1", "abcde", "", 34567]} explain select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 1 Using where select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); c j select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); @@ -80,7 +80,7 @@ insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), (3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); explain select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 2 Using where select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; c j 1 {"tags": ["aaa"]} @@ -171,15 +171,6 @@ key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), (5,'{"tags": [123]}'),(6,'{}'); -# An ORed access has to read every one of its keys, so we only use it -# when the engine can estimate all of them. The estimate is taken from -# the fulltext auxiliary tables, so flush the FTS cache into them. -set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; -set global innodb_optimize_fulltext_only=1; -optimize table t1; -Table Op Msg_type Msg_text -test.t1 optimize status OK -set global innodb_optimize_fulltext_only=@old_fulltext_only; explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx idx 0 NULL 4 Using where diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 9dff45e758ac2..85cb9bd674701 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -119,14 +119,6 @@ insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), (5,'{"tags": [123]}'),(6,'{}'); ---echo # An ORed access has to read every one of its keys, so we only use it ---echo # when the engine can estimate all of them. The estimate is taken from ---echo # the fulltext auxiliary tables, so flush the FTS cache into them. -set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; -set global innodb_optimize_fulltext_only=1; -optimize table t1; -set global innodb_optimize_fulltext_only=@old_fulltext_only; - explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; --echo # the same rows without the index: diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result index d1ae73486e60d..f9e7bf5bd5256 100644 --- a/mysql-test/main/multi_valued_index_notembedded.result +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -15,7 +15,7 @@ set optimizer_trace=1; # explain select * from t1 where json_contains(j->'$.tags','"aaa"'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t1 range idx idx 0 NULL 1 Using where select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; jd @@ -24,35 +24,55 @@ jd "table": "t1", "index": "idx", "match": "all", - "rows": 10, - "cost": 0.001, + "rows": 1, + "cost": 0.00171364, "ranges": ["616161"] } ] # -# Several element keys +# Several element keys. A row has to have all of them, so the rarest +# one alone already reads every row we may return: only that key is +# left in "ranges", and the JSON predicate in the WHERE clause discards +# whatever else the shorter search finds. # -explain select * from t1 where json_contains(j->'$.tags','["bbb","ccc"]'); +create table t3 (c int, j json, +key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +select c, mvi_encode(j->'$.tags', char(6)) from t3; +c mvi_encode(j->'$.tags', char(6)) +1 616161 626262 +2 626262 +3 626262 +4 626262 636363 +# "bbb" is in every row and "aaa" in one, so "aaa" is what we search for +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 10 Using where +1 SIMPLE t3 range idx3 idx3 0 NULL 1 Using where select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; jd [ { - "table": "t1", - "index": "idx", + "table": "t3", + "index": "idx3", "match": "all", - "rows": 10, - "cost": 0.001, + "rows": 1, + "cost": 0.00171364, "ranges": - [ - "626262", - "636363" - ] + ["616161"] } ] +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +drop table t3; # # Two tables: each entry names the table it belongs to # @@ -62,8 +82,8 @@ insert into t2 values (1,'{"tags": ["zzz"]}'); explain select * from t1,t2 where json_contains(t1.j->'$.tags','"aaa"') and json_contains(t2.j->'$.tags','"zzz"'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range idx2 idx2 0 NULL 10 Using where -1 SIMPLE t1 range idx idx 0 NULL 10 Using where; Using join buffer (flat, BNL join) +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +1 SIMPLE t2 range idx2 idx2 0 NULL 1 Using where; Using join buffer (flat, BNL join) select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; jd @@ -72,8 +92,8 @@ jd "table": "t1", "index": "idx", "match": "all", - "rows": 10, - "cost": 0.001, + "rows": 1, + "cost": 0.00171364, "ranges": ["616161"] }, @@ -81,18 +101,20 @@ jd "table": "t2", "index": "idx2", "match": "all", - "rows": 10, - "cost": 0.001, + "rows": 1, + "cost": 0.00171364, "ranges": ["7a7a7a"] } ] # -# JSON_OVERLAPS: the keys are ORed, so "match" is "any" +# JSON_OVERLAPS: the keys are ORed, so "match" is "any". A row matches +# through any one of them, so the estimates add up and every key stays +# in the query. # explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ALL idx NULL NULL NULL 2 Using where +1 SIMPLE t1 range idx idx 0 NULL 2 Using where select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; jd @@ -101,103 +123,15 @@ jd "table": "t1", "index": "idx", "match": "any", - "usable": false, - "cause": "the engine cannot estimate one of the keys", - "ranges": - [ - "626262", - "636363" - ] - } -] -# -# Everything above ran on a table whose words are still in the FTS -# cache, where the engine cannot estimate them: the conjunctive -# accesses fell back to a guess, and the disjunctive one was dropped. -# Flush the words into the auxiliary tables and the estimates are real. -# -create table t3 (c int, j json, -key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; -insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), -(3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); -set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; -set global innodb_optimize_fulltext_only=1; -optimize table t3; -Table Op Msg_type Msg_text -test.t3 optimize status OK -set global innodb_optimize_fulltext_only=@old_fulltext_only; -select c, mvi_encode(j->'$.tags', char(6)) from t3; -c mvi_encode(j->'$.tags', char(6)) -1 616161 626262 -2 626262 -3 626262 -4 626262 636363 -# -# "bbb" is in every row and "aaa" in one. A row has to have both, so -# the search for "aaa" alone already reads every row we may return: -# only that key is left in "ranges", and the JSON predicate in the -# WHERE clause discards whatever else the shorter search finds. -# -explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range idx3 idx3 0 NULL 1 Using where -select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd -from information_schema.optimizer_trace; -jd -[ - { - "table": "t3", - "index": "idx3", - "match": "all", - "rows": 1, - "cost": 0.00171364, - "ranges": - ["616161"] - } -] -select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; -c j -1 {"tags": ["aaa","bbb"]} -# the same rows without the index: -select * from t3 ignore index(idx3) -where json_contains(j->'$.tags','["aaa","bbb"]') order by c; -c j -1 {"tags": ["aaa","bbb"]} -# -# A row matches an OR through any of its keys, so the estimates add up -# and all the keys stay in the query. -# -explain select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]'); -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range idx3 idx3 0 NULL 2 Using where -select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd -from information_schema.optimizer_trace; -jd -[ - { - "table": "t3", - "index": "idx3", - "match": "any", "rows": 2, "cost": 0.00260808, "ranges": [ - "616161", + "626262", "636363" ] } ] -select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; -c j -1 {"tags": ["aaa","bbb"]} -4 {"tags": ["bbb","ccc"]} -# the same rows without the index: -select * from t3 ignore index(idx3) -where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; -c j -1 {"tags": ["aaa","bbb"]} -4 {"tags": ["bbb","ccc"]} -drop table t3; # # A predicate under a top-level OR gives no access, so nothing is printed # diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test index 304a7f2b80f35..5dccb0c0b63e0 100644 --- a/mysql-test/main/multi_valued_index_notembedded.test +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -27,14 +27,30 @@ from information_schema.optimizer_trace; --enable_view_protocol --echo # ---echo # Several element keys +--echo # Several element keys. A row has to have all of them, so the rarest +--echo # one alone already reads every row we may return: only that key is +--echo # left in "ranges", and the JSON predicate in the WHERE clause discards +--echo # whatever else the shorter search finds. --echo # +create table t3 (c int, j json, + key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +select c, mvi_encode(j->'$.tags', char(6)) from t3; + +--echo # "bbb" is in every row and "aaa" in one, so "aaa" is what we search for --disable_replay next_query Need to preserve optimizer trace -explain select * from t1 where json_contains(j->'$.tags','["bbb","ccc"]'); +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); --disable_view_protocol select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; --enable_view_protocol +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +--echo # the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; + +drop table t3; --echo # --echo # Two tables: each entry names the table it belongs to @@ -51,7 +67,9 @@ from information_schema.optimizer_trace; --enable_view_protocol --echo # ---echo # JSON_OVERLAPS: the keys are ORed, so "match" is "any" +--echo # JSON_OVERLAPS: the keys are ORed, so "match" is "any". A row matches +--echo # through any one of them, so the estimates add up and every key stays +--echo # in the query. --echo # --disable_replay next_query Need to preserve optimizer trace explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); @@ -60,56 +78,6 @@ select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd from information_schema.optimizer_trace; --enable_view_protocol ---echo # ---echo # Everything above ran on a table whose words are still in the FTS ---echo # cache, where the engine cannot estimate them: the conjunctive ---echo # accesses fell back to a guess, and the disjunctive one was dropped. ---echo # Flush the words into the auxiliary tables and the estimates are real. ---echo # -create table t3 (c int, j json, - key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; -insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), - (3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); -set @old_fulltext_only=@@global.innodb_optimize_fulltext_only; -set global innodb_optimize_fulltext_only=1; -optimize table t3; -set global innodb_optimize_fulltext_only=@old_fulltext_only; -select c, mvi_encode(j->'$.tags', char(6)) from t3; - ---echo # ---echo # "bbb" is in every row and "aaa" in one. A row has to have both, so ---echo # the search for "aaa" alone already reads every row we may return: ---echo # only that key is left in "ranges", and the JSON predicate in the ---echo # WHERE clause discards whatever else the shorter search finds. ---echo # ---disable_replay next_query Need to preserve optimizer trace -explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); ---disable_view_protocol -select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd -from information_schema.optimizer_trace; ---enable_view_protocol -select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; ---echo # the same rows without the index: -select * from t3 ignore index(idx3) -where json_contains(j->'$.tags','["aaa","bbb"]') order by c; - ---echo # ---echo # A row matches an OR through any of its keys, so the estimates add up ---echo # and all the keys stay in the query. ---echo # ---disable_replay next_query Need to preserve optimizer trace -explain select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]'); ---disable_view_protocol -select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd -from information_schema.optimizer_trace; ---enable_view_protocol -select * from t3 where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; ---echo # the same rows without the index: -select * from t3 ignore index(idx3) -where json_overlaps(j->'$.tags','["aaa","ccc"]') order by c; - -drop table t3; - --echo # --echo # A predicate under a top-level OR gives no access, so nothing is printed --echo # From 8f16f794ad7a771111b55910437a6fc2e00a688a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 13:35:26 +0300 Subject: [PATCH 16/39] Trivial cleanups and comments --- sql/opt_multi_valued_index.cc | 17 ++++++++++------- sql/opt_mvi_jsonfuncs.cc | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 02cca3fd56939..36f70688ff37b 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -329,7 +329,7 @@ void Mvi_access::estimate_records() List_iterator it(encoded); String *key, *rarest= NULL; ha_rows sum= 0, min_rows= 0; - bool unknown= false; + bool have_unknown_estimate= false; while ((key= it++)) { @@ -337,7 +337,7 @@ void Mvi_access::estimate_records() (uint) key->length()); if (rows == HA_POS_ERROR) { - unknown= true; + have_unknown_estimate= true; continue; } sum+= rows; @@ -348,18 +348,21 @@ void Mvi_access::estimate_records() } } - if (!conjunctive && unknown) + if (!conjunctive && have_unknown_estimate) { - /* We have to read this key and have no idea what that costs */ + /* + Disjunctive means we have to read all keys. For at least one, we have no idea + how many matches it has. Fall back to full scan. + */ records= table->stat_records(); read_time= DBL_MAX; return; } if (conjunctive && !rarest) { - /* Nothing was estimated. Keep the old guess and the query as it is */ - records= 10; - read_time= 0.001; + /* Nothing was estimated. Fall back to full table scan */ + records= table->stat_records(); + read_time= DBL_MAX; return; } diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc index d03e202e89a3a..5f913b8fdd71f 100644 --- a/sql/opt_mvi_jsonfuncs.cc +++ b/sql/opt_mvi_jsonfuncs.cc @@ -234,6 +234,7 @@ Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, return NULL; /* + TODO: is this really so: encode_mvi_key() must see the collation of the indexed expression: that is what decides how MVI_ENCODE built the keys that are in the index. */ From 409a649f16d41939485cddfdf1d8c4a6670bd1b2 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 19:02:00 +0300 Subject: [PATCH 17/39] Move QUICK_MVI_SELECT into opt_multi_valued_index.cc --- sql/opt_multi_valued_index.cc | 51 ++++++++++++++++++++++++++++++++++- sql/opt_range.h | 43 ----------------------------- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 36f70688ff37b..13ea1b53e9f28 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -19,6 +19,8 @@ #include "item_func.h" #include "my_json_writer.h" +static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, Mvi_access *access); + void Item_func_mvi_encode::print(String *str, enum_query_type query_type) { char buf[32]; @@ -584,13 +586,60 @@ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) */ if (!access->cost_is_known()) return NULL; - return new QUICK_MVI_SELECT(thd, table, access); + return create_quick_mvi_select(thd, table, access); } /**************************************************************************** QUICK_MVI_SELECT - reading a multi-valued index ****************************************************************************/ +struct Mvi_access; + +/* + Quick select that reads a multi-valued index. + + It runs a boolean-mode fulltext search over the index's hidden vcol, looking + for the encoded element keys of the JSON predicate this access was built + from. The scan is a necessary, not a sufficient condition: the JSON + predicate stays in the WHERE clause and does the exact filtering. + + Unlike FT_SELECT, there is no Item_func_match to have created the FT_INFO + for us, so we create it ourselves in reset() and own it. + + The methods are implemented in opt_multi_valued_index.cc. +*/ + +class QUICK_MVI_SELECT: public QUICK_SELECT_I +{ + Mvi_access *access; + FT_INFO *ft_handler; + StringBuffer<256> query; /* the boolean-mode ft query */ +public: + QUICK_MVI_SELECT(THD *thd, TABLE *table, Mvi_access *access_arg); + ~QUICK_MVI_SELECT(); + int init() override { return 0; } + int reset() override; + int get_next() override; + bool reverse_sorted() override { return false; } + /* + Fulltext results come back ordered by relevance, not by key, so there is + no sorted output to offer. QS_TYPE_MVI is not one of the types the + ORDER BY-by-index code paths consider, so they never ask. + */ + void need_sorted_output() override {} + int get_type() override { return QS_TYPE_MVI; } + void add_keys_and_lengths(String *key_names, String *used_lengths) override; + void add_used_key_part_to_set() override {} + Explain_quick_select *get_explain(MEM_ROOT *alloc) override; +#ifndef DBUG_OFF + void dbug_dump(int indent, bool verbose) override; +#endif +}; + +static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, Mvi_access *access) +{ + return new QUICK_MVI_SELECT(thd, table, access); +} QUICK_MVI_SELECT::QUICK_MVI_SELECT(THD *thd, TABLE *table, Mvi_access *access_arg) diff --git a/sql/opt_range.h b/sql/opt_range.h index 0e6790da6fe26..0ad39eeb5e929 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -2042,49 +2042,6 @@ class FT_SELECT: public QUICK_RANGE_SELECT }; -struct Mvi_access; - -/* - Quick select that reads a multi-valued index. - - It runs a boolean-mode fulltext search over the index's hidden vcol, looking - for the encoded element keys of the JSON predicate this access was built - from. The scan is a necessary, not a sufficient condition: the JSON - predicate stays in the WHERE clause and does the exact filtering. - - Unlike FT_SELECT, there is no Item_func_match to have created the FT_INFO - for us, so we create it ourselves in reset() and own it. - - The methods are implemented in opt_multi_valued_index.cc. -*/ - -class QUICK_MVI_SELECT: public QUICK_SELECT_I -{ - Mvi_access *access; - FT_INFO *ft_handler; - StringBuffer<256> query; /* the boolean-mode ft query */ -public: - QUICK_MVI_SELECT(THD *thd, TABLE *table, Mvi_access *access_arg); - ~QUICK_MVI_SELECT(); - int init() override { return 0; } - int reset() override; - int get_next() override; - bool reverse_sorted() override { return false; } - /* - Fulltext results come back ordered by relevance, not by key, so there is - no sorted output to offer. QS_TYPE_MVI is not one of the types the - ORDER BY-by-index code paths consider, so they never ask. - */ - void need_sorted_output() override {} - int get_type() override { return QS_TYPE_MVI; } - void add_keys_and_lengths(String *key_names, String *used_lengths) override; - void add_used_key_part_to_set() override {} - Explain_quick_select *get_explain(MEM_ROOT *alloc) override; -#ifndef DBUG_OFF - void dbug_dump(int indent, bool verbose) override; -#endif -}; - FT_SELECT *get_ft_select(THD *thd, TABLE *table, uint key); QUICK_RANGE_SELECT *get_quick_select_for_ref(THD *thd, TABLE *table, struct st_table_ref *ref, From d059022d6b5e44593abe962eb980985cc390d28a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sun, 6 Sep 2026 21:38:31 +0300 Subject: [PATCH 18/39] Keep the MVI access of a table in its JOIN_TAB The access was looked up through JOIN::get_mvi_access_for_table(), which indexed a per-JOIN array by table->tablenr. Nothing else about how a table is going to be read lives there, so put it where the rest does: JOIN_TAB::mvi_access. Mvi_context is then just the result of the WHERE analysis - the indexes and the accesses it found - and the choice of which access a table uses is made per table, where it belongs. setup_mvi_access_for_table() makes that choice and marks the index in const_keys and keys, which make_join_statistics() used to do in a loop of its own right after update_ref_and_keys(). It runs next to add_group_and_distinct_keys(), the other place that adds to const_keys for something the range optimizer would not find by itself, and just before the range analysis those bits exist for. get_quick_record_count() takes the JOIN_TAB rather than the TABLE now, which is all it needed the JOIN for. Also fix the header comment of Mvi_access::estimate_records(), which still described the old fallback for a conjunctive access that could not be estimated at all. Co-Authored-By: Claude Opus 5 (1M context) --- sql/opt_multi_valued_index.cc | 83 +++++++++++++++++++---------------- sql/opt_multi_valued_index.h | 14 +++--- sql/sql_select.cc | 39 ++++++---------- sql/sql_select.h | 12 ++--- 4 files changed, 73 insertions(+), 75 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 13ea1b53e9f28..98476e8255be2 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -315,10 +315,9 @@ bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) intersect the terms. This is the trade-off collect_mvi_keys() already makes for the keys it cannot encode - a shorter AND matches a superset of the rows, and the JSON predicate does the exact filtering. - Keys the engine cannot estimate take no part in the choice. If it could - not estimate a single one of them, we know nothing: keep the query as - it is and fall back to a guess low enough that the access is still - preferred over a table scan. + Keys the engine cannot estimate take no part in the choice. If it + could not estimate a single one of them we know nothing at all, so the + access is priced out just like a disjunctive one. TODO: read_time only accounts for reading the rows, not for the fulltext search that produces their rowids. @@ -448,26 +447,6 @@ void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) } -static void choose_mvi_access_for_tables(List *accesses, Mvi_access **best) -{ - List_iterator it(*accesses); - /* TODO: cost based */ - /* - TODO: merge - - json_contains(j->'$.tags','"a"') and - json_contains(j->'$.tags','"b"') - - (+ta +tb) - */ - while (Mvi_access *access= it++) - { - DBUG_ASSERT(access->index->vcol->table->tablenr < MAX_TABLES); - best[access->index->vcol->table->tablenr] = access; - } -} - - /* @brief Collect the MVI accesses allowed by the top-level AND-parts of `conds'. @@ -509,8 +488,8 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) Analyze the WHERE clause and find the MVI accesses it allows. @detail - The accesses are saved in join->mvi_ctx, where get_best_mvi_access() picks - them up during the range analysis of each table. + The accesses are saved in join->mvi_ctx, where setup_mvi_access_for_table() + picks them up, one table at a time. */ bool setup_mvi_quick(JOIN *join) @@ -531,24 +510,53 @@ bool setup_mvi_quick(JOIN *join) return true; if (ctx->accesses.is_empty()) return false; - choose_mvi_access_for_tables(&ctx->accesses, ctx->best); join->mvi_ctx= ctx; return false; } -Mvi_access *JOIN::get_mvi_access_for_table(TABLE *table) +/* + @brief + Pick the MVI access `tab' will use out of the ones the WHERE clause + allows, and let the range analysis see it. + + @detail + A fulltext key never gets a bit in const_keys or keys, so we set them + here. The const_keys bit is what makes the range analysis run for this + table, where get_best_mvi_access() turns the access into a quick select; + the keys bit puts the index into EXPLAIN's possible_keys. +*/ + +void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab) { - if (!mvi_ctx) - return NULL; - DBUG_ASSERT(table->tablenr < MAX_TABLES); - return mvi_ctx->best[table->tablenr]; + if (!join->mvi_ctx) + return; + List_iterator it(join->mvi_ctx->accesses); + /* TODO: cost based */ + /* + TODO: merge + + json_contains(j->'$.tags','"a"') and + json_contains(j->'$.tags','"b"') + + (+ta +tb) + */ + while (Mvi_access *access= it++) + { + if (access->index->vcol->table == tab->table) + tab->mvi_access= access; + } + if (tab->mvi_access) + { + tab->const_keys.set_bit(tab->mvi_access->index->keyno); + tab->keys.set_bit(tab->mvi_access->index->keyno); + } } /* @brief - Create a quick select for the best MVI access to `table', if there is one. + Create a quick select for the MVI access to `tab', if there is one. @detail The range optimizer cannot produce this access (it skips fulltext keys), @@ -556,14 +564,15 @@ Mvi_access *JOIN::get_mvi_access_for_table(TABLE *table) test_quick_select() came up with. */ -QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table) +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) { - Mvi_access *access= join->get_mvi_access_for_table(table); + Mvi_access *access= tab->mvi_access; + TABLE *table= tab->table; if (!access) return NULL; /* - We are called once per table for each of the two range analysis passes. - Probe the engine (and drop the keys we don't need) only on the first one. + estimate_records() drops element keys from the access, so it must run + only once even if we are called again for the same table. */ if (access->records == HA_POS_ERROR) access->estimate_records(); diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index bacdb0c6fc2af..d96a31bd7e657 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -69,13 +69,8 @@ class Mvi_context : public Sql_alloc List indexes; /* MVI accesses for all eligible predicates in WHERE */ List accesses; - /* The access we've chosen for each table, indexed by table->tablenr */ - Mvi_access *best[MAX_TABLES]; - Mvi_context(THD *thd_arg) : thd(thd_arg) - { - bzero(best, sizeof(best)); - } + Mvi_context(THD *thd_arg) : thd(thd_arg) {} }; /* Return the compatible json type */ @@ -91,5 +86,8 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, bool setup_mvi_quick(JOIN *join); -/* Create a quick select for the best MVI access to `table', if there is one */ -QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN *join, TABLE *table); +/* Pick the MVI access `tab' will use, and let the range analysis see it */ +void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab); + +/* Create a quick select for the MVI access to `tab', if there is one */ +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 4a5835c7e0438..00a213a1d7b04 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -140,8 +140,8 @@ static int sort_keyuse(const void *a, const void *b); static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables); static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables); -static bool get_quick_record_count(THD *thd, JOIN *join, SQL_SELECT *select, - TABLE *table, +static bool get_quick_record_count(THD *thd, SQL_SELECT *select, + JOIN_TAB *tab, const key_map *keys,ha_rows limit, ha_rows *quick_count); static void optimize_straight_join(JOIN *join, table_map join_tables); @@ -5513,9 +5513,8 @@ static void keep_cheaper_quick(TABLE *table, QUICK_SELECT_I **quick_ref, select with this key. @param thd Thread handle - @param join The join the table belongs to @param select Select to be examined - @param table The table of interest + @param tab The table of interest @param keys The keys of interest @param limit Maximum number of rows of interest @param quick_count Pointer to where we want the estimate written @@ -5525,12 +5524,13 @@ static void keep_cheaper_quick(TABLE *table, QUICK_SELECT_I **quick_ref, @retval true Error */ -static bool get_quick_record_count(THD *thd, JOIN *join, SQL_SELECT *select, - TABLE *table, +static bool get_quick_record_count(THD *thd, SQL_SELECT *select, + JOIN_TAB *tab, const key_map *keys,ha_rows limit, ha_rows *quick_count) { quick_select_return error; + TABLE *table= tab->table; DBUG_ENTER("get_quick_record_count"); uchar buff[STACK_BUFF_ALLOC]; if (unlikely(check_stack_overrun(thd, STACK_MIN_SIZE, buff))) @@ -5547,7 +5547,7 @@ static bool get_quick_record_count(THD *thd, JOIN *join, SQL_SELECT *select, here and keep it across the call: test_quick_select() deletes select->quick on entry. */ - QUICK_SELECT_I *mvi_quick= get_best_mvi_access(thd, join, table); + QUICK_SELECT_I *mvi_quick= get_best_mvi_access(thd, tab); /* EQ_FUNC and EQUAL_FUNC already sent unusable key notes (if any) during update_ref_and_keys(). Have only other functions raise notes @@ -5925,23 +5925,6 @@ make_join_statistics(JOIN *join, List &tables_list, print_keyuse_array_for_trace(thd, keyuse_array); } - /* - A fulltext key never gets a bit in const_keys or keys, so mark the MVI key - of every table that has an MVI access. The const_keys bit is what makes - the range analysis below run for that table, where get_best_mvi_access() - picks the access up; the keys bit puts the index into EXPLAIN's - possible_keys. - */ - for (JOIN_TAB *s= stat ; s < stat_end ; s++) - { - Mvi_access *acc= join->get_mvi_access_for_table(s->table); - if (acc) - { - s->const_keys.set_bit(acc->index->keyno); - s->keys.set_bit(acc->index->keyno); - } - } - join->const_table_map= no_rows_const_tables; join->const_tables= const_count; eliminate_tables(join); @@ -6275,6 +6258,12 @@ make_join_statistics(JOIN *join, List &tables_list, */ add_group_and_distinct_keys(join, s); + /* + Same for the multi-valued index this table can be read through: a + fulltext key never gets a bit of its own. + */ + setup_mvi_access_for_table(join, s); + /* This will be updated in calculate_cond_selectivity_for_table() */ s->table->set_cond_selectivity(1.0); DBUG_ASSERT(s->table->used_stat_records == 0 || @@ -6313,7 +6302,7 @@ make_join_statistics(JOIN *join, List &tables_list, (SORT_INFO*) 0, 1, &error); if (!select) goto error; - if (get_quick_record_count(join->thd, join, select, s->table, + if (get_quick_record_count(join->thd, select, s, &s->const_keys, join->row_limit, &records)) { /* There was an error in test_quick_select */ diff --git a/sql/sql_select.h b/sql/sql_select.h index 0b517076580d5..45fec98adf7f7 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -566,6 +566,11 @@ typedef struct st_join_table { key_map checked_keys; /**< Keys checked in find_best */ key_map needed_reg; key_map keys; /**< all keys with can be used */ + /* + The multi-valued index access to use for this table, or NULL if there is + none. Set by setup_mvi_access_for_table(). + */ + Mvi_access *mvi_access; /* Either #rows in the table or 1 for const table. */ ha_rows records; @@ -1793,8 +1798,8 @@ class JOIN :public Sql_alloc SELECT_LEX *select_lex; /* The result of the multi-valued index analysis, or NULL if there is no - usable MVI access. Produced by setup_mvi_quick(), used by - get_best_mvi_access() during range analysis. + usable MVI access. Produced by setup_mvi_quick(); the access each table + gets out of it is picked by setup_mvi_access_for_table(). */ Mvi_context *mvi_ctx; /** @@ -2011,9 +2016,6 @@ class JOIN :public Sql_alloc void init(THD *thd_arg, List &fields_arg, ulonglong select_options_arg, select_result *result_arg); - /* Return the MVI access chosen for `table', or NULL if there is none */ - Mvi_access *get_mvi_access_for_table(TABLE *table); - /* True if the plan guarantees that it will be returned zero or one row */ bool only_const_tables() { return const_tables == table_count; } /* Number of tables actually joined at the top level */ From cc80405b9a36d27df477e18922683cc6a7c5b2b4 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 13:01:20 +0300 Subject: [PATCH 19/39] Do the MVI analysis one table at a time setup_mvi_quick() ran once per JOIN, from JOIN::optimize_inner(): it collected the MV indexes of every leaf table into one list, walked the WHERE clause once, and left the accesses it found in JOIN::mvi_ctx for setup_mvi_access_for_table() to dig through, filtering by access->index->vcol->table == tab->table. Nothing about it needed to be JOIN-wide. Each JOIN_TAB now has its own Mvi_context describing the access to its own table, and everything setup_mvi_quick() did happens in setup_mvi_access_for_table(): collect that table's MV indexes, analyze the condition, pick the access. The context is only kept when there is an access to use, so the chosen one is Mvi_context::best and JOIN_TAB has a single MVI member. The table filter becomes a DBUG_ASSERT: ctx->indexes holds only this table's indexes and get_mvi_index() matches the predicate against those with Item::eq(), which compares Field pointers, so an access can only ever be on the table whose column the predicate names - even when two tables carry identical MVI definitions. The condition to analyze comes from get_sargable_cond(), the same one the range analysis of that table uses twenty lines later. For a table on the inner side of an outer join that is the ON expression rather than the WHERE clause, so MVI access now works there too. It is sound for the same reason the range optimizer may do it: the scan is a necessary, not a sufficient condition, the JSON predicate stays in the ON expression and does the exact filtering, and outer rows that find no match are NULL-complemented as usual. The new test checks both, against the same queries with IGNORE INDEX. Two consequences of the analysis no longer being a pre-pass: - It runs on the condition the range optimizer will see, after simplify_joins(), substitute_indexed_vcols_for_join() and optimize_cond(), rather than on the freshly parsed WHERE. - Const tables are not analyzed at all, the per-table loop having skipped them before this point. They never get range analysis. JOIN_TAB::mvi_ctx also needs no explicit reset between re-optimizations: make_join_statistics() bzeroes the JOIN_TAB array. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 55 +++++++++ mysql-test/main/multi_valued_index.test | 32 ++++++ sql/opt_multi_valued_index.cc | 131 ++++++++++------------ sql/opt_multi_valued_index.h | 19 ++-- sql/sql_select.cc | 13 +-- sql/sql_select.h | 17 +-- 6 files changed, 169 insertions(+), 98 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 8e00696c34feb..5fb3c830b38a3 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -242,6 +242,61 @@ c j c j 1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} 4 {"tags": ["aaa","bbb"]} drop table t1; +# The predicate does not have to be in the WHERE clause: for a table on +# the inner side of an outer join we look at the ON expression, which is +# what has to be true for the rows we read. +create table t0 (a int); +insert into t0 values (1),(2),(3); +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); +explain select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t0 ALL NULL NULL NULL NULL 3 +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using join buffer (flat, BNL join) +select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +a c j +1 1 {"tags": ["aaa"]} +1 3 {"tags": ["aaa","bbb"]} +2 1 {"tags": ["aaa"]} +2 3 {"tags": ["aaa","bbb"]} +3 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t0 left join t1 ignore index(idx) +on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +a c j +1 1 {"tags": ["aaa"]} +1 3 {"tags": ["aaa","bbb"]} +2 1 {"tags": ["aaa"]} +2 3 {"tags": ["aaa","bbb"]} +3 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +# The outer rows that find no match are still NULL-complemented +explain select * from t0 left join t1 +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t0 ALL NULL NULL NULL NULL 3 +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using join buffer (flat, BNL join) +select * from t0 left join t1 +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +a c j +1 1 {"tags": ["aaa"]} +2 NULL NULL +3 3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t0 left join t1 ignore index(idx) +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +a c j +1 1 {"tags": ["aaa"]} +2 NULL NULL +3 3 {"tags": ["aaa","bbb"]} +drop table t0, t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 85cb9bd674701..9556f61540fec 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -159,6 +159,38 @@ where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; drop table t1; +--echo # The predicate does not have to be in the WHERE clause: for a table on +--echo # the inner side of an outer join we look at the ON expression, which is +--echo # what has to be true for the rows we read. + +create table t0 (a int); +insert into t0 values (1),(2),(3); +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); + +explain select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"'); +select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +--echo # the same rows without the index: +select * from t0 left join t1 ignore index(idx) + on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; + +--echo # The outer rows that find no match are still NULL-complemented +explain select * from t0 left join t1 + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a; +select * from t0 left join t1 + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +--echo # the same rows without the index: +select * from t0 left join t1 ignore index(idx) + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; + +drop table t0, t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 98476e8255be2..c323d2f26c39e 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -233,39 +233,31 @@ bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) return false; } -/* Collect all the MVI indexes in `join' */ +/* Collect all the MVI indexes of `table' */ static -bool collect_mvi_vcols_for_join(JOIN *join, List *indexes) +bool collect_mvi_vcols_for_table(THD *thd, TABLE *table, + List *indexes) { - List_iterator ti(join->select_lex->leaf_tables); - TABLE_LIST *tl; - TABLE *table; - THD *thd= join->thd; - while ((tl= ti++)) + for (uint i=0; i < table->s->keys; i++) { - if (!(table= tl->table)) // non-merged semi-join or something like that + if (!table->keys_in_use_for_query.is_set(i)) continue; - for (uint i=0; i < table->s->keys; i++) - { - if (!table->keys_in_use_for_query.is_set(i)) - continue; - KEY *key= &table->key_info[i]; - for (uint kp=0; kp < key->user_defined_key_parts; kp++) + KEY *key= &table->key_info[i]; + for (uint kp=0; kp < key->user_defined_key_parts; kp++) + { + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy)) continue; + Field *field= key->key_part[kp].field; + if (field->invisible == INVISIBLE_FULL && + field->vcol_info && + field->vcol_info->expr->type() == Item::FUNC_ITEM && + ((Item_func *) field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC) { - /* TODO: "legacy" */ - if (!(key->flags & HA_FULLTEXT_legacy)) continue; - Field *field= key->key_part[kp].field; - if (field->invisible == INVISIBLE_FULL && - field->vcol_info && - field->vcol_info->expr->type() == Item::FUNC_ITEM && - ((Item_func *) field->vcol_info->expr)->functype() == - Item_func::MVI_ENCODE_FUNC) - { - Mv_index *index= new (thd->mem_root) Mv_index(field, i); - if (indexes->push_back(index)) - return TRUE; // Out of memory - } + Mv_index *index= new (thd->mem_root) Mv_index(field, i); + if (indexes->push_back(index)) + return TRUE; // Out of memory } } } @@ -485,53 +477,45 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) /* @brief - Analyze the WHERE clause and find the MVI accesses it allows. + Analyze `cond' and pick the MVI access `tab' will use, if any, and let + the range analysis see it. + + @param cond The condition the rows of this table have to satisfy: the + WHERE clause, or the ON expression when the table is on the + inner side of an outer join. That is what the range analysis + of this table uses, too. @detail - The accesses are saved in join->mvi_ctx, where setup_mvi_access_for_table() - picks them up, one table at a time. + The analysis is what tab->mvi_ctx ends up holding: the MV indexes of the + table, the accesses the condition allows on them, and the one of those we + are going to use. + + A fulltext key never gets a bit in const_keys or keys, so we set them + here. The const_keys bit is what makes the range analysis run for this + table, where get_best_mvi_access() turns the access into a quick select; + the keys bit puts the index into EXPLAIN's possible_keys. + + @return + true Out of memory + false Ok, tab->mvi_ctx is set if the table has an MVI access */ -bool setup_mvi_quick(JOIN *join) +bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) { - THD *thd= join->thd; Mvi_context *ctx; - /* mvi_ctx must describe this analysis only, including on the early exits */ - join->mvi_ctx= NULL; - if (!join->conds) + if (!cond) return false; if (!(ctx= new (thd->mem_root) Mvi_context(thd))) return true; - if (collect_mvi_vcols_for_join(join, &ctx->indexes)) + if (collect_mvi_vcols_for_table(thd, tab->table, &ctx->indexes)) return true; + /* Most tables have no MVI. Leave before we walk the condition */ if (ctx->indexes.is_empty()) return false; - if (collect_mvi_accesses(ctx, join->conds)) + if (collect_mvi_accesses(ctx, cond)) return true; - if (ctx->accesses.is_empty()) - return false; - join->mvi_ctx= ctx; - return false; -} - - -/* - @brief - Pick the MVI access `tab' will use out of the ones the WHERE clause - allows, and let the range analysis see it. - @detail - A fulltext key never gets a bit in const_keys or keys, so we set them - here. The const_keys bit is what makes the range analysis run for this - table, where get_best_mvi_access() turns the access into a quick select; - the keys bit puts the index into EXPLAIN's possible_keys. -*/ - -void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab) -{ - if (!join->mvi_ctx) - return; - List_iterator it(join->mvi_ctx->accesses); + List_iterator it(ctx->accesses); /* TODO: cost based */ /* TODO: merge @@ -543,14 +527,20 @@ void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab) */ while (Mvi_access *access= it++) { - if (access->index->vcol->table == tab->table) - tab->mvi_access= access; - } - if (tab->mvi_access) - { - tab->const_keys.set_bit(tab->mvi_access->index->keyno); - tab->keys.set_bit(tab->mvi_access->index->keyno); + /* + An access can only be on this table: ctx->indexes holds this table's + indexes and get_mvi_index() matches the predicate against those. + */ + DBUG_ASSERT(access->index->vcol->table == tab->table); + ctx->best= access; } + if (!ctx->best) + return false; + + tab->mvi_ctx= ctx; + tab->const_keys.set_bit(ctx->best->index->keyno); + tab->keys.set_bit(ctx->best->index->keyno); + return false; } @@ -566,10 +556,13 @@ void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab) QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) { - Mvi_access *access= tab->mvi_access; TABLE *table= tab->table; - if (!access) + Mvi_access *access; + if (!tab->mvi_ctx) return NULL; + /* We only keep the context when it has an access for us to use */ + access= tab->mvi_ctx->best; + DBUG_ASSERT(access); /* estimate_records() drops element keys from the access, so it must run only once even if we are called again for the same table. diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index d96a31bd7e657..87085e72f51da 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -60,17 +60,19 @@ struct Mvi_access : public Sql_alloc }; -/* The result of the MVI analysis of one JOIN */ +/* The result of the MVI analysis of one table */ class Mvi_context : public Sql_alloc { public: THD *thd; - /* All MV indexes in the JOIN */ + /* The MV indexes of the table */ List indexes; - /* MVI accesses for all eligible predicates in WHERE */ + /* MVI accesses for all eligible predicates on the table */ List accesses; + /* The access we've chosen out of the above */ + Mvi_access *best; - Mvi_context(THD *thd_arg) : thd(thd_arg) {} + Mvi_context(THD *thd_arg) : thd(thd_arg), best(NULL) {} }; /* Return the compatible json type */ @@ -84,10 +86,11 @@ enum json_value_types mvi_json_class(enum_field_types ftype); bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, CHARSET_INFO *cs, String *buf); -bool setup_mvi_quick(JOIN *join); - -/* Pick the MVI access `tab' will use, and let the range analysis see it */ -void setup_mvi_access_for_table(JOIN *join, JOIN_TAB *tab); +/* + Analyze `cond' and pick the MVI access `tab' will use, if any, and let the + range analysis see it +*/ +bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond); /* Create a quick select for the MVI access to `tab', if there is one */ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 00a213a1d7b04..bdc1f67862f42 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -510,7 +510,6 @@ void JOIN::init(THD *thd_arg, List &fields_arg, result= result_arg; lock= thd_arg->lock; select_lex= 0; //for safety - mvi_ctx= 0; select_distinct= MY_TEST(select_options & SELECT_DISTINCT); no_order= 0; simple_order= 0; @@ -2310,12 +2309,6 @@ JOIN::optimize_inner() optimize_schema_tables_memory_usage(select_lex->leaf_tables)) DBUG_RETURN(1); - if (setup_mvi_quick(this)) - { - error= 1; - DBUG_RETURN(1); - } - if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */ DBUG_RETURN(-1); @@ -6260,9 +6253,11 @@ make_join_statistics(JOIN *join, List &tables_list, /* Same for the multi-valued index this table can be read through: a - fulltext key never gets a bit of its own. + fulltext key never gets a bit of its own. Use the same condition the + range analysis below will use. */ - setup_mvi_access_for_table(join, s); + if (setup_mvi_access_for_table(thd, s, *get_sargable_cond(join, s->table))) + goto error; /* This will be updated in calculate_cond_selectivity_for_table() */ s->table->set_cond_selectivity(1.0); diff --git a/sql/sql_select.h b/sql/sql_select.h index 45fec98adf7f7..2e121aec363fa 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -35,6 +35,8 @@ #include "cset_narrowing.h" typedef struct st_join_table JOIN_TAB; +class Mvi_context; +struct Mvi_access; /* Values in optimize */ #define KEY_OPTIMIZE_EXISTS 1U #define KEY_OPTIMIZE_REF_OR_NULL 2U @@ -567,10 +569,10 @@ typedef struct st_join_table { key_map needed_reg; key_map keys; /**< all keys with can be used */ /* - The multi-valued index access to use for this table, or NULL if there is - none. Set by setup_mvi_access_for_table(). + The multi-valued index analysis of this table, or NULL if the table has + no MVI access. Produced by setup_mvi_access_for_table(). */ - Mvi_access *mvi_access; + Mvi_context *mvi_ctx; /* Either #rows in the table or 1 for const table. */ ha_rows records; @@ -1458,9 +1460,6 @@ class AGGR_OP :public Sql_alloc }; -class Mvi_context; -struct Mvi_access; - class JOIN :public Sql_alloc { private: @@ -1796,12 +1795,6 @@ class JOIN :public Sql_alloc SELECT_LEX_UNIT *unit; /// select that processed SELECT_LEX *select_lex; - /* - The result of the multi-valued index analysis, or NULL if there is no - usable MVI access. Produced by setup_mvi_quick(); the access each table - gets out of it is picked by setup_mvi_access_for_table(). - */ - Mvi_context *mvi_ctx; /** TRUE <=> optimizer must not mark any table as a constant table. This is needed for subqueries in form "a IN (SELECT .. UNION SELECT ..): From d4c8ff43ef7b744acc59f97bfa5ee3ec132a15c4 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 13:06:05 +0300 Subject: [PATCH 20/39] Make innodb_fts.estimate's clamped count deterministic The "deleted rows are still counted" case failed about one full-suite run in three: -Note 1105 fulltext_estimate('gamma')= 4 +Note 1105 fulltext_estimate('gamma')= 5 The number it checks is the clamp in ha_innobase::fulltext_estimate(), which is dict_table_get_n_rows() - the table statistics. The DELETE just before it changes half the rows, which queues a background statistics recalculation, and whether that has run by the time of the SELECT depends on how loaded the machine is. ANALYZE TABLE after the DELETE recalculates them on the spot and clears the counter that would have triggered the background one, so the clamp has one value to report. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/suite/innodb_fts/r/estimate.result | 5 +++++ mysql-test/suite/innodb_fts/t/estimate.test | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/mysql-test/suite/innodb_fts/r/estimate.result b/mysql-test/suite/innodb_fts/r/estimate.result index 39565fdce9de4..32e8ecedd1a15 100644 --- a/mysql-test/suite/innodb_fts/r/estimate.result +++ b/mysql-test/suite/innodb_fts/r/estimate.result @@ -135,6 +135,11 @@ test.t1 optimize status OK # reported -- the clamp, not the deletions, is what moved the number. # DELETE FROM t1 WHERE id > 4; +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze Warning Engine-independent statistics are not collected for column 'a' +test.t1 analyze status OK SET debug_dbug='+d,fulltext_estimate'; SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); COUNT(*) diff --git a/mysql-test/suite/innodb_fts/t/estimate.test b/mysql-test/suite/innodb_fts/t/estimate.test index 2cb489bd1d82e..77f188dfae25c 100644 --- a/mysql-test/suite/innodb_fts/t/estimate.test +++ b/mysql-test/suite/innodb_fts/t/estimate.test @@ -101,6 +101,10 @@ OPTIMIZE TABLE t1; --echo # reported -- the clamp, not the deletions, is what moved the number. --echo # DELETE FROM t1 WHERE id > 4; +# The clamp uses the table statistics, which a background recalculation may +# still be about to redo after that DELETE. Recalculate them here instead, +# so the number below does not depend on when that happens. +ANALYZE TABLE t1; SET debug_dbug='+d,fulltext_estimate'; SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); SET debug_dbug=''; From aef91d6cecb4354ddacac576df9b25c41d63cc4e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 13:21:54 +0300 Subject: [PATCH 21/39] Rename collect_mvi_vcols_for_table to collect_mvi_indexes_for_table It collects Mv_index objects, not vcols. Co-Authored-By: Claude Opus 5 (1M context) --- sql/opt_multi_valued_index.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index c323d2f26c39e..7631f75409dfb 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -235,8 +235,8 @@ bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) /* Collect all the MVI indexes of `table' */ static -bool collect_mvi_vcols_for_table(THD *thd, TABLE *table, - List *indexes) +bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, + List *indexes) { for (uint i=0; i < table->s->keys; i++) { @@ -507,7 +507,7 @@ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) return false; if (!(ctx= new (thd->mem_root) Mvi_context(thd))) return true; - if (collect_mvi_vcols_for_table(thd, tab->table, &ctx->indexes)) + if (collect_mvi_indexes_for_table(thd, tab->table, &ctx->indexes)) return true; /* Most tables have no MVI. Leave before we walk the condition */ if (ctx->indexes.is_empty()) From aab99cd471acb490a8919e3e3374dff52d6db66a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 14:00:10 +0300 Subject: [PATCH 22/39] Keep only the chosen Mvi_access in JOIN_TAB JOIN_TAB held the whole Mvi_context the analysis produced, but outside setup_mvi_access_for_table() the only thing ever read out of it was mvi_ctx->best. The rest is scratch: indexes feeds get_mvi_index(), accesses is what the last-wins loop picks best out of, and thd is there for the mvi_analyze() callbacks. So JOIN_TAB keeps the access itself, and the context becomes a local of setup_mvi_access_for_table() - which is what the TODO there asked for: nothing is allocated for a table that has no MVI key, or whose condition yields no access. The access outliving the context is safe because neither it nor the Mv_index it refers to belongs to the context: both are allocated on the MEM_ROOT, and the lists only hold link nodes. Co-Authored-By: Claude Opus 5 (1M context) --- sql/opt_multi_valued_index.cc | 40 ++++++++++++++++------------------- sql/opt_multi_valued_index.h | 10 +++++---- sql/sql_select.h | 7 +++--- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 7631f75409dfb..f9776cb68f3b1 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -486,9 +486,9 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) of this table uses, too. @detail - The analysis is what tab->mvi_ctx ends up holding: the MV indexes of the - table, the accesses the condition allows on them, and the one of those we - are going to use. + The analysis itself is scratch state: what we leave behind is the one + access we've settled on, in tab->mvi_access. It and the Mv_index it + refers to live on the MEM_ROOT, so they outlive `ctx'. A fulltext key never gets a bit in const_keys or keys, so we set them here. The const_keys bit is what makes the range analysis run for this @@ -497,25 +497,24 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) @return true Out of memory - false Ok, tab->mvi_ctx is set if the table has an MVI access + false Ok, tab->mvi_access is set if the table has an MVI access */ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) { - Mvi_context *ctx; + Mvi_context ctx(thd); + Mvi_access *best= NULL; if (!cond) return false; - if (!(ctx= new (thd->mem_root) Mvi_context(thd))) - return true; - if (collect_mvi_indexes_for_table(thd, tab->table, &ctx->indexes)) + if (collect_mvi_indexes_for_table(thd, tab->table, &ctx.indexes)) return true; /* Most tables have no MVI. Leave before we walk the condition */ - if (ctx->indexes.is_empty()) + if (ctx.indexes.is_empty()) return false; - if (collect_mvi_accesses(ctx, cond)) + if (collect_mvi_accesses(&ctx, cond)) return true; - List_iterator it(ctx->accesses); + List_iterator it(ctx.accesses); /* TODO: cost based */ /* TODO: merge @@ -528,18 +527,18 @@ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) while (Mvi_access *access= it++) { /* - An access can only be on this table: ctx->indexes holds this table's + An access can only be on this table: ctx.indexes holds this table's indexes and get_mvi_index() matches the predicate against those. */ DBUG_ASSERT(access->index->vcol->table == tab->table); - ctx->best= access; + best= access; } - if (!ctx->best) + if (!best) return false; - tab->mvi_ctx= ctx; - tab->const_keys.set_bit(ctx->best->index->keyno); - tab->keys.set_bit(ctx->best->index->keyno); + tab->mvi_access= best; + tab->const_keys.set_bit(best->index->keyno); + tab->keys.set_bit(best->index->keyno); return false; } @@ -557,12 +556,9 @@ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) { TABLE *table= tab->table; - Mvi_access *access; - if (!tab->mvi_ctx) + Mvi_access *access= tab->mvi_access; + if (!access) return NULL; - /* We only keep the context when it has an access for us to use */ - access= tab->mvi_ctx->best; - DBUG_ASSERT(access); /* estimate_records() drops element keys from the access, so it must run only once even if we are called again for the same table. diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 87085e72f51da..f38ae49629cb3 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -60,7 +60,11 @@ struct Mvi_access : public Sql_alloc }; -/* The result of the MVI analysis of one table */ +/* + The state of the MVI analysis of one table. It only lives for the duration + of setup_mvi_access_for_table(): the access that analysis settles on is + what outlives it. +*/ class Mvi_context : public Sql_alloc { public: @@ -69,10 +73,8 @@ class Mvi_context : public Sql_alloc List indexes; /* MVI accesses for all eligible predicates on the table */ List accesses; - /* The access we've chosen out of the above */ - Mvi_access *best; - Mvi_context(THD *thd_arg) : thd(thd_arg), best(NULL) {} + Mvi_context(THD *thd_arg) : thd(thd_arg) {} }; /* Return the compatible json type */ diff --git a/sql/sql_select.h b/sql/sql_select.h index 2e121aec363fa..a2cf078788883 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -35,7 +35,6 @@ #include "cset_narrowing.h" typedef struct st_join_table JOIN_TAB; -class Mvi_context; struct Mvi_access; /* Values in optimize */ #define KEY_OPTIMIZE_EXISTS 1U @@ -569,10 +568,10 @@ typedef struct st_join_table { key_map needed_reg; key_map keys; /**< all keys with can be used */ /* - The multi-valued index analysis of this table, or NULL if the table has - no MVI access. Produced by setup_mvi_access_for_table(). + The multi-valued index access to use for this table, or NULL if there is + none. Set by setup_mvi_access_for_table(). */ - Mvi_context *mvi_ctx; + Mvi_access *mvi_access; /* Either #rows in the table or 1 for const table. */ ha_rows records; From 2496f06156119254c0d846af72c543f8cce2f1fc Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 15:02:04 +0300 Subject: [PATCH 23/39] Show multi-valued indexes in SHOW CREATE TABLE create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY)))); printed the two columns and no index at all. Nobody had decided to hide it: it inherited invisibility from the internal column that backs it. The grammar makes DB_MVI_ INVISIBLE_FULL, init_from_binary_frm_image() turns a hidden key part into a hidden key, and store_create_info() skips keys with HA_INVISIBLE_KEY. Long unique hash keys - the other kind of key built over a column the user cannot name - are already exempted from that; exempt the multi-valued index the same way, and print it as KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) which is the expression it was declared with and all it takes to re-create it. The internal column stays out of the output: it cannot be printed as a column, since there is no syntax that would recreate the pairing. Item_func_mvi_encode::print() cannot produce that form. Its output is what pack_expression() puts in the FRM, and that is read back as a call of mvi_encode(), the only form the parser accepts outside an index definition. So the CAST spelling gets a printer of its own, sharing the type printing. SHOW INDEX and I_S.STATISTICS list the key now too - the key part is let through the invisibility filter - so they no longer need debug_dbug=test_invisible_index, and the tests stop setting it (it also injected a stray invisible1 column and key into their output). The same HA_INVISIBLE_KEY drove mysql_prepare_alter_table(), which drops such keys from the list of keys carried into the rebuilt table - and INVISIBLE_FULL columns from the list of columns. So ALTER TABLE t1 ADD COLUMN x INT; silently dropped the index. The key survives now, and its column is carried over with it, for exactly as long as the key lives: DROP KEY takes the column with it, so the name is free again afterwards. While at it, make_internal_field_name() looped forever when create_list is empty: dup_found started at true and the loop that clears it does not run. The MVI path is the only caller that can hit that, and it does - with ALTER TABLE ... ADD KEY ((CAST(... ARRAY))), which used to hang the server and now works. A fulltext key over several arrays, KEY idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY))) has no single expression to print and no syntax of its own to be read back, so it stays hidden, exactly as before. The optimizer still uses its parts. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 95 +++++++++++- mysql-test/main/multi_valued_index.test | 48 ++++++- .../multi_valued_index_notembedded.result | 1 - .../main/multi_valued_index_notembedded.test | 2 - sql/item_strfunc.h | 8 ++ sql/opt_multi_valued_index.cc | 135 +++++++++++++++--- sql/opt_multi_valued_index.h | 14 ++ sql/sql_show.cc | 15 ++ sql/sql_table.cc | 61 +++++++- sql/table.cc | 16 ++- 10 files changed, 363 insertions(+), 32 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 5fb3c830b38a3..3b48f0a32e0a0 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -1,15 +1,14 @@ # basic test -SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( `c` int(11) DEFAULT NULL, - `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored -t1 1 invisible1 1 invisible1 A 0 NULL NULL YES BTREE NO t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; set global innodb_ft_aux_table='test/t1'; @@ -297,6 +296,96 @@ a c j 2 NULL NULL 3 3 {"tags": ["aaa","bbb"]} drop table t0, t1; +# SHOW CREATE TABLE prints the expression the index was declared with. +# The column that holds the index keys is internal and is not printed. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +# That output is all it takes to re-create the index: feed it back in +create table t2 ( +`c` int(11) DEFAULT NULL, +`j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL +CHECK (json_valid(`j`)), +KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t2; +# An ALTER TABLE that rebuilds the table keeps the index +alter table t1 add column x int; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j x +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +# ... and it can be dropped and added by name like any other index +alter table t1 drop key idx; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +alter table t1 add key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL, + KEY `idx2` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx2 idx2 0 NULL 2 Using where +drop table t1; +# A fulltext key over several arrays has no single expression to show, +# so it stays hidden. +create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), +(CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 9556f61540fec..f297f51232ad2 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -1,8 +1,6 @@ ---source include/have_debug.inc --source include/have_innodb.inc --echo # basic test -SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; SHOW CREATE TABLE t1; @@ -191,6 +189,52 @@ order by t0.a; drop table t0, t1; +--echo # SHOW CREATE TABLE prints the expression the index was declared with. +--echo # The column that holds the index keys is internal and is not printed. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); +show create table t1; +show index from t1; + +--echo # That output is all it takes to re-create the index: feed it back in +create table t2 ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB; +show create table t2; +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +drop table t2; + +--echo # An ALTER TABLE that rebuilds the table keeps the index +alter table t1 add column x int; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo # ... and it can be dropped and added by name like any other index +alter table t1 drop key idx; +show create table t1; +alter table t1 add key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))); +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); + +drop table t1; + +--echo # A fulltext key over several arrays has no single expression to show, +--echo # so it stays hidden. +create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), + (CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; +show create table t1; +show index from t1; +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result index f9e7bf5bd5256..d9cfb56d503be 100644 --- a/mysql-test/main/multi_valued_index_notembedded.result +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -1,4 +1,3 @@ -SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb","ccc"]}'); diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test index 5dccb0c0b63e0..a8f3a2de96918 100644 --- a/mysql-test/main/multi_valued_index_notembedded.test +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -1,9 +1,7 @@ ---source include/have_debug.inc --source include/have_innodb.inc # The test uses the optimizer trace: --source include/not_embedded.inc -SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 3cb20823a36b3..f092e101d237d 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2672,8 +2672,16 @@ class Item_func_mvi_encode : public Item_str_ascii_func Lex_cast_type_st m_cast_type; String tmp_js; json_engine_t je; + /* Print the type the values are cast to, as CAST() spells it */ + void append_cast_type(String *str); public: void print(String *str, enum_query_type query_type) override; + /* + Print as the CAST(... AS ... ARRAY) the index was declared with, for + SHOW CREATE TABLE. print() cannot do this: what it produces goes into + the FRM, and that is parsed back as a call of this function. + */ + void print_as_array_cast(String *str); Item_func_mvi_encode(THD* thd, Item *expr, const Lex_cast_type_st &cast_type): Item_str_ascii_func(thd, expr), m_cast_type(cast_type) {} String *val_str_ascii(String *buf) override; diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index f9776cb68f3b1..15622a952a30c 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -21,14 +21,10 @@ static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, Mvi_access *access); -void Item_func_mvi_encode::print(String *str, enum_query_type query_type) +void Item_func_mvi_encode::append_cast_type(String *str) { char buf[32]; size_t length; - str->append(func_name_cstring()); - str->append('('); - args[0]->print(str, query_type); - str->append(','); const Name name= m_cast_type.type_handler()->name(); switch (m_cast_type.type_handler()->field_type()) { @@ -51,9 +47,43 @@ void Item_func_mvi_encode::print(String *str, enum_query_type query_type) str->append(buf, length); str->append(')'); } +} + + +void Item_func_mvi_encode::print(String *str, enum_query_type query_type) +{ + str->append(func_name_cstring()); + str->append('('); + args[0]->print(str, query_type); + str->append(','); + append_cast_type(str); str->append(')'); } + +/* + @brief + Print the index expression the way it was written: + + CAST( AS ARRAY) + + @detail + print() cannot do this. Its output is what pack_expression() writes into + the FRM, and that is read back as a call of mvi_encode(), which is the + only form the parser accepts outside an index definition. +*/ + +void Item_func_mvi_encode::print_as_array_cast(String *str) +{ + str->append(STRING_WITH_LEN("cast(")); + /* The same flags the other parts of a table definition are printed with */ + args[0]->print_for_table_def(str); + str->append(STRING_WITH_LEN(" as ")); + append_cast_type(str); + str->append(STRING_WITH_LEN(" array)")); +} + + /* TODO: this duplicates logic in Item_func_json_extract::val_int */ static longlong json_value_to_longlong(enum json_value_types type, CHARSET_INFO *cs, @@ -233,6 +263,81 @@ bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) return false; } + +/* + @brief + If `field' is the internal column that holds the keys of a multi-valued + index, return the mvi_encode() call that computes them. + + @detail + Only the multi-valued index DDL creates a hidden column computed by + MVI_ENCODE(), so this identifies one for certain. +*/ + +static Item_func_mvi_encode *mvi_vcol_expr(const Field *field) +{ + Item *expr; + if (field->invisible != INVISIBLE_FULL || !field->vcol_info || + !(expr= field->vcol_info->expr) || + expr->type() != Item::FUNC_ITEM || + ((Item_func *) expr)->functype() != Item_func::MVI_ENCODE_FUNC) + return NULL; + return (Item_func_mvi_encode *) expr; +} + + +bool is_mvi_vcol(const Field *field) +{ + return mvi_vcol_expr(field) != NULL; +} + + +/* + @brief + Is key #keyno of `table' a multi-valued index, that is, a fulltext key + over one internal MVI column? + + @detail + A fulltext key can be declared over several of them: + + KEY idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), + (CAST(j->'$.b' AS CHAR(6) ARRAY))) + + Such a key has no single defining expression to show and no syntax of + its own to be read back in, so it does not count as one here. The + optimizer still uses each of its parts, see + collect_mvi_indexes_for_table(). +*/ + +static Item_func_mvi_encode *mvi_key_expr(const TABLE *table, uint keyno) +{ + KEY *key= table->s->key_info + keyno; + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy) || key->user_defined_key_parts != 1) + return NULL; + /* + Take the field from the TABLE and not from the key part: the share's + Field objects have no expression, parse_vcol_defs() builds one for each + TABLE of the share. + */ + return mvi_vcol_expr(table->field[key->key_part[0].fieldnr - 1]); +} + + +bool is_mvi_key(const TABLE *table, uint keyno) +{ + return mvi_key_expr(table, keyno) != NULL; +} + + +void print_mvi_key_expr(String *str, const TABLE *table, uint keyno) +{ + Item_func_mvi_encode *mvi= mvi_key_expr(table, keyno); + DBUG_ASSERT(mvi); + mvi->print_as_array_cast(str); +} + + /* Collect all the MVI indexes of `table' */ static bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, @@ -244,21 +349,17 @@ bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, continue; KEY *key= &table->key_info[i]; + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy)) + continue; for (uint kp=0; kp < key->user_defined_key_parts; kp++) { - /* TODO: "legacy" */ - if (!(key->flags & HA_FULLTEXT_legacy)) continue; Field *field= key->key_part[kp].field; - if (field->invisible == INVISIBLE_FULL && - field->vcol_info && - field->vcol_info->expr->type() == Item::FUNC_ITEM && - ((Item_func *) field->vcol_info->expr)->functype() == - Item_func::MVI_ENCODE_FUNC) - { - Mv_index *index= new (thd->mem_root) Mv_index(field, i); - if (indexes->push_back(index)) - return TRUE; // Out of memory - } + if (!is_mvi_vcol(field)) + continue; + Mv_index *index= new (thd->mem_root) Mv_index(field, i); + if (indexes->push_back(index)) + return TRUE; // Out of memory } } return FALSE; // Ok diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index f38ae49629cb3..c76ae06fb296d 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -88,6 +88,20 @@ enum json_value_types mvi_json_class(enum_field_types ftype); bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, CHARSET_INFO *cs, String *buf); +/* + Is `field' the internal column that holds the keys of a multi-valued index? +*/ +bool is_mvi_vcol(const Field *field); + +/* Is key #keyno of `table' a multi-valued index? */ +bool is_mvi_key(const TABLE *table, uint keyno); + +/* + Print the expression key #keyno was declared with, in the CAST(... ARRAY) + form, for SHOW CREATE TABLE +*/ +void print_mvi_key_expr(String *str, const TABLE *table, uint keyno); + /* Analyze `cond' and pick the MVI access `tab' will use, if any, and let the range analysis see it diff --git a/sql/sql_show.cc b/sql/sql_show.cc index f716dc94e2b39..9bb6a0c205d5d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2476,6 +2476,20 @@ int show_create_table_ex(THD *thd, TABLE_LIST *table_list, const char *force_db, { if (key_info->flags & HA_INVISIBLE_KEY) continue; + if (is_mvi_key(table, i)) + { + /* + A multi-valued index. The column that holds its keys is internal and + is not printed, so print the expression the index was declared with + instead - that is also the only form that can be read back. + */ + packet->append(STRING_WITH_LEN(",\n KEY ")); + append_identifier(thd, packet, &key_info->name); + packet->append(STRING_WITH_LEN(" ((")); + print_mvi_key_expr(packet, table, i); + packet->append(STRING_WITH_LEN("))")); + continue; + } KEY_PART_INFO *key_part= key_info->key_part; bool found_primary=0; packet->append(STRING_WITH_LEN(",\n ")); @@ -7480,6 +7494,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, TABLE *table, for (uint j=0 ; j < key_info->user_defined_key_parts ; j++,key_part++) { if (key_part->field->invisible >= INVISIBLE_SYSTEM && + !is_mvi_key(show_table, i) && !DBUG_IF("test_completely_invisible")) { /* diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 1725697c65243..3210ed61a1f9c 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2778,13 +2778,19 @@ Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, { char buf[INTERNAL_FIELD_NAME_LENGTH]= {0}; LEX_CSTRING name= { buf, 0 }; - bool dup_found= true; - for (uint num= 1; dup_found; num++) + for (uint num= 1; ; num++) { + /* + Note this has to start at false: `create_list' can be empty, and then + the loop below does not run at all. + */ + bool dup_found= false; name.length= my_snprintf(buf, sizeof(buf), "%s%u", prefix, num); for (auto &dup_field : *create_list) if ((dup_found= dup_field.field_name.streq(name))) break; + if (!dup_found) + break; } return Lex_ident_column(thd->strmake_lex_cstring(name)); } @@ -8544,6 +8550,41 @@ void rename_field_in_list(Create_field *field, List *field_list) #endif +/* + @brief + Should `field', the internal column of a multi-valued index, survive this + ALTER TABLE? + + @detail + It only exists to hold the entries of one key, so it lives exactly as + long as that key does: a column whose key is being dropped goes with it, + and so does one that has no key left at all. +*/ + +static bool mvi_vcol_kept_by_alter(TABLE *table, Field *field, + Alter_info *alter_info) +{ + KEY *key_info= table->key_info; + if (!is_mvi_vcol(field)) + return false; + for (uint i= 0; i < table->s->total_keys; i++, key_info++) + { + if (!is_mvi_key(table, i) || key_info->key_part[0].field != field) + continue; + /* This is its key. Keep the column unless the key is going away */ + List_iterator drop_it(alter_info->drop_list); + while (Alter_drop *drop= drop_it++) + { + if (drop->type == Alter_drop::KEY && + Lex_ident_column(key_info->name).streq(drop->name)) + return false; + } + return true; + } + return false; +} + + /** Prepare column and key definitions for CREATE TABLE in ALTER TABLE. @@ -8710,7 +8751,14 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, bitmap_clear_all(&table->tmp_set); for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) { - if (field->invisible == INVISIBLE_FULL) + /* + Internal columns are re-created from scratch by the new table's DDL, + except the one that holds the keys of a multi-valued index: there is no + syntax that would re-create that one, so carry it over as it is, for as + long as its key is (see the key loop below). + */ + if (field->invisible == INVISIBLE_FULL && + !mvi_vcol_kept_by_alter(table, field, alter_info)) continue; Alter_drop *drop; if (field->type() == MYSQL_TYPE_VARCHAR) @@ -8894,7 +8942,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, } else { - DBUG_ASSERT(field->invisible == INVISIBLE_SYSTEM); + /* The internal column of a multi-valued index also goes last */ + DBUG_ASSERT(field->invisible == INVISIBLE_SYSTEM || is_mvi_vcol(field)); def= new (root) Create_field(thd, field, field); new_create_tail.push_back(def, root); } @@ -9112,6 +9161,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, for (uint i= 0; i < table->s->total_keys; i++, key_info++) { bool long_hash_key= false; + /* A multi-valued index. Its only key part is an internal column */ + const bool mvi_key= is_mvi_key(table, i); if (key_info->flags & HA_INVISIBLE_KEY) continue; Lex_ident_column key_name(key_info->name); @@ -9402,6 +9453,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key->without_overlaps= key_info->without_overlaps; key->period= table->s->period.name; key->old= true; + /* Let the key keep its internal key part, see init_key_part_spec() */ + key->invisible= mvi_key; new_key_list.push_back(key, root); } if (long_hash_key) diff --git a/sql/table.cc b/sql/table.cc index acc6377de1b50..db3c037ba8d9e 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3215,9 +3215,19 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->incompatible_version|= HA_CREATE_USED_CHARSET; key_part->type= field->key_type(); - if (field->invisible > INVISIBLE_USER && !field->vers_sys_field()) - if (keyinfo->algorithm != HA_KEY_ALG_LONG_HASH) - keyinfo->flags |= HA_INVISIBLE_KEY; + /* + A key part the user cannot name normally hides the whole key. Two + kinds of key are built that way on purpose and are not hidden: + a long unique, and a multi-valued index - a fulltext key over one + internal column holding the index keys. We cannot use is_mvi_key() + to recognize the latter: the vcol expressions are not parsed until + parse_vcol_defs(), long after this. + */ + if (field->invisible > INVISIBLE_USER && !field->vers_sys_field() && + keyinfo->algorithm != HA_KEY_ALG_LONG_HASH && + !(keyinfo->algorithm == HA_KEY_ALG_FULLTEXT && + keyinfo->user_defined_key_parts == 1)) + keyinfo->flags |= HA_INVISIBLE_KEY; if (field->null_ptr) { key_part->null_offset=(uint) ((uchar*) field->null_ptr - From 32833cdc4dd860b1f631cf98aae9421e837e2756 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 15:15:23 +0300 Subject: [PATCH 24/39] Only allow one key part in an index over an ARRAY create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY)))); was accepted without a word. Each ARRAY key part gets an internal column of its own, and they all became key parts of one fulltext key: the tokens of both arrays end up mixed in a single index, and the optimizer would then search that index for the keys of one array and get the rows of the other as well. There is also no way to show such a key, or to read one back. init_key_part_spec() now rejects a key that has an ARRAY key part and more than one key part, on both the CREATE TABLE and the ALTER TABLE path. The other order, KEY idx (c,(CAST(... ARRAY))), was already rejected: the ARRAY part makes the key FULLTEXT, and `c' cannot be part of one. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 24 +++++++++++------- mysql-test/main/multi_valued_index.test | 19 +++++++++++--- sql/opt_multi_valued_index.cc | 31 +++++++++++++---------- sql/opt_multi_valued_index.h | 1 + sql/sql_table.cc | 11 ++++++++ 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 3b48f0a32e0a0..b4554f03aa502 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -374,17 +374,23 @@ explain select * from t1 where json_contains(j->'$.tags','"aaa"'); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx2 idx2 0 NULL 2 Using where drop table t1; -# A fulltext key over several arrays has no single expression to show, -# so it stays hidden. +# An index over an ARRAY has exactly one key part. A key with several +# would have no defining expression to show and no way to be read back. create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci -show index from t1; -Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +ERROR 42000: Too many key parts specified; max 1 parts allowed +create table t1 (c int, j json, +key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; +ERROR 42000: Too many key parts specified; max 1 parts allowed +# The other order is rejected by the FULLTEXT check on `c' itself +create table t1 (c int, j json, +key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Column 'c' cannot be part of FULLTEXT index +# ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), +(CAST(j->'$.b' AS CHAR(6) ARRAY))); +ERROR 42000: Too many key parts specified; max 1 parts allowed drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index f297f51232ad2..ccadd4a3337cd 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -227,12 +227,23 @@ explain select * from t1 where json_contains(j->'$.tags','"aaa"'); drop table t1; ---echo # A fulltext key over several arrays has no single expression to show, ---echo # so it stays hidden. +--echo # An index over an ARRAY has exactly one key part. A key with several +--echo # would have no defining expression to show and no way to be read back. +--error ER_TOO_MANY_KEY_PARTS create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; -show create table t1; -show index from t1; +--error ER_TOO_MANY_KEY_PARTS +create table t1 (c int, j json, + key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; +--echo # The other order is rejected by the FULLTEXT check on `c' itself +--error ER_BAD_FT_COLUMN +create table t1 (c int, j json, + key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--echo # ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +--error ER_TOO_MANY_KEY_PARTS +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), + (CAST(j->'$.b' AS CHAR(6) ARRAY))); drop table t1; --echo # direct call of mvi_encode diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 15622a952a30c..c1068d041c710 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -274,11 +274,12 @@ bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) MVI_ENCODE(), so this identifies one for certain. */ -static Item_func_mvi_encode *mvi_vcol_expr(const Field *field) +static Item_func_mvi_encode *mvi_expr(field_visibility_t invisible, + const Virtual_column_info *vcol_info) { Item *expr; - if (field->invisible != INVISIBLE_FULL || !field->vcol_info || - !(expr= field->vcol_info->expr) || + if (invisible != INVISIBLE_FULL || !vcol_info || + !(expr= vcol_info->expr) || expr->type() != Item::FUNC_ITEM || ((Item_func *) expr)->functype() != Item_func::MVI_ENCODE_FUNC) return NULL; @@ -288,7 +289,14 @@ static Item_func_mvi_encode *mvi_vcol_expr(const Field *field) bool is_mvi_vcol(const Field *field) { - return mvi_vcol_expr(field) != NULL; + return mvi_expr(field->invisible, field->vcol_info) != NULL; +} + + +/* The same, on the way in: for a column that is being created */ +bool is_mvi_vcol(const Create_field *field) +{ + return mvi_expr(field->invisible, field->vcol_info) != NULL; } @@ -298,14 +306,10 @@ bool is_mvi_vcol(const Field *field) over one internal MVI column? @detail - A fulltext key can be declared over several of them: - - KEY idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), - (CAST(j->'$.b' AS CHAR(6) ARRAY))) - - Such a key has no single defining expression to show and no syntax of - its own to be read back in, so it does not count as one here. The - optimizer still uses each of its parts, see + init_key_part_spec() does not allow such a key to have more than one key + part. The check is here as well because a table created before it was + added may still have one, and there is no single expression to show for + it. The optimizer does use each of its parts, see collect_mvi_indexes_for_table(). */ @@ -320,7 +324,8 @@ static Item_func_mvi_encode *mvi_key_expr(const TABLE *table, uint keyno) Field objects have no expression, parse_vcol_defs() builds one for each TABLE of the share. */ - return mvi_vcol_expr(table->field[key->key_part[0].fieldnr - 1]); + Field *field= table->field[key->key_part[0].fieldnr - 1]; + return mvi_expr(field->invisible, field->vcol_info); } diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index c76ae06fb296d..42271773fa7cf 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -92,6 +92,7 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, Is `field' the internal column that holds the keys of a multi-valued index? */ bool is_mvi_vcol(const Field *field); +bool is_mvi_vcol(const Create_field *field); /* Is key #keyno of `table' a multi-valued index? */ bool is_mvi_key(const TABLE *table, uint keyno); diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 3210ed61a1f9c..3befe83a60d81 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2974,6 +2974,17 @@ my_bool init_key_part_spec(THD *thd, Alter_info *alter_info, DBUG_RETURN(TRUE); } + /* + An index over an ARRAY has exactly one key part. A key with several of + them has no defining expression to show in SHOW CREATE TABLE, and no + syntax of its own that would read it back in. + */ + if (is_mvi_vcol(column) && key.columns.elements != 1) + { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + DBUG_RETURN(TRUE); + } + const Type_handler *type_handler= column->type_handler(); switch(key.type) { From 3ff52728cc8d868cb6bd60f9280715a710f95612 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 7 Sep 2026 17:16:52 +0300 Subject: [PATCH 25/39] A plain KEY is the only index type allowed over an ARRAY create table t1 (j json, unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)))); create table t1 (j json, primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY)))); create table t1 (j json, fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)))); were all accepted without a word, and all produced the same thing: a plain index. The key type the user wrote was simply overwritten with Key::FULLTEXT, so the table ended up with no unique constraint, or no primary key, or with a FULLTEXT index that MATCH() finds nothing in - the index holds encoded element keys, not the text. What the server builds for an ARRAY is a fulltext index over those encoded elements, and it can only mean what a plain KEY means. Say so: reject any other type, in the grammar, before that overwrite loses what was asked for. CONSTRAINT ... UNIQUE and the ALTER TABLE forms go the same way. SPATIAL and VECTOR are already syntax errors for an ARRAY key part; they are in the switch anyway so it stays exhaustive. FOREIGN KEY is unaffected: it builds its key with Key::MULTIPLE and cannot be told apart here. It is rejected, further down, by the engine - "Foreign key constraint is incorrectly formed". The count of key parts is now also checked in the grammar, and not only in init_key_part_spec(). Otherwise the first ARRAY part of a two-part key sets the type to FULLTEXT, and the second part reports "Incorrect usage of FULLTEXT and ARRAY" for a key nobody declared FULLTEXT. As a side effect KEY idx (c,(CAST(... ARRAY))) now gives the same "max 1 parts" error as the other orders, instead of ER_BAD_FT_COLUMN for `c'. Co-Authored-By: Claude Opus 5 (1M context) --- mysql-test/main/multi_valued_index.result | 36 +++++++++++++++++++++-- mysql-test/main/multi_valued_index.test | 33 +++++++++++++++++++-- sql/sql_table.cc | 36 +++++++++++++++++++++++ sql/sql_table.h | 7 +++++ sql/sql_yacc.yy | 14 +++++++++ 5 files changed, 122 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index b4554f03aa502..7bad4a76db49a 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -382,16 +382,48 @@ ERROR 42000: Too many key parts specified; max 1 parts allowed create table t1 (c int, j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; ERROR 42000: Too many key parts specified; max 1 parts allowed -# The other order is rejected by the FULLTEXT check on `c' itself create table t1 (c int, j json, key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; -ERROR HY000: Column 'c' cannot be part of FULLTEXT index +ERROR 42000: Too many key parts specified; max 1 parts allowed # ... and the same through ALTER TABLE create table t1 (j json)engine=innodb; alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY))); ERROR 42000: Too many key parts specified; max 1 parts allowed drop table t1; +# A plain KEY is the only thing an index over an ARRAY can be. The +# index is a fulltext index over the encoded elements underneath, which +# would not enforce UNIQUE or PRIMARY KEY, and which MATCH() would find +# nothing in. +create table t1 (j json, +unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +create table t1 (j json, +constraint u unique ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +create table t1 (j json, +primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of PRIMARY KEY and ARRAY +create table t1 (j json, +fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of FULLTEXT and ARRAY +# ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +alter table t1 add unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +alter table t1 add primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of PRIMARY KEY and ARRAY +alter table t1 add fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of FULLTEXT and ARRAY +# the plain form is accepted +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.a') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index ccadd4a3337cd..cc6351a9a324b 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -235,8 +235,7 @@ create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), --error ER_TOO_MANY_KEY_PARTS create table t1 (c int, j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; ---echo # The other order is rejected by the FULLTEXT check on `c' itself ---error ER_BAD_FT_COLUMN +--error ER_TOO_MANY_KEY_PARTS create table t1 (c int, j json, key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; --echo # ... and the same through ALTER TABLE @@ -246,6 +245,36 @@ alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), (CAST(j->'$.b' AS CHAR(6) ARRAY))); drop table t1; +--echo # A plain KEY is the only thing an index over an ARRAY can be. The +--echo # index is a fulltext index over the encoded elements underneath, which +--echo # would not enforce UNIQUE or PRIMARY KEY, and which MATCH() would find +--echo # nothing in. +--error ER_WRONG_USAGE +create table t1 (j json, + unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + constraint u unique ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; + +--echo # ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +--error ER_WRONG_USAGE +alter table t1 add unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--error ER_WRONG_USAGE +alter table t1 add primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--error ER_WRONG_USAGE +alter table t1 add fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--echo # the plain form is accepted +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +show create table t1; +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 3befe83a60d81..1dc2b1da1384a 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2771,6 +2771,42 @@ static int mysql_add_invisible_field(THD *thd, List * field_list, } #endif +/* + @brief + Can an index over an ARRAY be of the type `key' was declared with? + + @detail + Only a plain KEY can. What the server builds is a fulltext index over + the encoded elements of the array, which does not implement what any of + the other types would promise: UNIQUE and PRIMARY KEY would not be + enforced, and MATCH() against a FULLTEXT one would find nothing. They + used to be accepted and quietly turned into a plain index. + + @return + true No, and an error is raised +*/ + +bool check_mvi_key_type(const Key *key) +{ + const char *type= NULL; + switch (key->type) { + case Key::PRIMARY: type= "PRIMARY KEY"; break; + case Key::UNIQUE: type= "UNIQUE"; break; + case Key::FULLTEXT: type= "FULLTEXT"; break; + case Key::SPATIAL: type= "SPATIAL"; break; + case Key::VECTOR: type= "VECTOR"; break; + case Key::MULTIPLE: /* A plain KEY: the only type an ARRAY can have */ + case Key::FOREIGN_KEY: /* Both of these are built with Key::MULTIPLE, so */ + case Key::IGNORE_KEY: /* they never reach us under their own name */ + break; + } + if (!type) + return false; + my_error(ER_WRONG_USAGE, MYF(0), type, "ARRAY"); + return true; +} + + #define INTERNAL_FIELD_NAME_LENGTH 30 Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, diff --git a/sql/sql_table.h b/sql/sql_table.h index 171a7aa075ec9..a1e5aeb04eae3 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -228,4 +228,11 @@ bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, List *create_list); + +class Key; +/* + Check that an index over an ARRAY may be of the type `key' was declared + with. Raises an error if it may not. +*/ +bool check_mvi_key_type(const Key *key); #endif /* SQL_TABLE_INCLUDED */ diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 78e7905e9d897..f0d83128d1932 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -7652,6 +7652,20 @@ key_part: multi_valued_key_part: '(' CAST_SYM '(' expr AS cast_type ARRAY_SYM ')' ')' { + /* + An index over an ARRAY has exactly one key part. Catch a second + one here, before the type of the key is overwritten below and + check_mvi_key_type() starts seeing FULLTEXT instead of what the + user wrote. A part that comes *after* the ARRAY one is caught + in init_key_part_spec(). + */ + if (unlikely(Lex->last_key->columns.elements)) + { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + MYSQL_YYABORT; + } + if (unlikely(check_mvi_key_type(Lex->last_key))) + MYSQL_YYABORT; /* TODO: check fts_min_token_size is 4, warn if not */ /* Create a Create_field */ Create_field *f= new (thd->mem_root) Create_field(); From bdf696e8e688da57d62abc2f5bc31244b96478db Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 8 Sep 2026 12:16:11 +1000 Subject: [PATCH 26/39] MDEV-40168 Fix cast to int arrays --- mysql-test/main/multi_valued_index.result | 38 ++++++++++++++++++++++- mysql-test/main/multi_valued_index.test | 22 ++++++++++++- sql/opt_multi_valued_index.cc | 6 ++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 7bad4a76db49a..4e75185a3f325 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -1,4 +1,4 @@ -# basic test +# basic tests create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; SHOW CREATE TABLE t1; Table Create Table @@ -65,6 +65,42 @@ c j 2 {"tags": ["1", "abcde", "", 34567]} DROP TABLE t1; set global innodb_ft_aux_table=@old_innodb_ft_aux_table; +create table t1 (j json, key idx ((CAST(j->'$.n' AS INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as int array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS SIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as int array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as unsigned array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as unsigned array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; # top level or create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; explain diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index cc6351a9a324b..76086acb727cc 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -1,6 +1,6 @@ --source include/have_innodb.inc ---echo # basic test +--echo # basic tests create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; SHOW CREATE TABLE t1; @@ -35,6 +35,26 @@ DROP TABLE t1; set global innodb_ft_aux_table=@old_innodb_ft_aux_table; +create table t1 (j json, key idx ((CAST(j->'$.n' AS INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS SIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + --echo # top level or create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index c1068d041c710..e1d7d3292aa16 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -35,6 +35,12 @@ void Item_func_mvi_encode::append_cast_type(String *str) str->append(buf, length); str->append(')'); break; + case MYSQL_TYPE_LONGLONG: + if (m_cast_type.type_handler()->is_unsigned()) + str->append(STRING_WITH_LEN("unsigned")); + else + str->append(STRING_WITH_LEN("int")); + break; default: str->append(name.ptr(), name.length()); break; From 5bf3ce53bb18d840a9b041031ba5c59a633df0af Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 8 Sep 2026 16:15:36 +1000 Subject: [PATCH 27/39] MDEV-40168 nested array handling and json validation --- mysql-test/main/multi_valued_index.result | 38 ++++++++++++++++++-- mysql-test/main/multi_valued_index.test | 15 ++++++++ sql/opt_multi_valued_index.cc | 44 +++++++++++++++-------- sql/opt_mvi_jsonfuncs.cc | 25 ++++++++++--- 4 files changed, 101 insertions(+), 21 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 4e75185a3f325..f1b748a110f1c 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -31,6 +31,17 @@ WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION xxxx 2 3 2 2 16 xxxx 2 3 2 3 18 insert into t1 values (4, '{}'); +insert into t1 values (5, '{"tags": [[["1"], "34567"]]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +312e30 3 3 1 3 0 +31xx 2 5 2 2 0 +31xx 2 5 2 5 0 +3334353637 3 5 2 3 7 +3334353637 3 5 2 5 5 +6162636465 2 2 1 2 5 +xxxx 2 3 2 2 16 +xxxx 2 3 2 3 18 explain select * from t1 where json_contains(j->'$.tags', '"abcde"'); id select_type table type possible_keys key key_len ref rows Extra @@ -47,6 +58,7 @@ c j select * from t1 where json_contains(j->'$.tags', '"1"'); c j 2 {"tags": ["1", "abcde", "", 34567]} +5 {"tags": [[["1"], "34567"]]} explain select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); id select_type table type possible_keys key key_len ref rows Extra @@ -63,6 +75,19 @@ c j select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); c j 2 {"tags": ["1", "abcde", "", 34567]} +explain +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +c j +5 {"tags": [[["1"], "34567"]]} +select * from t1 where json_contains(j->'$.tags', '[[["34567"], "1"]'); +c j +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_contains' +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1", {}}]'); +c j DROP TABLE t1; set global innodb_ft_aux_table=@old_innodb_ft_aux_table; create table t1 (j json, key idx ((CAST(j->'$.n' AS INT ARRAY)))); @@ -170,8 +195,6 @@ drop table t1,t2; # matches row 2, which the '"aaa"' index scan would not return. create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [{"a":1}]}'); -Warnings: -Warning 4204 Invalid vector format at offset: 3 for '[{"a": 1}]'. Must be a valid JSON array of numbers. prepare s from 'select * from t1 where json_contains(j->''$.tags'', ?)'; set @p='"aaa"'; execute s using @p; @@ -479,3 +502,14 @@ mvi_encode('[1, 42, "3 "]', binary(6)) select mvi_encode('[1, 42]', char(6)); mvi_encode('[1, 42]', char(6)) +select mvi_encode('[1, 42, {}}]', char(6)); +mvi_encode('[1, 42, {}}]', char(6)) +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'mvi_encode' at position 11 +select mvi_encode('[[1], 42]', int); +mvi_encode('[[1], 42]', int) +8000000000000001 800000000000002a +select mvi_encode('[1]]', int); +mvi_encode('[1]]', int) +8000000000000001 diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 76086acb727cc..7a8461285f049 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -16,6 +16,8 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; insert into t1 values (3, '{"tags": ["1.0", "34567", "", 34567]}'); SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; insert into t1 values (4, '{}'); +insert into t1 values (5, '{"tags": [[["1"], "34567"]]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; explain select * from t1 where json_contains(j->'$.tags', '"abcde"'); @@ -30,6 +32,13 @@ explain select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); +explain +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); + +# Invalid JSONs, errors not raised during optimization +select * from t1 where json_contains(j->'$.tags', '[[["34567"], "1"]'); +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1", {}}]'); DROP TABLE t1; @@ -302,3 +311,9 @@ select mvi_encode('[1, 42, "3 "]', char(6)); select mvi_encode('[1, 42, " "]', char(6)); select mvi_encode('[1, 42, "3 "]', binary(6)); select mvi_encode('[1, 42]', char(6)); +select mvi_encode('[1, 42, {}}]', char(6)); +select mvi_encode('[[1], 42]', int); +# trailing junk after outer array is accepted, consistent with normal +# JSON_CONTAINS behaviour. NOTE that mysql would fail `select +# JSON_CONTAINS('[1]]', '1');` but not mariadb +select mvi_encode('[1]]', int); diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index e1d7d3292aa16..946b553aed981 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -130,6 +130,7 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, { enum_field_types cast_ftype= cast_th->field_type(); bool is_unsigned= cast_th->is_unsigned(); + /* TODO: 42 hardcoded */ StringBuffer<42> sorted; /* Skip encoding on type incompatibility */ if (mvi_json_class(cast_ftype) != je->value_type) @@ -188,41 +189,46 @@ String *Item_func_mvi_encode::val_str_ascii(String *buf) return nullptr; CHARSET_INFO *cs= value->charset(); const Type_handler *cast_th= m_cast_type.type_handler(); - bool end_ok= false, at_least_one= false; + bool at_least_one= false; const uchar *start= reinterpret_cast(value->ptr()); const uchar *end= start + value->length(); + int depth= 0; DBUG_ASSERT(fixed()); buf->length(0); buf->set_charset(&my_charset_latin1_bin); - if (json_scan_start(&je, cs, start, end) || - json_read_value(&je)) + if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) goto json_error; if (je.value_type != JSON_VALUE_ARRAY) goto error_format; - /* TODO: deduplicate, so that ["34567", 34567] yield only one token */ + /* TODO: deduplicate, so that ["34567", "34567"] yield only one token */ do { switch (je.state) { case JST_ARRAY_START: + depth++; continue; case JST_ARRAY_END: - /* - TODO: do something different when an empty string is - returned, i.e. at_least_one == false to avoid wasting index - space? - */ - if (at_least_one) - buf->length(buf->length() - 1); - end_ok = true; + if (--depth == 0) + goto array_done; break; case JST_VALUE: { if (json_read_value(&je)) goto json_error; - + if (je.value_type == JSON_VALUE_ARRAY) + { + depth++; + break; + } + if (je.value_type == JSON_VALUE_OBJECT) + { + if (json_skip_level(&je)) + goto json_error; + break; + } if (!encode_mvi_key(&je, cast_th, cs, buf)) { buf->append(' '); @@ -234,9 +240,17 @@ String *Item_func_mvi_encode::val_str_ascii(String *buf) goto error_format; } } while (json_scan_next(&je) == 0); + goto json_error; - if (end_ok) - return buf; +array_done: + /* + TODO: do something different when an empty string is + returned, i.e. at_least_one == false to avoid wasting index + space? + */ + if (at_least_one) + buf->length(buf->length() - 1); + return buf; error_format: { diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc index 5f913b8fdd71f..b487370933913 100644 --- a/sql/opt_mvi_jsonfuncs.cc +++ b/sql/opt_mvi_jsonfuncs.cc @@ -91,6 +91,7 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, Item_func_mvi_encode *mvitem= (Item_func_mvi_encode *) index->vcol->vcol_info->expr; const Type_handler *cast_th= mvitem->cast_type().type_handler(); + int depth= 0; buf.length(0); buf.set_charset(&my_charset_latin1_bin); @@ -115,20 +116,36 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, // JSON_VALUE_ARRAY /* TODO: deduplicate? */ + /* + TODO: the logic here parallels + Item_func_mvi_encode::val_str_ascii. A refactoring is called for + */ do { buf.length(0); switch (je->state) { - /* TODO: nested array? */ case JST_ARRAY_START: - continue; + depth++; + break; case JST_ARRAY_END: + if (--depth == 0) + return access; break; case JST_VALUE: { if (json_read_value(je)) return NULL; - + if (je->state == JST_ARRAY_START) + { + depth++; + break; + } + if (je->value_type == JSON_VALUE_OBJECT) + { + if (json_skip_level(je) || !conjunctive) + return NULL; + break; + } if (encode_mvi_key(je, cast_th, cs, &buf)) { /* See above: only an AND of the keys tolerates a missing one */ @@ -148,7 +165,7 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, } } while (json_scan_next(je) == 0); - return access; + return depth > 0 ? NULL : access; } From a2d80ea2fcdfab52b219d2d9ffaf22ac8324372f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 8 Sep 2026 09:29:20 +0300 Subject: [PATCH 28/39] Undo whitespace changes to reduce diff size --- sql/item.h | 1 - sql/opt_range.h | 1 - sql/sql_explain.h | 2 +- sql/sql_select.h | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/item.h b/sql/item.h index e64a8fe689262..52114d6a8b140 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2864,7 +2864,6 @@ class Item :public Value_source, return false; } - protected: /* Service function for public method shallow_copy_with_checks(). diff --git a/sql/opt_range.h b/sql/opt_range.h index 0ad39eeb5e929..b0c38213c9306 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -2041,7 +2041,6 @@ class FT_SELECT: public QUICK_RANGE_SELECT int get_type() override { return QS_TYPE_FULLTEXT; } }; - FT_SELECT *get_ft_select(THD *thd, TABLE *table, uint key); QUICK_RANGE_SELECT *get_quick_select_for_ref(THD *thd, TABLE *table, struct st_table_ref *ref, diff --git a/sql/sql_explain.h b/sql/sql_explain.h index a5a092009ce42..e1405aac4a8bc 100644 --- a/sql/sql_explain.h +++ b/sql/sql_explain.h @@ -705,7 +705,7 @@ class Explain_quick_select : public Sql_alloc const int quick_type; - bool is_basic() + bool is_basic() { return (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || diff --git a/sql/sql_select.h b/sql/sql_select.h index a2cf078788883..92e53d49b5c4b 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1794,7 +1794,7 @@ class JOIN :public Sql_alloc SELECT_LEX_UNIT *unit; /// select that processed SELECT_LEX *select_lex; - /** + /** TRUE <=> optimizer must not mark any table as a constant table. This is needed for subqueries in form "a IN (SELECT .. UNION SELECT ..): when we optimize the select that reads the results of the union from a From 520d2745a8ed50f8e6942bcf5cfac72faa2f25d2 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 8 Sep 2026 09:29:57 +0300 Subject: [PATCH 29/39] Inline the mvi_key variable into its only use mysql_prepare_alter_table() computed it at the top of the key loop and read it ~290 lines below, at the one place that wants it. Nothing in between can change the answer, and the early call also ran for keys that never reach the assignment. Co-Authored-By: Claude Opus 5 (1M context) --- sql/sql_table.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 1dc2b1da1384a..0fa21881adce7 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -9208,8 +9208,6 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, for (uint i= 0; i < table->s->total_keys; i++, key_info++) { bool long_hash_key= false; - /* A multi-valued index. Its only key part is an internal column */ - const bool mvi_key= is_mvi_key(table, i); if (key_info->flags & HA_INVISIBLE_KEY) continue; Lex_ident_column key_name(key_info->name); @@ -9500,8 +9498,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key->without_overlaps= key_info->without_overlaps; key->period= table->s->period.name; key->old= true; - /* Let the key keep its internal key part, see init_key_part_spec() */ - key->invisible= mvi_key; + /* + A multi-valued index: its only key part is an internal column. Let + the key keep it, see init_key_part_spec(). + */ + key->invisible= is_mvi_key(table, i); new_key_list.push_back(key, root); } if (long_hash_key) From 1c5722003f1e107ac2016f0548ad043a833680d8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 8 Sep 2026 13:14:02 +0300 Subject: [PATCH 30/39] Move the multi-valued key part DDL out of the grammar multi_valued_key_part: built a whole schema object in its action: the hidden DB_MVI_ column with its MVI_ENCODE() vcol, the rewrite of the key into an invisible fulltext one, and the key part naming the column. That is DDL, and it belongs with the rest of the multi-valued index code, not in sql_yacc.yy where nobody reading opt_multi_valued_index.cc will find it. It becomes add_mvi_key_part(), which returns the key part or NULL if it raised an error, and the production is three lines. check_mvi_key_type() follows its only caller and turns static, so sql_table.h loses a declaration and the forward `class Key;` that existed only for it. No behaviour change. Co-Authored-By: Claude Opus 5 (1M context) --- sql/opt_multi_valued_index.cc | 106 ++++++++++++++++++++++++++++++++++ sql/opt_multi_valued_index.h | 7 +++ sql/sql_table.cc | 36 ------------ sql/sql_table.h | 6 -- sql/sql_yacc.yy | 40 +------------ 5 files changed, 114 insertions(+), 81 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 946b553aed981..0b27b2c32df26 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -16,6 +16,7 @@ #include "mariadb.h" #include "sql_select.h" +#include "sql_table.h" /* make_internal_field_name */ #include "item_func.h" #include "my_json_writer.h" @@ -363,6 +364,111 @@ void print_mvi_key_expr(String *str, const TABLE *table, uint keyno) } +/* + @brief + Can an index over an ARRAY be of the type `key' was declared with? + + @detail + Only a plain KEY can. What the server builds is a fulltext index over + the encoded elements of the array, which does not implement what any of + the other types would promise: UNIQUE and PRIMARY KEY would not be + enforced, and MATCH() against a FULLTEXT one would find nothing. They + used to be accepted and quietly turned into a plain index. + + @return + true No, and an error is raised +*/ + +static bool check_mvi_key_type(const Key *key) +{ + const char *type= NULL; + switch (key->type) { + case Key::PRIMARY: type= "PRIMARY KEY"; break; + case Key::UNIQUE: type= "UNIQUE"; break; + case Key::FULLTEXT: type= "FULLTEXT"; break; + case Key::SPATIAL: type= "SPATIAL"; break; + case Key::VECTOR: type= "VECTOR"; break; + case Key::MULTIPLE: /* A plain KEY: the only type an ARRAY can have */ + case Key::FOREIGN_KEY: /* Both of these are built with Key::MULTIPLE, so */ + case Key::IGNORE_KEY: /* they never reach us under their own name */ + break; + } + if (!type) + return false; + my_error(ER_WRONG_USAGE, MYF(0), type, "ARRAY"); + return true; +} + + +/* + @brief + Handle a `(CAST(expr AS type ARRAY))' key part: turn the key being + defined into a multi-valued index over a new internal column. + + @detail + There is no field to index directly, so the DDL builds one: a hidden + stored column computed by MVI_ENCODE(), holding the encoded elements of + the array, and a fulltext index over it. That pairing is what a + multi-valued index is, see is_mvi_key(). + + Both the column and the key are invisible: there is no syntax that would + name the column, and SHOW CREATE TABLE prints the key with the expression + it was declared with instead, see print_mvi_key_expr(). + + @return + The key part naming the new column, or NULL if an error was raised +*/ + +Key_part_spec *add_mvi_key_part(THD *thd, Item *expr, + const Lex_cast_type_st &cast_type) +{ + LEX *lex= thd->lex; + Key *key= lex->last_key; + + /* + An index over an ARRAY has exactly one key part. Catch a second one here, + before the type of the key is overwritten below and check_mvi_key_type() + starts seeing FULLTEXT instead of what the user wrote. A part that comes + *after* the ARRAY one is caught in init_key_part_spec(). + */ + if (unlikely(key->columns.elements)) + { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + return NULL; + } + if (unlikely(check_mvi_key_type(key))) + return NULL; + + /* TODO: check fts_min_token_size is 4, warn if not */ + Create_field *f= new (thd->mem_root) Create_field(); + Item *vcol_expr= + new (thd->mem_root) Item_func_mvi_encode(thd, expr, cast_type); + if (unlikely(!f || !vcol_expr)) + return NULL; + + /* Has to run before `f' joins the list it looks for a free name in */ + const Lex_ident_column fname= + make_internal_field_name(thd, "DB_MVI_", &lex->alter_info.create_list); + + Virtual_column_info *v= add_virtual_expression(thd, vcol_expr); + if (unlikely(!v)) + return NULL; + v->set_vcol_type(VCOL_GENERATED_STORED); + + f->invisible= INVISIBLE_FULL; + f->set_handler(&type_handler_blob); + f->charset= &my_charset_latin1_bin; + f->vcol_info= v; + lex->init_last_field(f, &fname); + lex->alter_info.create_list.push_back(f, thd->mem_root); + + key->type= Key::FULLTEXT; + key->invisible= true; + + return new (thd->mem_root) Key_part_spec(&fname, 0, /*gen=*/true); +} + + /* Collect all the MVI indexes of `table' */ static bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 42271773fa7cf..04cc2eebc87f7 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -103,6 +103,13 @@ bool is_mvi_key(const TABLE *table, uint keyno); */ void print_mvi_key_expr(String *str, const TABLE *table, uint keyno); +/* + DDL: handle a `(CAST(expr AS type ARRAY))' key part of the key being + defined. Returns NULL if an error was raised. +*/ +Key_part_spec *add_mvi_key_part(THD *thd, Item *expr, + const Lex_cast_type_st &cast_type); + /* Analyze `cond' and pick the MVI access `tab' will use, if any, and let the range analysis see it diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 0fa21881adce7..5655d6e83cf73 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2771,42 +2771,6 @@ static int mysql_add_invisible_field(THD *thd, List * field_list, } #endif -/* - @brief - Can an index over an ARRAY be of the type `key' was declared with? - - @detail - Only a plain KEY can. What the server builds is a fulltext index over - the encoded elements of the array, which does not implement what any of - the other types would promise: UNIQUE and PRIMARY KEY would not be - enforced, and MATCH() against a FULLTEXT one would find nothing. They - used to be accepted and quietly turned into a plain index. - - @return - true No, and an error is raised -*/ - -bool check_mvi_key_type(const Key *key) -{ - const char *type= NULL; - switch (key->type) { - case Key::PRIMARY: type= "PRIMARY KEY"; break; - case Key::UNIQUE: type= "UNIQUE"; break; - case Key::FULLTEXT: type= "FULLTEXT"; break; - case Key::SPATIAL: type= "SPATIAL"; break; - case Key::VECTOR: type= "VECTOR"; break; - case Key::MULTIPLE: /* A plain KEY: the only type an ARRAY can have */ - case Key::FOREIGN_KEY: /* Both of these are built with Key::MULTIPLE, so */ - case Key::IGNORE_KEY: /* they never reach us under their own name */ - break; - } - if (!type) - return false; - my_error(ER_WRONG_USAGE, MYF(0), type, "ARRAY"); - return true; -} - - #define INTERNAL_FIELD_NAME_LENGTH 30 Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, diff --git a/sql/sql_table.h b/sql/sql_table.h index a1e5aeb04eae3..3d34515cc43f2 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -229,10 +229,4 @@ bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, List *create_list); -class Key; -/* - Check that an index over an ARRAY may be of the type `key' was declared - with. Raises an error if it may not. -*/ -bool check_mvi_key_type(const Key *key); #endif /* SQL_TABLE_INCLUDED */ diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index f0d83128d1932..31725d459c8f8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -7652,46 +7652,8 @@ key_part: multi_valued_key_part: '(' CAST_SYM '(' expr AS cast_type ARRAY_SYM ')' ')' { - /* - An index over an ARRAY has exactly one key part. Catch a second - one here, before the type of the key is overwritten below and - check_mvi_key_type() starts seeing FULLTEXT instead of what the - user wrote. A part that comes *after* the ARRAY one is caught - in init_key_part_spec(). - */ - if (unlikely(Lex->last_key->columns.elements)) - { - my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); - MYSQL_YYABORT; - } - if (unlikely(check_mvi_key_type(Lex->last_key))) - MYSQL_YYABORT; - /* TODO: check fts_min_token_size is 4, warn if not */ - /* Create a Create_field */ - Create_field *f= new (thd->mem_root) Create_field(); - LEX_CSTRING fname= make_internal_field_name(thd, "DB_MVI_", &Lex->alter_info.create_list); - Item *vcol_expr= - new (thd->mem_root) Item_func_mvi_encode(thd, $4, $6); - - if (unlikely(!f)) - MYSQL_YYABORT; - - f->invisible= INVISIBLE_FULL; - Lex->last_key->invisible= true; - f->set_handler(&type_handler_blob); - f->charset= &my_charset_latin1_bin; - Lex->last_key->type= Key::FULLTEXT; - Lex->init_last_field(f, &fname); - Lex->alter_info.create_list.push_back(f, thd->mem_root); - - /* Create a vcol */ - Virtual_column_info *v= add_virtual_expression(thd, vcol_expr); - if (unlikely(!v)) + if (unlikely(!($$= add_mvi_key_part(thd, $4, $6)))) MYSQL_YYABORT; - Lex->last_field->vcol_info= v; - Lex->last_field->vcol_info->set_vcol_type(VCOL_GENERATED_STORED); - - $$= new (thd->mem_root) Key_part_spec(&fname, 0, /*gen=*/true); } ; From 2d4e9d5010ecce2a2000f34cd2f9bc8514695c13 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 8 Sep 2026 13:40:53 +0300 Subject: [PATCH 31/39] opt_mvi_jsonfuncs.cc: Move the code, const-ify, add comments. --- sql/opt_mvi_jsonfuncs.cc | 202 +++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 94 deletions(-) diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc index b487370933913..9d2e29eaca5df 100644 --- a/sql/opt_mvi_jsonfuncs.cc +++ b/sql/opt_mvi_jsonfuncs.cc @@ -27,9 +27,24 @@ #include "sql_select.h" #include "item_func.h" +static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, + CHARSET_INFO *cs, const String *json, + bool conjunctive, json_engine_t *je); + + /* - Find Multi-Value Index created over array_indexed_expr. + @brief + Find Multi-Value Index created over array_indexed_expr. + + @detail + Search the table for an index declared as + + INDEX idx ((CAST(array_indexed_expr AS ARRAY)); + + NOTE: we currently we only locate one such index. What if there are + multiple? */ + static Mv_index *get_mvi_index(List *indexes, Item *array_indexed_expr) { @@ -52,6 +67,97 @@ static Mv_index *get_mvi_index(List *indexes, } +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + which is true when ALL of the elements have a match, so the keys are + ANDed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + DBUG_ASSERT(fixed()); + + if (arg_count > 2 || !a2_constant) + return NULL; + /* Find the MVI that matches the first argument */ + if (!(index= get_mvi_index(indexes, args[0]))) + return NULL; + + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + + return collect_mvi_keys(thd, index, args[0]->collation.collation, val, + true, &je); +} + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + We can use MVI index when the predicate has either of the forms: + + JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') + JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) + + JSON_OVERLAPS is true when ANY of the elements has a match, so the keys + are ORed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + uint literal_arg; + String *json; + StringBuffer<256> tmp; + DBUG_ASSERT(fixed()); + + if ((index= get_mvi_index(indexes, args[0]))) + literal_arg= 1; + else if ((index= get_mvi_index(indexes, args[1]))) + literal_arg= 0; + else + return NULL; + + if (!args[literal_arg]->const_item()) + return NULL; + if (!(json= args[literal_arg]->val_json(&tmp))) + return NULL; + + /* + TODO: is this really so: + encode_mvi_key() must see the collation of the indexed expression: that + is what decides how MVI_ENCODE built the keys that are in the index. + */ + return collect_mvi_keys(thd, index, + args[1 - literal_arg]->collation.collation, json, + false, &je); +} + + /* @brief Collect the element keys to search `index' for from a JSON literal. @@ -81,7 +187,7 @@ static Mv_index *get_mvi_index(List *indexes, */ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, - CHARSET_INFO *cs, String *json, + CHARSET_INFO *cs, const String *json, bool conjunctive, json_engine_t *je) { Mvi_access *access= NULL; @@ -169,98 +275,6 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, } -/* - @brief - Check if we can use Multi-Value Index access to read rows for this - predicate, if yes create an access descriptor. - - @detail - Check if this item is a - - JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') - - which is true when ALL of the elements have a match, so the keys are - ANDed. - - @return - The access descriptor, or NULL if the predicate cannot use an MVI. -*/ - -Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, - List *indexes) -{ - Mv_index *index; - DBUG_ASSERT(fixed()); - - if (arg_count > 2 || !a2_constant) - return NULL; - /* Find the MVI that matches the first argument */ - if (!(index= get_mvi_index(indexes, args[0]))) - return NULL; - - if (!a2_parsed) - { - val= args[1]->val_json(&tmp_val); - a2_parsed= true; - } - if (!val) - return NULL; - - return collect_mvi_keys(thd, index, args[0]->collation.collation, val, - true, &je); -} - - -/* - @brief - Check if we can use Multi-Value Index access to read rows for this - predicate, if yes create an access descriptor. - - @detail - We can use MVI index when the predicate has either of the forms: - - JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') - JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) - - JSON_OVERLAPS is true when ANY of the elements has a match, so the keys - are ORed. - - @return - The access descriptor, or NULL if the predicate cannot use an MVI. -*/ - -Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, - List *indexes) -{ - Mv_index *index; - uint literal_arg; - String *json; - StringBuffer<256> tmp; - DBUG_ASSERT(fixed()); - - if ((index= get_mvi_index(indexes, args[0]))) - literal_arg= 1; - else if ((index= get_mvi_index(indexes, args[1]))) - literal_arg= 0; - else - return NULL; - - if (!args[literal_arg]->const_item()) - return NULL; - if (!(json= args[literal_arg]->val_json(&tmp))) - return NULL; - - /* - TODO: is this really so: - encode_mvi_key() must see the collation of the indexed expression: that - is what decides how MVI_ENCODE built the keys that are in the index. - */ - return collect_mvi_keys(thd, index, - args[1 - literal_arg]->collation.collation, json, - false, &je); -} - - /* Add `access' to the context, if there is one. Returns true on error */ static bool add_mvi_access(Mvi_context *ctx, Mvi_access *access) From b8ccf8b7f50a7a3779211306f0e52e347e668617 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 8 Sep 2026 19:41:49 +0300 Subject: [PATCH 32/39] Improve comments, formatting. --- sql/item_strfunc.h | 2 +- sql/opt_multi_valued_index.cc | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index f092e101d237d..231fc51865df9 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2673,7 +2673,7 @@ class Item_func_mvi_encode : public Item_str_ascii_func String tmp_js; json_engine_t je; /* Print the type the values are cast to, as CAST() spells it */ - void append_cast_type(String *str); + void append_cast_type(String *str) const; public: void print(String *str, enum_query_type query_type) override; /* diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index 0b27b2c32df26..f50ef3744ce31 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -20,9 +20,13 @@ #include "item_func.h" #include "my_json_writer.h" -static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, Mvi_access *access); +static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, + Mvi_access *access); -void Item_func_mvi_encode::append_cast_type(String *str) +/* + Append to *str string representation of m_cast_type. +*/ +void Item_func_mvi_encode::append_cast_type(String *str) const { char buf[32]; size_t length; @@ -112,7 +116,7 @@ static longlong json_value_to_longlong(enum json_value_types type, }; } -/* Lifted from Type_handler method of the same name */ +/* Copied from Type_handler::store_sort_key_longlong */ static void store_sort_key_longlong(uchar *to, bool unsigned_flag, longlong value) { @@ -183,6 +187,13 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, return false; } + +/* + @brief + Parse the JSON array argument and return a string that will be fed to the + fulltext index. +*/ + String *Item_func_mvi_encode::val_str_ascii(String *buf) { String *value= args[0]->val_json(&tmp_js); From c5bbce66ea7266517088197b31313ab445747191 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 9 Sep 2026 14:31:38 +1000 Subject: [PATCH 33/39] MDEV-40168 Add testcases for when there's both a ft index and an mvi --- mysql-test/main/multi_valued_index.result | 30 +++++++++++++++++++++++ mysql-test/main/multi_valued_index.test | 23 +++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index f1b748a110f1c..0daccc1f4e553 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -483,6 +483,36 @@ t1 CREATE TABLE `t1` ( KEY `idx` ((cast(json_extract(`j`,'$.a') as char(6) array))) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci drop table t1; +# MATCH() and a multi-valued index on the same table +create table t1 (c int, txt text, j json, +fulltext key ft (txt), +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'alpha beta','{"tags": ["aaa"]}'), +(2,'gamma','{"tags": ["bbb"]}'), +(3,'alpha','{"tags": ["aaa","bbb"]}'); +## mvi chosen +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +## ft chosen +explain select * from t1 +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext ft,idx ft 0 1 Using where +select * from t1 +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +c txt j +1 alpha beta {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +# the same rows without the index +select * from t1 ignore index(idx) +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +c txt j +1 alpha beta {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 7a8461285f049..bafd8f5017ee4 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -304,6 +304,29 @@ alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); show create table t1; drop table t1; +--echo # MATCH() and a multi-valued index on the same table +create table t1 (c int, txt text, j json, + fulltext key ft (txt), + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'alpha beta','{"tags": ["aaa"]}'), + (2,'gamma','{"tags": ["bbb"]}'), + (3,'alpha','{"tags": ["aaa","bbb"]}'); + +--echo ## mvi chosen +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +--echo ## ft chosen +explain select * from t1 + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"'); +select * from t1 + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +--echo # the same rows without the index +select * from t1 ignore index(idx) + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; + +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); From f490dd75a12c7510aee3a825b0f2314992151328 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 9 Sep 2026 16:23:26 +1000 Subject: [PATCH 34/39] MDEV-40168 Add some DDL tests --- mysql-test/main/multi_valued_index.result | 222 ++++++++++++++++++++++ mysql-test/main/multi_valued_index.test | 129 +++++++++++++ 2 files changed, 351 insertions(+) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 0daccc1f4e553..62ee785c654b1 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -513,6 +513,228 @@ c txt j 1 alpha beta {"tags": ["aaa"]} 3 alpha {"tags": ["aaa","bbb"]} drop table t1; +# +# CREATE TABLE ... LIKE +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +create table t2 like t1; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +# one internal column, named from scratch in the new table +show index from t2; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t2 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t2 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t2, t1; +# +# Renaming the base column +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +alter table t1 rename column j to jj; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `jj` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`jj`)), + KEY `idx` ((cast(json_extract(`jj`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(jj->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(jj->'$.tags','"aaa"') order by c; +c jj +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(jj->'$.tags','"aaa"') order by c; +c jj +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# ... and through CHANGE COLUMN, which renames and retypes at once +alter table t1 change jj j json; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t1; +# +# Dropping the base column +# +# The hidden column is computed from it, so the drop has to be refused +# while the index exists. It must not leave an index over a column +# that is gone. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'); +alter table t1 drop column j; +ERROR 42S22: Unknown column 'j' in 'GENERATED ALWAYS AS' +# the table is unchanged and still usable +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +# dropping the index first makes it possible +alter table t1 drop key idx, drop column j; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +# +# ALGORITHM=INSTANT / COPY / INPLACE +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +## adding a multi-valued index costs the table instant ADD COLUMN +alter table t1 add column x int, algorithm=instant; +ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +alter table t1 add column y int, algorithm=inplace; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `y` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j y +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +## the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j y +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +alter table t1 add column z int, algorithm=copy; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `y` int(11) DEFAULT NULL, + `z` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using filesort +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j y z +1 {"tags": ["aaa"]} NULL NULL +3 {"tags": ["aaa","bbb"]} NULL NULL +## adding the index in place to a table that already has rows +create table t2 (c int, j json) engine=innodb; +insert into t2 select c, j from t1; +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +## ... it needs a rebuild, which ALGORITHM=COPY does +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=copy; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +## the same rows without the index: +select * from t2 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +## ... rebuild still needed needed even when a table already has a fulltext index +create table t3 (c int, txt text, j json, fulltext key ft (txt))engine=innodb; +insert into t3 select c, 'alpha', j from t1; +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=copy; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `c` int(11) DEFAULT NULL, + `txt` text DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + FULLTEXT KEY `ft` (`txt`), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t3 where json_contains(j->'$.tags','"aaa"') order by c; +c txt j +1 alpha {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +drop table t3; +drop table t1, t2; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index bafd8f5017ee4..3d57073555743 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -327,6 +327,135 @@ order by c; drop table t1; +--echo # +--echo # CREATE TABLE ... LIKE +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +create table t2 like t1; +show create table t2; +--echo # one internal column, named from scratch in the new table +show index from t2; +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +--echo # the same rows without the index: +select * from t2 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +drop table t2, t1; + +--echo # +--echo # Renaming the base column +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +alter table t1 rename column j to jj; +show create table t1; +explain select * from t1 where json_contains(jj->'$.tags','"aaa"'); +select * from t1 where json_contains(jj->'$.tags','"aaa"') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(jj->'$.tags','"aaa"') order by c; + +--echo # ... and through CHANGE COLUMN, which renames and retypes at once +alter table t1 change jj j json; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +drop table t1; + +--echo # +--echo # Dropping the base column +--echo # +--echo # The hidden column is computed from it, so the drop has to be refused +--echo # while the index exists. It must not leave an index over a column +--echo # that is gone. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'); + +--error ER_BAD_FIELD_ERROR +alter table t1 drop column j; +--echo # the table is unchanged and still usable +show create table t1; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo # dropping the index first makes it possible +alter table t1 drop key idx, drop column j; +show create table t1; + +drop table t1; + +--echo # +--echo # ALGORITHM=INSTANT / COPY / INPLACE +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +--echo ## adding a multi-valued index costs the table instant ADD COLUMN +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 add column x int, algorithm=instant; +show create table t1; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +alter table t1 add column y int, algorithm=inplace; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +--echo ## the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +alter table t1 add column z int, algorithm=copy; +show create table t1; +explain +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo ## adding the index in place to a table that already has rows +create table t2 (c int, j json) engine=innodb; +insert into t2 select c, j from t1; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=inplace; + +--echo ## ... it needs a rebuild, which ALGORITHM=COPY does +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=copy; +show create table t2; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +--echo ## the same rows without the index: +select * from t2 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +--echo ## ... rebuild still needed needed even when a table already has a fulltext index +create table t3 (c int, txt text, j json, fulltext key ft (txt))engine=innodb; +insert into t3 select c, 'alpha', j from t1; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=inplace; +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=copy; +show create table t3; +select * from t3 where json_contains(j->'$.tags','"aaa"') order by c; +drop table t3; +drop table t1, t2; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); From c9d8277a8866a6d9d622890013b2c09b0f3a657c Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 9 Sep 2026 18:05:55 +1000 Subject: [PATCH 35/39] MDEV-40168 Resolve two TODOs 1. Merge conjunctives on the same index. Deduplicate while we are at it 2. Update costing Co-Authored-By: Claude Opus 5 --- mysql-test/main/multi_valued_index.result | 100 +++++++++- mysql-test/main/multi_valued_index.test | 72 +++++++ sql/opt_multi_valued_index.cc | 224 +++++++++++++++------- sql/opt_multi_valued_index.h | 15 +- sql/sql_select.h | 7 +- 5 files changed, 346 insertions(+), 72 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 62ee785c654b1..5aad1b26ac147 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -78,7 +78,7 @@ c j explain select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx idx 0 NULL 2 Using where +1 SIMPLE t1 range idx idx 0 NULL 1 Using where select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); c j 5 {"tags": [[["1"], "34567"]]} @@ -735,6 +735,104 @@ c txt j 3 alpha {"tags": ["aaa","bbb"]} drop table t3; drop table t1, t2; +# +# Merging the accesses of several predicates on one index +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'), +(4,'{"tags": ["aaa","ccc"]}'); +# Two JSON_CONTAINS on the same path are one index search for both +# keys, not a search for one of them with the other left to the +# WHERE clause. +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"bbb"') order by c; +c j +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"bbb"') order by c; +c j +3 {"tags": ["aaa","bbb"]} +# a key we already search for is not searched for twice +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +select * from t1 where json_contains(j->'$.tags','["aaa","aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# JSON_OVERLAPS does not merge: (a OR b) AND (c OR d) is not a +# boolean-mode query we can build, so the two stay separate +# candidates and the cheaper one is used. +select * from t1 +where json_overlaps(j->'$.tags','["aaa","bbb"]') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# ... and a conjunctive and a disjunctive access on one index do not +# merge either +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +drop table t1; +# Accesses on two different indexes cannot be merged into one +# fulltext search, so the cheaper one is chosen and the other +# predicate is left to the WHERE clause. +create table t1 (c int, j json, +key idx_t ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +key idx_n ((CAST(j->'$.nums' AS UNSIGNED ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"], "nums": [1,2]}'), +(2,'{"tags": ["bbb"], "nums": [2,3]}'), +(3,'{"tags": ["aaa"], "nums": [3,4]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx_t` ((cast(json_extract(`j`,'$.tags') as char(6) array))), + KEY `idx_n` ((cast(json_extract(`j`,'$.nums') as unsigned array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.nums','3') order by c; +c j +3 {"tags": ["aaa"], "nums": [3,4]} +# the same rows without the indexes: +select * from t1 ignore index(idx_t,idx_n) +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.nums','3') order by c; +c j +3 {"tags": ["aaa"], "nums": [3,4]} +drop table t1; # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); mvi_encode('[1, 42, "3"]', int) diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 3d57073555743..8aee174ecf29f 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -456,6 +456,78 @@ select * from t3 where json_contains(j->'$.tags','"aaa"') order by c; drop table t3; drop table t1, t2; +--echo # +--echo # Merging the accesses of several predicates on one index +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'), + (4,'{"tags": ["aaa","ccc"]}'); + +--echo # Two JSON_CONTAINS on the same path are one index search for both +--echo # keys, not a search for one of them with the other left to the +--echo # WHERE clause. +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"bbb"') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"bbb"') order by c; + +--echo # a key we already search for is not searched for twice +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"aaa"') order by c; +select * from t1 where json_contains(j->'$.tags','["aaa","aaa"]') order by c; + +--echo # JSON_OVERLAPS does not merge: (a OR b) AND (c OR d) is not a +--echo # boolean-mode query we can build, so the two stay separate +--echo # candidates and the cheaper one is used. +select * from t1 + where json_overlaps(j->'$.tags','["aaa","bbb"]') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_overlaps(j->'$.tags','["aaa","bbb"]') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; + +--echo # ... and a conjunctive and a disjunctive access on one index do not +--echo # merge either +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; + +drop table t1; + +--echo # Accesses on two different indexes cannot be merged into one +--echo # fulltext search, so the cheaper one is chosen and the other +--echo # predicate is left to the WHERE clause. + +create table t1 (c int, j json, + key idx_t ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + key idx_n ((CAST(j->'$.nums' AS UNSIGNED ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"], "nums": [1,2]}'), + (2,'{"tags": ["bbb"], "nums": [2,3]}'), + (3,'{"tags": ["aaa"], "nums": [3,4]}'); +show create table t1; + +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.nums','3') order by c; +--echo # the same rows without the indexes: +select * from t1 ignore index(idx_t,idx_n) + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.nums','3') order by c; + +drop table t1; + --echo # direct call of mvi_encode select mvi_encode('[1, 42, "3"]', int); select mvi_encode('[1, 42, "3"]', unsigned); diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index f50ef3744ce31..fc746f4d9d655 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -517,6 +517,20 @@ bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) { + List_iterator it(encoded); + String *have; + /* + A key we already search for adds nothing: '+ka +ka' matches what '+ka' + matches, and so does 'ka ka'. The lists are a handful of elements, so + the scan is cheaper than the extra fulltext term would be. + */ + while ((have= it++)) + { + if (have->length() == key->length() && + !memcmp(have->ptr(), key->ptr(), key->length())) + return false; + } + String *s= new (mem_root) String; const char *copy= (const char *) memdup_root(mem_root, key->ptr(), key->length()); @@ -529,8 +543,40 @@ bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) /* @brief - Estimate how many records this access will read, and simplify the access - if that lets us read fewer. + Fold another access on the same index into this one. + + @detail + collect_mvi_accesses() only takes the top-level AND-parts of the + condition, so every access it produces has to be true for every row of + the result. Two conjunctive accesses on one index therefore require the + union of their keys, and one search for '+ka +kb' finds what two separate + searches would - more selectively than either, at the price of one. + + Only conjunctive accesses merge. Two disjunctive ones would need + '(ka kb) (kc kd)' to mean (a OR b) AND (c OR d), which the boolean-mode + query build_ft_query() puts together has no syntax for. They stay + separate candidates and get_best_mvi_access() picks between them. +*/ + +bool Mvi_access::merge(MEM_ROOT *mem_root, Mvi_access *other) +{ + List_iterator it(other->encoded); + String *key; + DBUG_ASSERT(can_merge(other)); + /* Merging changes the key set, so it has to happen before we cost it */ + DBUG_ASSERT(records == HA_POS_ERROR); + while ((key= it++)) + { + if (add_key(mem_root, key)) + return true; + } + return false; +} + + +/* + @brief + Estimate how many records this access will read. @detail The engine gives us an estimate for one element key at a time (the @@ -544,15 +590,23 @@ bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) out of the plan instead. - Conjunctive access (JSON_CONTAINS) reads the rows that have all of the - keys, so the rarest key alone bounds the result. Use its estimate, and - drop the other keys from the query: reading the rarest key and letting - the WHERE clause discard the rest is not worse than having the engine - intersect the terms. This is the trade-off collect_mvi_keys() already - makes for the keys it cannot encode - a shorter AND matches a superset - of the rows, and the JSON predicate does the exact filtering. - Keys the engine cannot estimate take no part in the choice. If it - could not estimate a single one of them we know nothing at all, so the - access is priced out just like a disjunctive one. + keys. The engine estimates one key at a time and cannot intersect them + for us, so assume the keys are independent: + + rows ~ N * PROD(r_i / N) + + clamped to the rarest key, which is a hard upper bound. The assumption + under-estimates correlated keys - the elements of a tag array often + are - but the rarest key alone over-estimates by orders of magnitude + as soon as the keys are at all selective, and every term we keep in + the query is a term the engine intersects instead of us fetching the + row and having the WHERE clause discard it. + + Keys the engine cannot estimate take no part in the estimate but stay + in the query: a longer AND only narrows the scan, and the JSON + predicate does the exact filtering either way. If it could not + estimate a single one of them we know nothing at all, so the access is + priced out just like a disjunctive one. TODO: read_time only accounts for reading the rows, not for the fulltext search that produces their rowids. @@ -563,8 +617,11 @@ void Mvi_access::estimate_records() TABLE *table= index->vcol->table; handler *file= table->file; List_iterator it(encoded); - String *key, *rarest= NULL; - ha_rows sum= 0, min_rows= 0; + String *key; + const double n_rows= rows2double(table->stat_records()); + double sum= 0.0, isect= n_rows; + ha_rows min_rows= HA_POS_ERROR; + uint n_estimated= 0; bool have_unknown_estimate= false; while ((key= it++)) @@ -576,17 +633,16 @@ void Mvi_access::estimate_records() have_unknown_estimate= true; continue; } - sum+= rows; - if (!rarest || rows < min_rows) - { - min_rows= rows; - rarest= key; - } + n_estimated++; + sum+= rows2double(rows); + set_if_smaller(min_rows, rows); + if (n_rows >= 1.0) + isect*= rows2double(rows) / n_rows; } if (!conjunctive && have_unknown_estimate) { - /* + /* Disjunctive means we have to read all keys. For at least one, we have no idea how many matches it has. Fall back to full scan. */ @@ -594,7 +650,7 @@ void Mvi_access::estimate_records() read_time= DBL_MAX; return; } - if (conjunctive && !rarest) + if (conjunctive && !n_estimated) { /* Nothing was estimated. Fall back to full table scan */ records= table->stat_records(); @@ -604,17 +660,12 @@ void Mvi_access::estimate_records() if (conjunctive) { - /* Search for the rarest key only */ - it.rewind(); - while ((key= it++)) - { - if (key != rarest) - it.remove(); - } - records= min_rows; + /* The rows that have all of the keys, see above */ + records= n_rows >= 1.0 ? (ha_rows) isect : (ha_rows) 1; + set_if_smaller(records, min_rows); } else - records= sum; + records= (ha_rows) sum; set_if_smaller(records, table->stat_records()); set_if_bigger(records, (ha_rows) 1); @@ -729,9 +780,15 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) of this table uses, too. @detail - The analysis itself is scratch state: what we leave behind is the one - access we've settled on, in tab->mvi_access. It and the Mv_index it - refers to live on the MEM_ROOT, so they outlive `ctx'. + The analysis itself is scratch state: what we leave behind is the list of + accesses in tab->mvi_accesses. They and the Mv_index objects they refer + to live on the MEM_ROOT, so they outlive `ctx'. + + Accesses on one index that both require all of their keys are merged + here, see Mvi_access::merge(). What is left is one candidate per index + and kind, and get_best_mvi_access() prices those and picks one. We put + no price on anything here: the estimate probes the engine's fulltext + index, and this runs for every table of the join. A fulltext key never gets a bit in const_keys or keys, so we set them here. The const_keys bit is what makes the range analysis run for this @@ -740,13 +797,15 @@ static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) @return true Out of memory - false Ok, tab->mvi_access is set if the table has an MVI access + false Ok, tab->mvi_accesses is set if the table has any MVI access */ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) { Mvi_context ctx(thd); - Mvi_access *best= NULL; + MEM_ROOT *mem_root= thd->mem_root; + List *kept; + if (!cond) return false; if (collect_mvi_indexes_for_table(thd, tab->table, &ctx.indexes)) @@ -756,32 +815,43 @@ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) return false; if (collect_mvi_accesses(&ctx, cond)) return true; + if (ctx.accesses.is_empty()) + return false; - List_iterator it(ctx.accesses); - /* TODO: cost based */ - /* - TODO: merge - - json_contains(j->'$.tags','"a"') and - json_contains(j->'$.tags','"b"') + if (!(kept= new (mem_root) List)) + return true; - (+ta +tb) - */ + List_iterator it(ctx.accesses); while (Mvi_access *access= it++) { + Mvi_access *into; /* An access can only be on this table: ctx.indexes holds this table's indexes and get_mvi_index() matches the predicate against those. */ DBUG_ASSERT(access->index->vcol->table == tab->table); - best= access; + + /* Fold it into an access we already keep, if the two are compatible */ + List_iterator kit(*kept); + while ((into= kit++)) + { + if (into->can_merge(access)) + break; + } + if (into) + { + if (into->merge(mem_root, access)) + return true; + continue; + } + + if (kept->push_back(access, mem_root)) + return true; + tab->const_keys.set_bit(access->index->keyno); + tab->keys.set_bit(access->index->keyno); } - if (!best) - return false; - tab->mvi_access= best; - tab->const_keys.set_bit(best->index->keyno); - tab->keys.set_bit(best->index->keyno); + tab->mvi_accesses= kept; return false; } @@ -794,20 +864,38 @@ bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) The range optimizer cannot produce this access (it skips fulltext keys), so the caller creates it here and compares its cost with whatever test_quick_select() came up with. + + This is the only place an MVI access is priced. Where a table has more + than one - accesses on different indexes, which cannot be merged into a + single fulltext search - the cheapest one wins, on the same read_time + scale keep_cheaper_quick() then uses against the range access. */ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) { TABLE *table= tab->table; - Mvi_access *access= tab->mvi_access; - if (!access) + Mvi_access *access, *best= NULL; + + if (!tab->mvi_accesses) return NULL; - /* - estimate_records() drops element keys from the access, so it must run - only once even if we are called again for the same table. - */ - if (access->records == HA_POS_ERROR) - access->estimate_records(); + + List_iterator it(*tab->mvi_accesses); + while ((access= it++)) + { + /* estimate_records() probes the engine, so do it at most once */ + if (access->records == HA_POS_ERROR) + access->estimate_records(); + /* + best_access_path() takes a quick select to be cheaper than a table + scan without checking (the range optimizer only proposes a quick when + it is), so an access we could not put a price on is no use to us. + */ + if (!access->cost_is_known()) + continue; + if (!best || access->read_time < best->read_time) + best= access; + } + if (unlikely(thd->trace_started())) { /* @@ -818,16 +906,20 @@ QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) Json_writer_object trace_wrapper(thd); Json_writer_object trace_mvi(thd, "multi_value_index_use"); trace_mvi.add_table_name(table); - access->print_json(thd, &trace_mvi); + Json_writer_array trace_candidates(thd, "candidates"); + it.rewind(); + while ((access= it++)) + { + Json_writer_object trace_one(thd); + access->print_json(thd, &trace_one); + if (access == best) + trace_one.add("chosen", true); + } } - /* - best_access_path() takes a quick select to be cheaper than a table scan - without checking (the range optimizer only proposes a quick when it is), - so an access we could not put a price on has to be dropped here. - */ - if (!access->cost_is_known()) + + if (!best) return NULL; - return create_quick_mvi_select(thd, table, access); + return create_quick_mvi_select(thd, table, best); } diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index 04cc2eebc87f7..e6013ad7909a7 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -43,6 +43,17 @@ struct Mvi_access : public Sql_alloc /* Build: Add one encoded element key */ bool add_key(MEM_ROOT *mem_root, const String *key); + /* + Build: can `other' be folded into this access? Two accesses on the same + index that both require all of their keys are the same thing as one + access requiring the union of the keys. + */ + bool can_merge(const Mvi_access *other) const + { return index == other->index && conjunctive && other->conjunctive; } + + /* Build: fold `other' into this access. can_merge() must hold */ + bool merge(MEM_ROOT *mem_root, Mvi_access *other); + /* Usage: Estimate how many records this access will read */ void estimate_records(); @@ -62,8 +73,8 @@ struct Mvi_access : public Sql_alloc /* The state of the MVI analysis of one table. It only lives for the duration - of setup_mvi_access_for_table(): the access that analysis settles on is - what outlives it. + of setup_mvi_access_for_table(): the accesses that analysis settles on are + what outlive it. */ class Mvi_context : public Sql_alloc { diff --git a/sql/sql_select.h b/sql/sql_select.h index 92e53d49b5c4b..0fe0bd8b7b97e 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -568,10 +568,11 @@ typedef struct st_join_table { key_map needed_reg; key_map keys; /**< all keys with can be used */ /* - The multi-valued index access to use for this table, or NULL if there is - none. Set by setup_mvi_access_for_table(). + The multi-valued index accesses this table's predicates allow, or NULL if + there are none. Set by setup_mvi_access_for_table(); get_best_mvi_access() + prices them and picks one. */ - Mvi_access *mvi_access; + List *mvi_accesses; /* Either #rows in the table or 1 for const table. */ ha_rows records; From 67c55469666c0368cb0d16ded7d08fb04be298b1 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 9 Sep 2026 15:02:53 +0300 Subject: [PATCH 36/39] More comments, code readability. No functional changes. --- sql/opt_multi_valued_index.cc | 44 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index fc746f4d9d655..e6b32c3d4b29d 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -130,6 +130,25 @@ static void store_sort_key_longlong(uchar *to, bool unsigned_flag, to[0]= (uchar) (value >> 56) ^ (unsigned_flag ? 0 : 128); } + +/* + @brief + Encode the current JSON value in *je to either store or look it up in + Multi-Value Index. The index uses cast_th datatype. + + @detail + The encoded value shouldn't have space, punctuation or other similar + characters, as we're using the default Fulltext parser and want the + encoded value treated as one "term". + + If the value cannot be encoded this means it is not stored, also + searches won't find any matches for it. + + @return + false Encoded successfully + true The JSON value cannot be represented in the index datatype. +*/ + bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, CHARSET_INFO *cs, String *buf) { @@ -146,27 +165,36 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, switch(cast_ftype) { case MYSQL_TYPE_LONGLONG: - store_sort_key_longlong( - (uchar *) sorted.c_ptr(), is_unsigned, - json_value_to_longlong(je->value_type, cs, - (char *) je->value, je->value_len)); + { + longlong val= json_value_to_longlong(je->value_type, cs, + (char *) je->value, + je->value_len); sorted.length(8); + store_sort_key_longlong((uchar *) sorted.c_ptr(), + is_unsigned, val); break; - /* TODO: unquote? */ - /* CHAR(n) => LONG BLOB */ + } + /* TODO: unquote? */ + /* CHAR(n) => LONG BLOB */ case MYSQL_TYPE_LONG_BLOB: { /* Trim trailing whitespaces if possible */ if (!(cs->state & MY_CS_NOPAD)) + { je->value_len= (int) cs->lengthsp((const char *) je->value, je->value_len); + } if (my_binary_compare(cs)) + { sorted.set((char *) je->value, je->value_len, &my_charset_latin1_bin); + } else { - my_strnxfrm_ret_t rc= cs->strnxfrm( - (uchar *) sorted.c_ptr(), 42, 42, je->value, je->value_len, 0); + // TODO: Is this ever used outside of "SELECT MVI_ENCODE()" ? + my_strnxfrm_ret_t rc= + cs->strnxfrm((uchar *) sorted.c_ptr(), /*buffer_size*/42, + /*n_weights*/ 42, je->value, je->value_len, 0); sorted.length(rc.m_result_length); } break; From 8827afb510bd202b51509aac7b86e94e6985eb53 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 10 Sep 2026 15:42:13 +1000 Subject: [PATCH 37/39] MDEV-40168 Don't use an MVI for an depth-2 OVERLAPS array with no keys An element that is itself an array is flattened, the same way MVI_ENCODE flattens the document. JSON_OVERLAPS does not flatten: it only matches such an element against a document element that is an array too, compared whole (json_compare_arrays_in_order()). The flattening here is still safe as it will produce only false positives that will be eliminated by a recheck. The only exception is when the nested array yields no key at all i.e. [], [[]], [[],[]], [[[]]], etc. Such an element may match a document element that has no key of ours either, so nothing we could search for would find that row. Give up in this case, as for a failed encoding. select json_overlaps('[[]]', '[[], "aaa"]'); is true, but the document has no key in the index and the scan for the "aaa" key does not return it. Give up on the access in that case, as we already do for an element that cannot be encoded. Co-Authored-By: Claude Opus 5 --- mysql-test/main/multi_valued_index.result | 93 +++++++++++++++++++++++ mysql-test/main/multi_valued_index.test | 51 +++++++++++++ sql/opt_mvi_jsonfuncs.cc | 27 ++++++- 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 5aad1b26ac147..3c9bdd0257e21 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -300,6 +300,99 @@ c j c j 1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} 4 {"tags": ["aaa","bbb"]} drop table t1; +# JSON_OVERLAPS and nested arrays +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [["aaa"]]}'), +(3,'{"tags": [["aaa"],"ccc"]}'),(4,'{"tags": [["ccc"]]}'), +(5,'{"tags": [[]]}'); +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +explain select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +# the same when the element is nothing but empty arrays +explain select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +# An empty array deeper inside the depth-2 array does not +# matter: what the depth-2 array as a whole flattens to is not +# empty. +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +c j +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +c j +# JSON_CONTAINS is not affected: it flattens nested arrays itself, and +# dropping a key from an AND is always safe. +explain select * from t1 where json_contains(j->'$.tags','[[],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_contains(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +# Wrong results are consistent. Bug separate from this feature. +explain +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 5 Using where +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +select * from t1 ignore index(idx) where json_overlaps(j->'$.tags', '[[]]'); +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +drop table t1; # The predicate does not have to be in the WHERE clause: for a table on # the inner side of an outer join we look at the ON expression, which is # what has to be true for the rows we read. diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index 8aee174ecf29f..d3614ce2cf90c 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -186,6 +186,57 @@ where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; drop table t1; +--echo # JSON_OVERLAPS and nested arrays +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [["aaa"]]}'), + (3,'{"tags": [["aaa"],"ccc"]}'),(4,'{"tags": [["ccc"]]}'), + (5,'{"tags": [[]]}'); + +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]'); +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa"]]') order by c; + +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; + +--echo # the same when the element is nothing but empty arrays +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; + +--echo # An empty array deeper inside the depth-2 array does not +--echo # matter: what the depth-2 array as a whole flattens to is not +--echo # empty. +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]'); +select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; + +--echo # JSON_CONTAINS is not affected: it flattens nested arrays itself, and +--echo # dropping a key from an AND is always safe. +--replace_column 9 # +explain select * from t1 where json_contains(j->'$.tags','[[],"aaa"]'); +select * from t1 where json_contains(j->'$.tags','[[],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[[],"aaa"]') order by c; + +--echo # Wrong results are consistent. Bug separate from this feature. +explain +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +select * from t1 ignore index(idx) where json_overlaps(j->'$.tags', '[[]]'); +drop table t1; + --echo # The predicate does not have to be in the WHERE clause: for a table on --echo # the inner side of an outer join we look at the ON expression, which is --echo # what has to be true for the rows we read. diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc index 9d2e29eaca5df..157eeea82c323 100644 --- a/sql/opt_mvi_jsonfuncs.cc +++ b/sql/opt_mvi_jsonfuncs.cc @@ -182,6 +182,18 @@ Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, so that row has no key in the index for us to find it by. Dropping the key would lose it. Give up on the access instead. + An element that is itself an array is flattened, the same way + MVI_ENCODE flattens the document. JSON_OVERLAPS does not flatten: + it only matches such an element against a document element that is + an array too, compared whole (json_compare_arrays_in_order()). The + flattening here is still safe as it will produce only false + positives that will be eliminated by a recheck. The only exception + is when the nested array yields no key at all i.e. [], [[]], + [[],[]], [[[]]], etc. Such an element may match a document element + that has no key of ours either, so nothing we could search for + would find that row. Give up in this case, as for a failed + encoding. + @return The access descriptor, or NULL if the predicate cannot use this MVI. */ @@ -198,6 +210,11 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, (Item_func_mvi_encode *) index->vcol->vcol_info->expr; const Type_handler *cast_th= mvitem->cast_type().type_handler(); int depth= 0; + /* + The number of keys collected when inside a current depth-2 array + element. Only used for an OR / JSON_OVERLAPS + */ + uint keys_before_level2_array= 0; buf.length(0); buf.set_charset(&my_charset_latin1_bin); @@ -236,6 +253,13 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, case JST_ARRAY_END: if (--depth == 0) return access; + /* + Closed a top-level element that was an array. See above: for an OR + it has to have contributed at least one key. + */ + if (depth == 1 && !conjunctive && + (access ? access->encoded.elements : 0) == keys_before_level2_array) + return NULL; break; case JST_VALUE: { @@ -243,7 +267,8 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, return NULL; if (je->state == JST_ARRAY_START) { - depth++; + if (++depth == 2) + keys_before_level2_array= access ? access->encoded.elements : 0; break; } if (je->value_type == JSON_VALUE_OBJECT) From b575e0d5b0676970dee9a3bba1bc3a45d198d9d2 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 10 Sep 2026 18:16:06 +1000 Subject: [PATCH 38/39] MDEV-40168 Factor out the MVI array walk WIP NOTE (ycp): this is a squash of two commits. The first one factors the walk out using two new classes. A bit too heavy handed imo. The commit can be found at 68ae51b89bc850a4093e1b91c7da80ee74fa61bf. I reviewed that commit and it looks ok. The second commit redid the refactoring to an iterator. I have not yet reviewed this new implementation yet. MDEV-40168 Factor out the MVI array walk Item_func_mvi_encode::val_str_ascii() and collect_mvi_keys() walked a JSON array the same way: descend into a nested array, skip an object, encode everything else with encode_mvi_key(). The two copies had to agree for the index to answer correctly, and they had already drifted once. Move the walk into walk_mvi_json_array() and give the callers a visitor each: Mvi_key_appender, which joins the keys with a space into the fulltext document of a row, and Mvi_key_collector, which turns them into the keys of an Mvi_access and decides when a missing key means we have to give up on the index. The walk reports why it stopped, so the encoding side can still tell a malformed document from one that is not an array, and the query side can pick the scalar literal out of MVI_WALK_NOT_ARRAY. No functional changes. Co-Authored-By: Claude Opus 5 MDEV-40168 Turn the MVI array walk into an iterator The walk was a visitor: five virtual callbacks, each returning a bool to stop the walk, and a key_buffer() the walk asked for a buffer with. Two callers do not need that much interface, and neither of them wanted its control flow inverted -- both had to encode "give up" as a bool that the walk then turned back into a result code. Pull instead of push. Mvi_array_iterator::start() and next() return the next thing the walk found -- a key, an element with no key, a nested array opened or closed, or one of the ways the walk ends -- and the caller loops over them with its own control flow: MVI_ENCODE goes back to its gotos, collect_mvi_keys() to plain returns, and the state it kept in a visitor is local variables again. The buffer to encode into is a constructor argument, so MVI_ENCODE still gets its keys written straight into the document it is building. Mvi_json_array_visitor, Mvi_key_appender and Mvi_key_collector go away. What made the walk worth sharing is unchanged: the two sides of the index cannot disagree about what the keys of a document are, because this is the only place that makes them. No functional changes. Co-Authored-By: Claude Opus 5 --- sql/opt_multi_valued_index.cc | 179 +++++++++++++++++++++++----------- sql/opt_multi_valued_index.h | 101 ++++++++++++++++++- sql/opt_mvi_jsonfuncs.cc | 130 +++++++++++------------- 3 files changed, 274 insertions(+), 136 deletions(-) diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index e6b32c3d4b29d..b019de1fa1a5f 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -145,8 +145,9 @@ static void store_sort_key_longlong(uchar *to, bool unsigned_flag, searches won't find any matches for it. @return - false Encoded successfully + false Encoded successfully, the key is appended to *buf true The JSON value cannot be represented in the index datatype. + Nothing is appended. */ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, @@ -154,8 +155,7 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, { enum_field_types cast_ftype= cast_th->field_type(); bool is_unsigned= cast_th->is_unsigned(); - /* TODO: 42 hardcoded */ - StringBuffer<42> sorted; + StringBuffer sorted; /* Skip encoding on type incompatibility */ if (mvi_json_class(cast_ftype) != je->value_type) return true; @@ -193,8 +193,10 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, { // TODO: Is this ever used outside of "SELECT MVI_ENCODE()" ? my_strnxfrm_ret_t rc= - cs->strnxfrm((uchar *) sorted.c_ptr(), /*buffer_size*/42, - /*n_weights*/ 42, je->value, je->value_len, 0); + cs->strnxfrm((uchar *) sorted.c_ptr(), + /*buffer_size*/ MVI_KEY_IMAGE_MAX_LEN, + /*n_weights*/ MVI_KEY_IMAGE_MAX_LEN, + je->value, je->value_len, 0); sorted.length(rc.m_result_length); } break; @@ -216,80 +218,139 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, } +/* + @brief + Read the element the scan is positioned on. See Mvi_array_iterator. + + TODO: deduplicate, so that ["34567", "34567"] yield only one key +*/ + +Mvi_walk_event Mvi_array_iterator::read_element() +{ + uint32 key_start; + if (json_read_value(m_je)) + return MVI_WALK_JSON_ERROR; + + if (m_je->value_type == JSON_VALUE_ARRAY) + { + m_event_depth= ++m_depth; + return MVI_NESTED_START; + } + m_event_depth= m_depth; + + if (m_je->value_type == JSON_VALUE_OBJECT) + return json_skip_level(m_je) ? MVI_WALK_JSON_ERROR : MVI_NO_KEY; + + key_start= m_key->length(); + if (encode_mvi_key(m_je, m_cast_th, m_cs, m_key)) + { + m_key->length(key_start); /* Leave the buffer as we found it */ + return MVI_NO_KEY; + } + return MVI_KEY; +} + + +Mvi_walk_event Mvi_array_iterator::start(const uchar *start, const uchar *end) +{ + if (json_scan_start(m_je, m_cs, start, end) || json_read_value(m_je)) + return MVI_WALK_JSON_ERROR; + + if (m_je->value_type != JSON_VALUE_ARRAY) + return MVI_WALK_NOT_ARRAY; + + /* The scan is on the JST_ARRAY_START of the array we are to walk */ + m_depth= m_event_depth= 1; + return next(); +} + + +Mvi_walk_event Mvi_array_iterator::next() +{ + /* The scan ending before the array is closed is an error */ + if (json_scan_next(m_je)) + return MVI_WALK_JSON_ERROR; + + switch (m_je->state) + { + case JST_ARRAY_END: + m_event_depth= m_depth--; + /* Trailing junk after the outer array is ignored */ + return m_depth == 0 ? MVI_WALK_END : MVI_NESTED_END; + case JST_VALUE: + return read_element(); + default: + /* + A nested array is opened by read_element(), out of the value it + reads, so JST_ARRAY_START does not come back here either. + */ + return MVI_WALK_BAD_FORMAT; + } +} + + /* @brief Parse the JSON array argument and return a string that will be fed to the fulltext index. + + @detail + The keys are encoded straight into *buf, so there is nothing to copy. + A separator follows every one of them, including the last, which is + taken back off at the end: putting it in front of every key but the + first would leave one behind when an element turns out to have no key. */ String *Item_func_mvi_encode::val_str_ascii(String *buf) { String *value= args[0]->val_json(&tmp_js); + Mvi_walk_event event; + DBUG_ASSERT(fixed()); if ((null_value= !value)) return nullptr; - CHARSET_INFO *cs= value->charset(); - const Type_handler *cast_th= m_cast_type.type_handler(); - bool at_least_one= false; - const uchar *start= reinterpret_cast(value->ptr()); - const uchar *end= start + value->length(); - int depth= 0; - DBUG_ASSERT(fixed()); buf->length(0); buf->set_charset(&my_charset_latin1_bin); - if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) - goto json_error; - - if (je.value_type != JSON_VALUE_ARRAY) - goto error_format; - - /* TODO: deduplicate, so that ["34567", "34567"] yield only one token */ - do { - switch (je.state) + Mvi_array_iterator it(&je, value->charset(), m_cast_type.type_handler(), + buf); + for (event= it.start(reinterpret_cast(value->ptr()), + reinterpret_cast(value->end())); + !mvi_walk_stopped(event); + event= it.next()) + { + /* + The key is already in place, only the separator is left to add. An + append that fails is out of memory -- my_malloc() reports it, see + Binary_string::realloc_raw() -- and a document that silently loses a + key loses rows, so give up on the row instead. + */ + if (event == MVI_KEY && buf->append(' ')) { - case JST_ARRAY_START: - depth++; - continue; - case JST_ARRAY_END: - if (--depth == 0) - goto array_done; - break; - case JST_VALUE: - { - if (json_read_value(&je)) - goto json_error; - if (je.value_type == JSON_VALUE_ARRAY) - { - depth++; - break; - } - if (je.value_type == JSON_VALUE_OBJECT) - { - if (json_skip_level(&je)) - goto json_error; - break; - } - if (!encode_mvi_key(&je, cast_th, cs, buf)) - { - buf->append(' '); - at_least_one= true; - } - break; - } - default: - goto error_format; + null_value= true; + return nullptr; } - } while (json_scan_next(&je) == 0); - goto json_error; + } + + switch (event) + { + case MVI_WALK_END: + break; + case MVI_WALK_NOT_ARRAY: + case MVI_WALK_BAD_FORMAT: + goto error_format; + default: /* MVI_WALK_JSON_ERROR */ + goto json_error; + } + + /* Take the separator that follows the last key back off */ + if (buf->length()) + buf->length(buf->length() - 1); -array_done: /* TODO: do something different when an empty string is - returned, i.e. at_least_one == false to avoid wasting index - space? + returned, i.e. the document has no key at all, to avoid wasting + index space? */ - if (at_least_one) - buf->length(buf->length() - 1); return buf; error_format: diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index e6013ad7909a7..ed0962f5b0c00 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -92,13 +92,110 @@ class Mvi_context : public Sql_alloc enum json_value_types mvi_json_class(enum_field_types ftype); /* - Encode one JSON value into the form it has in the index. Returns true if - the value cannot be encoded for this index and has to be skipped. + A key is one fulltext token, so it cannot be longer than the maximum + token size the engine will index: 84 characters (HA_FT_MAXCHARLEN, which + is also the default and the maximum of innodb_ft_max_token_size). The key + is the hex of the key image, so that image is at most half of it. + + TODO: innodb_ft_max_token_size can be set lower than its default, and + innodb_ft_min_token_size higher, and then the engine drops keys we + consider valid. Validate both against the index at DDL time. +*/ +#define MVI_KEY_IMAGE_MAX_LEN 42 +#define MVI_ENCODED_KEY_MAX_LEN (MVI_KEY_IMAGE_MAX_LEN * 2) + +/* + Encode one JSON value into the form it has in the index, appending it to + `buf'. Returns true if the value cannot be encoded for this index and has + to be skipped, in which case nothing is appended. Shared with opt_mvi_jsonfuncs.cc. */ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, CHARSET_INFO *cs, String *buf); +/* + What an Mvi_array_iterator stopped at. Everything from MVI_WALK_END on + is the end of the walk. +*/ +enum Mvi_walk_event +{ + MVI_KEY, /* An element that is in the index, see key_buffer */ + MVI_NO_KEY, /* An element that is not: an object, or one that + cannot be encoded in the index datatype */ + MVI_NESTED_START, /* An element that is an array was opened */ + MVI_NESTED_END, /* ... and closed. depth() is its depth */ + + MVI_WALK_END, /* The array was walked to its end */ + MVI_WALK_NOT_ARRAY, /* The document is not an array. *je holds the value */ + MVI_WALK_BAD_FORMAT, /* The document is not a JSON we can make sense of */ + MVI_WALK_JSON_ERROR /* Malformed JSON. The error is in je->s.error */ +}; + +inline bool mvi_walk_stopped(Mvi_walk_event event) +{ return event >= MVI_WALK_END; } + +/* + @brief + Walk a JSON array, encoding its elements for a multi-valued index of + the cast_th datatype. + + @detail + Both sides of the index read the elements this way: MVI_ENCODE, which + turns them into the fulltext document of a row, and the optimizer, + which turns them into the keys to search that document for. They agree + on what the keys of a document are because this is where the keys are + made. + + An element that is an array is walked too, so that the keys of a nested + array are the keys of its elements, with MVI_NESTED_START and + MVI_NESTED_END around them. + + A key is appended to the `key' buffer the iterator was given. The + caller decides what that buffer is: MVI_ENCODE hands over the document + it is building, and gets the key encoded into it with nothing to copy + afterwards; the optimizer hands over a scratch buffer and empties it + between keys. An element that turns out to have no key leaves the + buffer as it was. + + Usage: + + Mvi_array_iterator it(je, cs, cast_th, &buf); + for (event= it.start(str, end); !mvi_walk_stopped(event); + event= it.next()) + { ... } +*/ + +class Mvi_array_iterator +{ + json_engine_t * const m_je; + CHARSET_INFO * const m_cs; + const Type_handler * const m_cast_th; + String * const m_key; + int m_depth; /* The array we are in. 1 is the outer one */ + int m_event_depth; + + Mvi_walk_event read_element(); +public: + Mvi_array_iterator(json_engine_t *je, CHARSET_INFO *cs, + const Type_handler *cast_th, String *key) + : m_je(je), m_cs(cs), m_cast_th(cast_th), m_key(key), + m_depth(0), m_event_depth(0) {} + + /* Position on the first element of the array between `start' and `end' */ + Mvi_walk_event start(const uchar *start, const uchar *end); + + /* Move on to the next element */ + Mvi_walk_event next(); + + /* + The depth of the array the last event is about: the one that was opened + or closed for MVI_NESTED_START / MVI_NESTED_END, the one the element + belongs to for MVI_KEY / MVI_NO_KEY. The array being walked is depth 1, + so a depth-2 array is an element of it. + */ + int depth() const { return m_event_depth; } +}; + /* Is `field' the internal column that holds the keys of a multi-valued index? */ diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc index 157eeea82c323..61535fc03743b 100644 --- a/sql/opt_mvi_jsonfuncs.cc +++ b/sql/opt_mvi_jsonfuncs.cc @@ -158,6 +158,12 @@ Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, } +static uint mvi_key_count(const Mvi_access *access) +{ + return access ? access->encoded.elements : 0; +} + + /* @brief Collect the element keys to search `index' for from a JSON literal. @@ -194,6 +200,10 @@ Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, would find that row. Give up in this case, as for a failed encoding. + Anything the walk itself does not like -- malformed JSON, an object + where an array should be -- means no access either. The predicate is + still there to parse the literal and raise whatever it raises. + @return The access descriptor, or NULL if the predicate cannot use this MVI. */ @@ -202,101 +212,71 @@ static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, CHARSET_INFO *cs, const String *json, bool conjunctive, json_engine_t *je) { - Mvi_access *access= NULL; - StringBuffer<256> buf; - const uchar *start= reinterpret_cast(json->ptr()); - const uchar *end= start + json->length(); Item_func_mvi_encode *mvitem= (Item_func_mvi_encode *) index->vcol->vcol_info->expr; const Type_handler *cast_th= mvitem->cast_type().type_handler(); - int depth= 0; + Mvi_access *access= NULL; + /* One key at a time: add_key() copies it onto the mem_root */ + StringBuffer buf; /* - The number of keys collected when inside a current depth-2 array - element. Only used for an OR / JSON_OVERLAPS + The number of keys we had collected when the depth-2 array element we + are inside of was opened. Only used for an OR / JSON_OVERLAPS. */ uint keys_before_level2_array= 0; + Mvi_walk_event event; - buf.length(0); buf.set_charset(&my_charset_latin1_bin); + Mvi_array_iterator it(je, cs, cast_th, &buf); - if (json_scan_start(je, cs, start, end) || json_read_value(je)) - return NULL; - - if (je->value_type == JSON_VALUE_UNINITIALIZED || - je->value_type == JSON_VALUE_OBJECT) - return NULL; - - if (je->value_type != JSON_VALUE_ARRAY) + for (event= it.start(reinterpret_cast(json->ptr()), + reinterpret_cast(json->end())); + !mvi_walk_stopped(event); + event= it.next()) { - /* A scalar: JSON_CONTAINS(expr, '123') */ - if (encode_mvi_key(je, cast_th, cs, &buf)) - return NULL; - if (!(access= new (thd->mem_root) Mvi_access(index, conjunctive)) || - access->add_key(thd->mem_root, &buf)) - return NULL; - return access; - } - // JSON_VALUE_ARRAY - - /* TODO: deduplicate? */ - /* - TODO: the logic here parallels - Item_func_mvi_encode::val_str_ascii. A refactoring is called for - */ - do { - buf.length(0); - switch (je->state) + switch (event) { - case JST_ARRAY_START: - depth++; - break; - case JST_ARRAY_END: - if (--depth == 0) - return access; - /* - Closed a top-level element that was an array. See above: for an OR - it has to have contributed at least one key. - */ - if (depth == 1 && !conjunctive && - (access ? access->encoded.elements : 0) == keys_before_level2_array) - return NULL; - break; - case JST_VALUE: - { - if (json_read_value(je)) - return NULL; - if (je->state == JST_ARRAY_START) - { - if (++depth == 2) - keys_before_level2_array= access ? access->encoded.elements : 0; - break; - } - if (je->value_type == JSON_VALUE_OBJECT) - { - if (json_skip_level(je) || !conjunctive) - return NULL; - break; - } - if (encode_mvi_key(je, cast_th, cs, &buf)) - { - /* See above: only an AND of the keys tolerates a missing one */ - if (!conjunctive) - return NULL; - break; - } + case MVI_KEY: if (!access && !(access= new (thd->mem_root) Mvi_access(index, conjunctive))) return NULL; if (access->add_key(thd->mem_root, &buf)) return NULL; + buf.length(0); /* The next key starts fresh */ + break; + case MVI_NO_KEY: + /* Only an AND of the keys tolerates a missing one, see above */ + if (!conjunctive) + return NULL; + break; + case MVI_NESTED_START: + if (it.depth() == 2) + keys_before_level2_array= mvi_key_count(access); + break; + case MVI_NESTED_END: + /* An element that is an array and yielded no key at all */ + if (it.depth() == 2 && !conjunctive && + mvi_key_count(access) == keys_before_level2_array) + return NULL; break; - } default: - return NULL; + DBUG_ASSERT(0); /* The walk has not stopped */ } - } while (json_scan_next(je) == 0); + } + + if (event == MVI_WALK_END) + return access; + if (event != MVI_WALK_NOT_ARRAY) + return NULL; - return depth > 0 ? NULL : access; + /* A scalar: JSON_CONTAINS(expr, '123'). It is in *je */ + if (je->value_type == JSON_VALUE_UNINITIALIZED || + je->value_type == JSON_VALUE_OBJECT || + encode_mvi_key(je, cast_th, cs, &buf)) + return NULL; + if (!(access= new (thd->mem_root) Mvi_access(index, conjunctive)) || + access->add_key(thd->mem_root, &buf)) + return NULL; + return access; } From 538507700e43dc2577fc53218610aabfc1e69ad4 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 10 Sep 2026 18:06:35 +1000 Subject: [PATCH 39/39] MDEV-40168 Make an over-long key a prefix key instead of no key A key image that does not fit in a fulltext token was rejected, which costs the index for that value entirely: no key in the document, and an OVERLAPS that mentions the value gives up on the index altogether. Cut the image down to MVI_KEY_IMAGE_MAX_LEN instead. Two values that agree on that many bytes then share a key, which costs false positives and nothing else, since the predicate is rechecked on every row the index produces. This is what the non-binary path has always done -- strnxfrm() is asked for exactly that many bytes of weights and cannot return more -- so it makes the binary path, which is the one a JSON column takes, behave the same. No wildcard is needed in the fulltext query for this. Both sides of the index cut at the same point, so the key a search builds for a long value is the same string as the token the document has for it, and an exact term match finds it. A trailing '*' would only widen the term to keys that are longer than the one searched for, and after the cut there are none. Co-Authored-By: Claude Opus 5 --- mysql-test/main/multi_valued_index.result | 83 +++++++++++++++++++++++ mysql-test/main/multi_valued_index.test | 66 ++++++++++++++++++ sql/opt_multi_valued_index.cc | 23 +++++-- sql/opt_multi_valued_index.h | 4 +- 4 files changed, 168 insertions(+), 8 deletions(-) diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result index 3c9bdd0257e21..e899012b9b86b 100644 --- a/mysql-test/main/multi_valued_index.result +++ b/mysql-test/main/multi_valued_index.result @@ -393,6 +393,77 @@ c j 4 {"tags": [["ccc"]]} 5 {"tags": [[]]} drop table t1; +# A value whose key image is longer than a fulltext token can be is +# keyed by the first 42 bytes of it: the engine drops a token longer +# than that, on the DML path and on the index build path alike, and +# a value with no key at all is one the index cannot be used for. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(60) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'); +insert into t1 values (2, concat('{"tags": ["', repeat('b',43), '"]}')); +insert into t1 values (3, concat('{"tags": ["', repeat('c',42), '"]}')); +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +# the same rows without the index: +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +explain select c from t1 +where json_contains(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_contains(j->'$.tags', json_array(repeat('b',43))) order by c; +c +Warnings: +Warning 4036 Character disallowed in JSON in argument 2 to function 'json_contains' at position 45 +# a value that is exactly as long as the key image can be +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('c',42))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +c +3 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +c +3 +# Rows 2 and 4 agree on the first 42 bytes and share a key, so the +# scan reads both and the predicate keeps the one that matches. +insert into t1 values (4, concat('{"tags": ["', repeat('b',43), 'z"]}')); +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +# and the other way round +select c from t1 +where json_overlaps(j->'$.tags', +json_array(concat(repeat('b',43),'z'))) order by c; +c +4 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', +json_array(concat(repeat('b',43),'z'))) order by c; +c +4 +drop table t1; # The predicate does not have to be in the WHERE clause: for a table on # the inner side of an outer join we look at the ON expression, which is # what has to be true for the rows we read. @@ -956,3 +1027,15 @@ mvi_encode('[[1], 42]', int) select mvi_encode('[1]]', int); mvi_encode('[1]]', int) 8000000000000001 +select mvi_encode(concat('["', repeat('b',42), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',42), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["', repeat('b',43), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',43), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["', repeat('b',100), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',100), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)); +mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)) +414141 424242424242424242424242424242424242424242424242424242424242424242424242424242424242 diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test index d3614ce2cf90c..17d704409f736 100644 --- a/mysql-test/main/multi_valued_index.test +++ b/mysql-test/main/multi_valued_index.test @@ -237,6 +237,62 @@ select * from t1 where json_overlaps(j->'$.tags', '[[]]'); select * from t1 ignore index(idx) where json_overlaps(j->'$.tags', '[[]]'); drop table t1; +--echo # A value whose key image is longer than a fulltext token can be is +--echo # keyed by the first 42 bytes of it: the engine drops a token longer +--echo # than that, on the DML path and on the index build path alike, and +--echo # a value with no key at all is one the index cannot be used for. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(60) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'); +insert into t1 values (2, concat('{"tags": ["', repeat('b',43), '"]}')); +insert into t1 values (3, concat('{"tags": ["', repeat('c',42), '"]}')); + +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +--echo # the same rows without the index: +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; + +--replace_column 9 # +explain select c from t1 + where json_contains(j->'$.tags', json_array(repeat('b',43))); +# TODO: why is there a warning 4036 here? +select c from t1 + where json_contains(j->'$.tags', json_array(repeat('b',43))) order by c; + +--echo # a value that is exactly as long as the key image can be +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('c',42))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; + +--echo # Rows 2 and 4 agree on the first 42 bytes and share a key, so the +--echo # scan reads both and the predicate keeps the one that matches. +insert into t1 values (4, concat('{"tags": ["', repeat('b',43), 'z"]}')); +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +--echo # and the other way round +select c from t1 + where json_overlaps(j->'$.tags', + json_array(concat(repeat('b',43),'z'))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', + json_array(concat(repeat('b',43),'z'))) order by c; + +drop table t1; + --echo # The predicate does not have to be in the WHERE clause: for a table on --echo # the inner side of an outer join we look at the ON expression, which is --echo # what has to be true for the rows we read. @@ -592,3 +648,13 @@ select mvi_encode('[[1], 42]', int); # JSON_CONTAINS behaviour. NOTE that mysql would fail `select # JSON_CONTAINS('[1]]', '1');` but not mariadb select mvi_encode('[1]]', int); + +# A key image longer than 42 bytes does not fit in a fulltext token, so +# it is cut down to 42 and the key is a prefix key. Only a binary +# collation gets there: strnxfrm() is asked for 42 bytes and cannot +# return more. +select mvi_encode(concat('["', repeat('b',42), '"]'), char(60)); +# the same key as above +select mvi_encode(concat('["', repeat('b',43), '"]'), char(60)); +select mvi_encode(concat('["', repeat('b',100), '"]'), char(60)); +select mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)); diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc index b019de1fa1a5f..7dc013f7ee6fa 100644 --- a/sql/opt_multi_valued_index.cc +++ b/sql/opt_multi_valued_index.cc @@ -144,6 +144,16 @@ static void store_sort_key_longlong(uchar *to, bool unsigned_flag, If the value cannot be encoded this means it is not stored, also searches won't find any matches for it. + A key image longer than a fulltext token can be is cut short instead: + the engine drops a token that long, on the DML path and on the index + build path alike (fts_check_token()), and a value with no key in the + index is a value the index cannot be used for at all. Two values that + agree on the first MVI_KEY_IMAGE_MAX_LEN bytes of their image then + share a key, which costs false positives and nothing else -- the + predicate is rechecked on every row the index produces. The strnxfrm() + branch below has always worked that way; it asks for exactly that many + bytes of weights and cannot get more back. + @return false Encoded successfully, the key is appended to *buf true The JSON value cannot be represented in the index datatype. @@ -205,10 +215,14 @@ bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, return true; } - /* 2. hex */ + /* 2. cut what the engine would not index down to what it will, see above */ + if (sorted.length() > MVI_KEY_IMAGE_MAX_LEN) + sorted.length(MVI_KEY_IMAGE_MAX_LEN); + + /* 3. hex */ buf->append_hex(sorted.c_ptr(), sorted.length()); - /* 3. pad */ + /* 4. pad */ if (sorted.length() == 0) buf->append(STRING_WITH_LEN("xxxx")); else if (sorted.length() == 1) @@ -346,11 +360,6 @@ String *Item_func_mvi_encode::val_str_ascii(String *buf) if (buf->length()) buf->length(buf->length() - 1); - /* - TODO: do something different when an empty string is - returned, i.e. the document has no key at all, to avoid wasting - index space? - */ return buf; error_format: diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h index ed0962f5b0c00..b17a44426107b 100644 --- a/sql/opt_multi_valued_index.h +++ b/sql/opt_multi_valued_index.h @@ -95,7 +95,9 @@ enum json_value_types mvi_json_class(enum_field_types ftype); A key is one fulltext token, so it cannot be longer than the maximum token size the engine will index: 84 characters (HA_FT_MAXCHARLEN, which is also the default and the maximum of innodb_ft_max_token_size). The key - is the hex of the key image, so that image is at most half of it. + is the hex of the key image, so that image is at most half of it. An + image longer than that is cut down to it, making the key a prefix key: + see encode_mvi_key(). TODO: innodb_ft_max_token_size can be set lower than its default, and innodb_ft_min_token_size higher, and then the engine drops keys we