diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index d38f2a48c737b..6852fbb19a650 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -72,6 +72,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/mf_iocache.cc ../sql/my_decimal.cc ../sql/net_serv.cc ../sql/opt_range.cc ../sql/opt_group_by_cardinality.cc + ../sql/opt_multi_valued_index.cc ../sql/opt_rewrite_date_cmp.cc ../sql/opt_rewrite_remove_casefold.cc ../sql/opt_sargable_left.cc diff --git a/mysql-test/main/multi_valued_index.opt b/mysql-test/main/multi_valued_index.opt new file mode 100644 index 0000000000000..a076db4e5b886 --- /dev/null +++ b/mysql-test/main/multi_valued_index.opt @@ -0,0 +1 @@ +--innodb_ft_index_cache diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result new file mode 100644 index 0000000000000..e899012b9b86b --- /dev/null +++ b/mysql-test/main/multi_valued_index.result @@ -0,0 +1,1041 @@ +# basic tests +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; +insert into t1 values (1, '{"tags": [1, 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +insert into t1 values (2, '{"tags": ["1", "abcde", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +31xx 2 2 1 2 0 +6162636465 2 2 1 2 5 +xxxx 2 2 1 2 16 +insert into t1 values (3, '{"tags": ["1.0", "34567", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +312e30 3 3 1 3 0 +31xx 2 2 1 2 0 +3334353637 3 3 1 3 7 +6162636465 2 2 1 2 5 +xxxx 2 3 2 2 16 +xxxx 2 3 2 3 18 +insert into t1 values (4, '{}'); +insert into t1 values (5, '{"tags": [[["1"], "34567"]]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +312e30 3 3 1 3 0 +31xx 2 5 2 2 0 +31xx 2 5 2 5 0 +3334353637 3 5 2 3 7 +3334353637 3 5 2 5 5 +6162636465 2 2 1 2 5 +xxxx 2 3 2 2 16 +xxxx 2 3 2 3 18 +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +select * from t1 where json_contains(j->'$.tags', '1.0'); +c j +1 {"tags": [1, 34567]} +select * from t1 where json_contains(j->'$.tags', '1'); +c j +1 {"tags": [1, 34567]} +select * from t1 where json_contains(j->'$.tags', '"1"'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +5 {"tags": [[["1"], "34567"]]} +explain +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +explain +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +c j +select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); +c j +2 {"tags": ["1", "abcde", "", 34567]} +explain +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +c j +5 {"tags": [[["1"], "34567"]]} +select * from t1 where json_contains(j->'$.tags', '[[["34567"], "1"]'); +c j +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_contains' +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1", {}}]'); +c j +DROP TABLE t1; +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; +create table t1 (j json, key idx ((CAST(j->'$.n' AS INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as int array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS SIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as int array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as unsigned array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.n') as unsigned array))) +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +# top level or +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 1 Using where +DROP TABLE t1; +# MVI access is chosen on cost, and survives the re-optimization +# that a LIMIT triggers in make_join_select() +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); +explain select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +c j +1 {"tags": ["aaa"]} +# No MATCH() is added to the WHERE clause +# ("filtered" depends on the engine's row estimate, so hide it) +explain extended select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range idx idx 0 NULL # # Using where +Warnings: +Note 1003 select `test`.`t1`.`c` AS `c`,`test`.`t1`.`j` AS `j` from `test`.`t1` where json_contains(json_extract(`test`.`t1`.`j`,'$.tags'),'"aaa"') +# A predicate under a top-level OR cannot use the index: the scan +# would drop the rows that only match the other branch +explain select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +# The scan is re-initialized for each execution: MVI table on the +# inner side of a join, and in a correlated subquery +create table t2 (a int) engine=innodb; +insert into t2 values (1),(2),(3); +explain select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL # +1 SIMPLE t1 range idx idx 0 NULL # Using where; Using join buffer (flat, BNL join) +select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +a c j +1 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +set @@join_cache_level=0; +select * from t2 straight_join t1 +where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +a c j +1 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +set @@join_cache_level=default; +select a, (select count(*) from t1 +where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) from t2; +a (select count(*) from t1 +where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) +1 1 +2 0 +3 1 +drop table t1,t2; +# Repeated optimization: prepared statements and stored procedures. +# '"aaa"' can use the index; an object argument cannot, but still +# matches row 2, which the '"aaa"' index scan would not return. +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [{"a":1}]}'); +prepare s from 'select * from t1 where json_contains(j->''$.tags'', ?)'; +set @p='"aaa"'; +execute s using @p; +c j +1 {"tags": ["aaa"]} +set @p='{"a":1}'; +execute s using @p; +c j +2 {"tags": [{"a":1}]} +set @p='"aaa"'; +execute s using @p; +c j +1 {"tags": ["aaa"]} +deallocate prepare s; +create procedure p1(x json) select * from t1 where json_contains(j->'$.tags', x); +call p1('"aaa"'); +c j +1 {"tags": ["aaa"]} +call p1('{"a":1}'); +c j +2 {"tags": [{"a":1}]} +call p1('"aaa"'); +c j +1 {"tags": ["aaa"]} +drop procedure p1; +drop table t1; +# JSON_OVERLAPS is sargable too. It is true when ANY of the elements +# has a match, so the keys are ORed, and it is symmetric: the indexed +# expression can be either argument. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), +(5,'{"tags": [123]}'),(6,'{}'); +explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 4 Using where +select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# the indexed expression as the second argument +explain select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 4 Using where +select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": ["bbb"]} +4 {"tags": ["aaa","bbb"]} +# a scalar instead of an array +select * from t1 where json_overlaps(j->'$.tags','"ccc"') order by c; +c j +3 {"tags": ["ccc"]} +# 123 cannot be encoded for a CHAR array, so it is not in the index +# either. Row 5 matches the predicate through it, so an index scan +# for the remaining key would lose that row: don't use the index. +explain select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +4 {"tags": ["aaa","bbb"]} +5 {"tags": [123]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +4 {"tags": ["aaa","bbb"]} +5 {"tags": [123]} +# JSON_CONTAINS may skip such an element: dropping a key from an AND +# only makes the scan less selective, so it stays a superset. +explain select * from t1 where json_contains(j->'$.tags','[123,"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','[123,"aaa"]') order by c; +c j +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[123,"aaa"]') order by c; +c j +# Forms that cannot use the index. None of these may change the result. +select * from t1 where json_overlaps(j->'$.tags', NULL) order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','[]') order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','{"a":1}') order by c; +c j +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +# neither argument is a constant +select * from t1 a, t1 b +where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; +c j c j +1 {"tags": ["aaa"]} 1 {"tags": ["aaa"]} +1 {"tags": ["aaa"]} 4 {"tags": ["aaa","bbb"]} +drop table t1; +# JSON_OVERLAPS and nested arrays +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [["aaa"]]}'), +(3,'{"tags": [["aaa"],"ccc"]}'),(4,'{"tags": [["ccc"]]}'), +(5,'{"tags": [[]]}'); +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +explain select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +# the same when the element is nothing but empty arrays +explain select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +# An empty array deeper inside the depth-2 array does not +# matter: what the depth-2 array as a whole flattens to is not +# empty. +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +c j +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +c j +# JSON_CONTAINS is not affected: it flattens nested arrays itself, and +# dropping a key from an AND is always safe. +explain select * from t1 where json_contains(j->'$.tags','[[],"aaa"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select * from t1 where json_contains(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[[],"aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +# Wrong results are consistent. Bug separate from this feature. +explain +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 5 Using where +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +select * from t1 ignore index(idx) where json_overlaps(j->'$.tags', '[[]]'); +c j +2 {"tags": [["aaa"]]} +3 {"tags": [["aaa"],"ccc"]} +4 {"tags": [["ccc"]]} +5 {"tags": [[]]} +drop table t1; +# A value whose key image is longer than a fulltext token can be is +# keyed by the first 42 bytes of it: the engine drops a token longer +# than that, on the DML path and on the index build path alike, and +# a value with no key at all is one the index cannot be used for. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(60) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'); +insert into t1 values (2, concat('{"tags": ["', repeat('b',43), '"]}')); +insert into t1 values (3, concat('{"tags": ["', repeat('c',42), '"]}')); +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +# the same rows without the index: +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +explain select c from t1 +where json_contains(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_contains(j->'$.tags', json_array(repeat('b',43))) order by c; +c +Warnings: +Warning 4036 Character disallowed in JSON in argument 2 to function 'json_contains' at position 45 +# a value that is exactly as long as the key image can be +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('c',42))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +c +3 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +c +3 +# Rows 2 and 4 agree on the first 42 bytes and share a key, so the +# scan reads both and the predicate keeps the one that matches. +insert into t1 values (4, concat('{"tags": ["', repeat('b',43), 'z"]}')); +explain select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL # Using where +select c from t1 +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +c +2 +# and the other way round +select c from t1 +where json_overlaps(j->'$.tags', +json_array(concat(repeat('b',43),'z'))) order by c; +c +4 +select c from t1 ignore index(idx) +where json_overlaps(j->'$.tags', +json_array(concat(repeat('b',43),'z'))) order by c; +c +4 +drop table t1; +# The predicate does not have to be in the WHERE clause: for a table on +# the inner side of an outer join we look at the ON expression, which is +# what has to be true for the rows we read. +create table t0 (a int); +insert into t0 values (1),(2),(3); +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); +explain select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t0 ALL NULL NULL NULL NULL 3 +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using join buffer (flat, BNL join) +select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +a c j +1 1 {"tags": ["aaa"]} +1 3 {"tags": ["aaa","bbb"]} +2 1 {"tags": ["aaa"]} +2 3 {"tags": ["aaa","bbb"]} +3 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t0 left join t1 ignore index(idx) +on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +a c j +1 1 {"tags": ["aaa"]} +1 3 {"tags": ["aaa","bbb"]} +2 1 {"tags": ["aaa"]} +2 3 {"tags": ["aaa","bbb"]} +3 1 {"tags": ["aaa"]} +3 3 {"tags": ["aaa","bbb"]} +# The outer rows that find no match are still NULL-complemented +explain select * from t0 left join t1 +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t0 ALL NULL NULL NULL NULL 3 +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using join buffer (flat, BNL join) +select * from t0 left join t1 +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +a c j +1 1 {"tags": ["aaa"]} +2 NULL NULL +3 3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t0 left join t1 ignore index(idx) +on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +a c j +1 1 {"tags": ["aaa"]} +2 NULL NULL +3 3 {"tags": ["aaa","bbb"]} +drop table t0, t1; +# SHOW CREATE TABLE prints the expression the index was declared with. +# The column that holds the index keys is internal and is not printed. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +# That output is all it takes to re-create the index: feed it back in +create table t2 ( +`c` int(11) DEFAULT NULL, +`j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL +CHECK (json_valid(`j`)), +KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t2; +# An ALTER TABLE that rebuilds the table keeps the index +alter table t1 add column x int; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j x +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +# ... and it can be dropped and added by name like any other index +alter table t1 drop key idx; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +alter table t1 add key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `x` int(11) DEFAULT NULL, + KEY `idx2` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx2 idx2 0 NULL 2 Using where +drop table t1; +# An index over an ARRAY has exactly one key part. A key with several +# would have no defining expression to show and no way to be read back. +create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), +(CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; +ERROR 42000: Too many key parts specified; max 1 parts allowed +create table t1 (c int, j json, +key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; +ERROR 42000: Too many key parts specified; max 1 parts allowed +create table t1 (c int, j json, +key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR 42000: Too many key parts specified; max 1 parts allowed +# ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), +(CAST(j->'$.b' AS CHAR(6) ARRAY))); +ERROR 42000: Too many key parts specified; max 1 parts allowed +drop table t1; +# A plain KEY is the only thing an index over an ARRAY can be. The +# index is a fulltext index over the encoded elements underneath, which +# would not enforce UNIQUE or PRIMARY KEY, and which MATCH() would find +# nothing in. +create table t1 (j json, +unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +create table t1 (j json, +constraint u unique ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +create table t1 (j json, +primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of PRIMARY KEY and ARRAY +create table t1 (j json, +fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +ERROR HY000: Incorrect usage of FULLTEXT and ARRAY +# ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +alter table t1 add unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of UNIQUE and ARRAY +alter table t1 add primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of PRIMARY KEY and ARRAY +alter table t1 add fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +ERROR HY000: Incorrect usage of FULLTEXT and ARRAY +# the plain form is accepted +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.a') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +# MATCH() and a multi-valued index on the same table +create table t1 (c int, txt text, j json, +fulltext key ft (txt), +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'alpha beta','{"tags": ["aaa"]}'), +(2,'gamma','{"tags": ["bbb"]}'), +(3,'alpha','{"tags": ["aaa","bbb"]}'); +## mvi chosen +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +## ft chosen +explain select * from t1 +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext ft,idx ft 0 1 Using where +select * from t1 +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +c txt j +1 alpha beta {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +# the same rows without the index +select * from t1 ignore index(idx) +where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +c txt j +1 alpha beta {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +drop table t1; +# +# CREATE TABLE ... LIKE +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +create table t2 like t1; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +# one internal column, named from scratch in the new table +show index from t2; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t2 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t2 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t2, t1; +# +# Renaming the base column +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +alter table t1 rename column j to jj; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `jj` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`jj`)), + KEY `idx` ((cast(json_extract(`jj`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(jj->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(jj->'$.tags','"aaa"') order by c; +c jj +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(jj->'$.tags','"aaa"') order by c; +c jj +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +# ... and through CHANGE COLUMN, which renames and retypes at once +alter table t1 change jj j json; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +drop table t1; +# +# Dropping the base column +# +# The hidden column is computed from it, so the drop has to be refused +# while the index exists. It must not leave an index over a column +# that is gone. +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'); +alter table t1 drop column j; +ERROR 42S22: Unknown column 'j' in 'GENERATED ALWAYS AS' +# the table is unchanged and still usable +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +# dropping the index first makes it possible +alter table t1 drop key idx, drop column j; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop table t1; +# +# ALGORITHM=INSTANT / COPY / INPLACE +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'); +## adding a multi-valued index costs the table instant ADD COLUMN +alter table t1 add column x int, algorithm=instant; +ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +alter table t1 add column y int, algorithm=inplace; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `y` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j y +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +## the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j y +1 {"tags": ["aaa"]} NULL +3 {"tags": ["aaa","bbb"]} NULL +alter table t1 add column z int, algorithm=copy; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `y` int(11) DEFAULT NULL, + `z` int(11) DEFAULT NULL, + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where; Using filesort +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +c j y z +1 {"tags": ["aaa"]} NULL NULL +3 {"tags": ["aaa","bbb"]} NULL NULL +## adding the index in place to a table that already has rows +create table t2 (c int, j json) engine=innodb; +insert into t2 select c, j from t1; +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +## ... it needs a rebuild, which ALGORITHM=COPY does +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=copy; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range idx idx 0 NULL 2 Using where +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +## the same rows without the index: +select * from t2 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +## ... rebuild still needed needed even when a table already has a fulltext index +create table t3 (c int, txt text, j json, fulltext key ft (txt))engine=innodb; +insert into t3 select c, 'alpha', j from t1; +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +algorithm=copy; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `c` int(11) DEFAULT NULL, + `txt` text DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + FULLTEXT KEY `ft` (`txt`), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t3 where json_contains(j->'$.tags','"aaa"') order by c; +c txt j +1 alpha {"tags": ["aaa"]} +3 alpha {"tags": ["aaa","bbb"]} +drop table t3; +drop table t1, t2; +# +# Merging the accesses of several predicates on one index +# +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["aaa","bbb"]}'), +(4,'{"tags": ["aaa","ccc"]}'); +# Two JSON_CONTAINS on the same path are one index search for both +# keys, not a search for one of them with the other left to the +# WHERE clause. +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"bbb"') order by c; +c j +3 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"bbb"') order by c; +c j +3 {"tags": ["aaa","bbb"]} +# a key we already search for is not searched for twice +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.tags','"aaa"') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +select * from t1 where json_contains(j->'$.tags','["aaa","aaa"]') order by c; +c j +1 {"tags": ["aaa"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# JSON_OVERLAPS does not merge: (a OR b) AND (c OR d) is not a +# boolean-mode query we can build, so the two stay separate +# candidates and the cheaper one is used. +select * from t1 +where json_overlaps(j->'$.tags','["aaa","bbb"]') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +2 {"tags": ["bbb"]} +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# ... and a conjunctive and a disjunctive access on one index do not +# merge either +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +# the same rows without the index: +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','"aaa"') +and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +c j +3 {"tags": ["aaa","bbb"]} +4 {"tags": ["aaa","ccc"]} +drop table t1; +# Accesses on two different indexes cannot be merged into one +# fulltext search, so the cheaper one is chosen and the other +# predicate is left to the WHERE clause. +create table t1 (c int, j json, +key idx_t ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), +key idx_n ((CAST(j->'$.nums' AS UNSIGNED ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"], "nums": [1,2]}'), +(2,'{"tags": ["bbb"], "nums": [2,3]}'), +(3,'{"tags": ["aaa"], "nums": [3,4]}'); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + KEY `idx_t` ((cast(json_extract(`j`,'$.tags') as char(6) array))), + KEY `idx_n` ((cast(json_extract(`j`,'$.nums') as unsigned array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +select * from t1 +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.nums','3') order by c; +c j +3 {"tags": ["aaa"], "nums": [3,4]} +# the same rows without the indexes: +select * from t1 ignore index(idx_t,idx_n) +where json_contains(j->'$.tags','"aaa"') +and json_contains(j->'$.nums','3') order by c; +c j +3 {"tags": ["aaa"], "nums": [3,4]} +drop table t1; +# direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +mvi_encode('[1, 42, "3"]', int) +8000000000000001 800000000000002a +select mvi_encode('[1, 42, "3"]', unsigned); +mvi_encode('[1, 42, "3"]', unsigned) +0000000000000001 000000000000002a +select mvi_encode('[1, 42, "3 "]', char(6)); +mvi_encode('[1, 42, "3 "]', char(6)) +33xx +select mvi_encode('[1, 42, " "]', char(6)); +mvi_encode('[1, 42, " "]', char(6)) +xxxx +select mvi_encode('[1, 42, "3 "]', binary(6)); +mvi_encode('[1, 42, "3 "]', binary(6)) +33xx +select mvi_encode('[1, 42]', char(6)); +mvi_encode('[1, 42]', char(6)) + +select mvi_encode('[1, 42, {}}]', char(6)); +mvi_encode('[1, 42, {}}]', char(6)) +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'mvi_encode' at position 11 +select mvi_encode('[[1], 42]', int); +mvi_encode('[[1], 42]', int) +8000000000000001 800000000000002a +select mvi_encode('[1]]', int); +mvi_encode('[1]]', int) +8000000000000001 +select mvi_encode(concat('["', repeat('b',42), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',42), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["', repeat('b',43), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',43), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["', repeat('b',100), '"]'), char(60)); +mvi_encode(concat('["', repeat('b',100), '"]'), char(60)) +424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +select mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)); +mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)) +414141 424242424242424242424242424242424242424242424242424242424242424242424242424242424242 diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test new file mode 100644 index 0000000000000..17d704409f736 --- /dev/null +++ b/mysql-test/main/multi_valued_index.test @@ -0,0 +1,660 @@ +--source include/have_innodb.inc + +--echo # basic tests + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +show index from t1; + +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; + +insert into t1 values (1, '{"tags": [1, 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (2, '{"tags": ["1", "abcde", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (3, '{"tags": ["1.0", "34567", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +insert into t1 values (4, '{}'); +insert into t1 values (5, '{"tags": [[["1"], "34567"]]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; + +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +select * from t1 where json_contains(j->'$.tags', '1.0'); +select * from t1 where json_contains(j->'$.tags', '1'); +select * from t1 where json_contains(j->'$.tags', '"1"'); +explain +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '["1", "abcde"]'); +explain +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '[1, "abcde"]'); +select * from t1 where json_contains(j->'$.tags', '["1", 34567]'); +explain +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1"]'); + +# Invalid JSONs, errors not raised during optimization +select * from t1 where json_contains(j->'$.tags', '[[["34567"], "1"]'); +select * from t1 where json_contains(j->'$.tags', '[["34567"], "1", {}}]'); + +DROP TABLE t1; + +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS SIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +create table t1 (j json, key idx ((CAST(j->'$.n' AS UNSIGNED INT ARRAY)))); +insert into t1 values ('{"n": [123, "456"]}'); +show create table t1; +drop table t1; + +--echo # top level or + +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; + +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); + +DROP TABLE t1; + +--echo # MVI access is chosen on cost, and survives the re-optimization +--echo # that a LIMIT triggers in make_join_select() + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); + +explain select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; +select * from t1 where json_contains(j->'$.tags','"aaa"') limit 1; + +--echo # No MATCH() is added to the WHERE clause +--echo # ("filtered" depends on the engine's row estimate, so hide it) +--replace_column 9 # 10 # +explain extended select * from t1 where json_contains(j->'$.tags','"aaa"'); + +--echo # A predicate under a top-level OR cannot use the index: the scan +--echo # would drop the rows that only match the other branch +--replace_column 9 # +explain select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; +select * from t1 where json_contains(j->'$.tags','"aaa"') or c=2; + +--echo # The scan is re-initialized for each execution: MVI table on the +--echo # inner side of a join, and in a correlated subquery +create table t2 (a int) engine=innodb; +insert into t2 values (1),(2),(3); +--replace_column 9 # +explain select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +set @@join_cache_level=0; +select * from t2 straight_join t1 + where json_contains(t1.j->'$.tags','"aaa"') and t2.a=t1.c; +set @@join_cache_level=default; +select a, (select count(*) from t1 + where json_contains(t1.j->'$.tags','"aaa"') and t1.c=t2.a) from t2; + +drop table t1,t2; + +--echo # Repeated optimization: prepared statements and stored procedures. +--echo # '"aaa"' can use the index; an object argument cannot, but still +--echo # matches row 2, which the '"aaa"' index scan would not return. + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [{"a":1}]}'); + +prepare s from 'select * from t1 where json_contains(j->''$.tags'', ?)'; +set @p='"aaa"'; +execute s using @p; +set @p='{"a":1}'; +execute s using @p; +set @p='"aaa"'; +execute s using @p; +deallocate prepare s; + +create procedure p1(x json) select * from t1 where json_contains(j->'$.tags', x); +call p1('"aaa"'); +call p1('{"a":1}'); +call p1('"aaa"'); +drop procedure p1; + +drop table t1; + +--echo # JSON_OVERLAPS is sargable too. It is true when ANY of the elements +--echo # has a match, so the keys are ORed, and it is symmetric: the indexed +--echo # expression can be either argument. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["ccc"]}'),(4,'{"tags": ["aaa","bbb"]}'), + (5,'{"tags": [123]}'),(6,'{}'); + +explain select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]'); +select * from t1 where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','["aaa","bbb"]') order by c; + +--echo # the indexed expression as the second argument +explain select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags'); +select * from t1 where json_overlaps('["aaa","bbb"]', j->'$.tags') order by c; + +--echo # a scalar instead of an array +select * from t1 where json_overlaps(j->'$.tags','"ccc"') order by c; + +--echo # 123 cannot be encoded for a CHAR array, so it is not in the index +--echo # either. Row 5 matches the predicate through it, so an index scan +--echo # for the remaining key would lose that row: don't use the index. +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[123,"aaa"]') order by c; + +--echo # JSON_CONTAINS may skip such an element: dropping a key from an AND +--echo # only makes the scan less selective, so it stays a superset. +explain select * from t1 where json_contains(j->'$.tags','[123,"aaa"]'); +select * from t1 where json_contains(j->'$.tags','[123,"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[123,"aaa"]') order by c; + +--echo # Forms that cannot use the index. None of these may change the result. +select * from t1 where json_overlaps(j->'$.tags', NULL) order by c; +select * from t1 where json_overlaps(j->'$.tags','[]') order by c; +select * from t1 where json_overlaps(j->'$.tags','{"a":1}') order by c; +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +--echo # neither argument is a constant +select * from t1 a, t1 b +where json_overlaps(a.j->'$.tags', b.j->'$.tags') and a.c=1 order by b.c; + +drop table t1; + +--echo # JSON_OVERLAPS and nested arrays +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": [["aaa"]]}'), + (3,'{"tags": [["aaa"],"ccc"]}'),(4,'{"tags": [["ccc"]]}'), + (5,'{"tags": [[]]}'); + +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]'); +select * from t1 where json_overlaps(j->'$.tags','[["aaa"]]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa"]]') order by c; + +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[],"aaa"]') order by c; + +--echo # the same when the element is nothing but empty arrays +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]'); +select * from t1 where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[[[]],"aaa"]') order by c; + +--echo # An empty array deeper inside the depth-2 array does not +--echo # matter: what the depth-2 array as a whole flattens to is not +--echo # empty. +--replace_column 9 # +explain select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]'); +select * from t1 where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; +select * from t1 ignore index(idx) +where json_overlaps(j->'$.tags','[["aaa",[]]]') order by c; + +--echo # JSON_CONTAINS is not affected: it flattens nested arrays itself, and +--echo # dropping a key from an AND is always safe. +--replace_column 9 # +explain select * from t1 where json_contains(j->'$.tags','[[],"aaa"]'); +select * from t1 where json_contains(j->'$.tags','[[],"aaa"]') order by c; +select * from t1 ignore index(idx) +where json_contains(j->'$.tags','[[],"aaa"]') order by c; + +--echo # Wrong results are consistent. Bug separate from this feature. +explain +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +select * from t1 where json_overlaps(j->'$.tags', '[[]]'); +select * from t1 ignore index(idx) where json_overlaps(j->'$.tags', '[[]]'); +drop table t1; + +--echo # A value whose key image is longer than a fulltext token can be is +--echo # keyed by the first 42 bytes of it: the engine drops a token longer +--echo # than that, on the DML path and on the index build path alike, and +--echo # a value with no key at all is one the index cannot be used for. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(60) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'); +insert into t1 values (2, concat('{"tags": ["', repeat('b',43), '"]}')); +insert into t1 values (3, concat('{"tags": ["', repeat('c',42), '"]}')); + +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +--echo # the same rows without the index: +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; + +--replace_column 9 # +explain select c from t1 + where json_contains(j->'$.tags', json_array(repeat('b',43))); +# TODO: why is there a warning 4036 here? +select c from t1 + where json_contains(j->'$.tags', json_array(repeat('b',43))) order by c; + +--echo # a value that is exactly as long as the key image can be +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('c',42))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('c',42))) order by c; + +--echo # Rows 2 and 4 agree on the first 42 bytes and share a key, so the +--echo # scan reads both and the predicate keeps the one that matches. +insert into t1 values (4, concat('{"tags": ["', repeat('b',43), 'z"]}')); +--replace_column 9 # +explain select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))); +select c from t1 + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', json_array(repeat('b',43))) order by c; +--echo # and the other way round +select c from t1 + where json_overlaps(j->'$.tags', + json_array(concat(repeat('b',43),'z'))) order by c; +select c from t1 ignore index(idx) + where json_overlaps(j->'$.tags', + json_array(concat(repeat('b',43),'z'))) order by c; + +drop table t1; + +--echo # The predicate does not have to be in the WHERE clause: for a table on +--echo # the inner side of an outer join we look at the ON expression, which is +--echo # what has to be true for the rows we read. + +create table t0 (a int); +insert into t0 values (1),(2),(3); +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'),(4,'{}'); + +explain select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"'); +select * from t0 left join t1 on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; +--echo # the same rows without the index: +select * from t0 left join t1 ignore index(idx) + on json_contains(t1.j->'$.tags','"aaa"') +order by t0.a, t1.c; + +--echo # The outer rows that find no match are still NULL-complemented +explain select * from t0 left join t1 + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a; +select * from t0 left join t1 + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; +--echo # the same rows without the index: +select * from t0 left join t1 ignore index(idx) + on json_contains(t1.j->'$.tags','"aaa"') and t1.c=t0.a +order by t0.a; + +drop table t0, t1; + +--echo # SHOW CREATE TABLE prints the expression the index was declared with. +--echo # The column that holds the index keys is internal and is not printed. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); +show create table t1; +show index from t1; + +--echo # That output is all it takes to re-create the index: feed it back in +create table t2 ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + CHECK (json_valid(`j`)), + KEY `idx` ((cast(json_extract(`j`,'$.tags') as char(6) array))) +) ENGINE=InnoDB; +show create table t2; +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +drop table t2; + +--echo # An ALTER TABLE that rebuilds the table keeps the index +alter table t1 add column x int; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo # ... and it can be dropped and added by name like any other index +alter table t1 drop key idx; +show create table t1; +alter table t1 add key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))); +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); + +drop table t1; + +--echo # An index over an ARRAY has exactly one key part. A key with several +--echo # would have no defining expression to show and no way to be read back. +--error ER_TOO_MANY_KEY_PARTS +create table t1 (j json, key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), + (CAST(j->'$.b' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_TOO_MANY_KEY_PARTS +create table t1 (c int, j json, + key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)),c))engine=innodb; +--error ER_TOO_MANY_KEY_PARTS +create table t1 (c int, j json, + key idx (c,(CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--echo # ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +--error ER_TOO_MANY_KEY_PARTS +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY)), + (CAST(j->'$.b' AS CHAR(6) ARRAY))); +drop table t1; + +--echo # A plain KEY is the only thing an index over an ARRAY can be. The +--echo # index is a fulltext index over the encoded elements underneath, which +--echo # would not enforce UNIQUE or PRIMARY KEY, and which MATCH() would find +--echo # nothing in. +--error ER_WRONG_USAGE +create table t1 (j json, + unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + constraint u unique ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; +--error ER_WRONG_USAGE +create table t1 (j json, + fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))))engine=innodb; + +--echo # ... and the same through ALTER TABLE +create table t1 (j json)engine=innodb; +--error ER_WRONG_USAGE +alter table t1 add unique key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--error ER_WRONG_USAGE +alter table t1 add primary key ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--error ER_WRONG_USAGE +alter table t1 add fulltext key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +--echo # the plain form is accepted +alter table t1 add key idx ((CAST(j->'$.a' AS CHAR(6) ARRAY))); +show create table t1; +drop table t1; + +--echo # MATCH() and a multi-valued index on the same table +create table t1 (c int, txt text, j json, + fulltext key ft (txt), + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'alpha beta','{"tags": ["aaa"]}'), + (2,'gamma','{"tags": ["bbb"]}'), + (3,'alpha','{"tags": ["aaa","bbb"]}'); + +--echo ## mvi chosen +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +--echo ## ft chosen +explain select * from t1 + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"'); +select * from t1 + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; +--echo # the same rows without the index +select * from t1 ignore index(idx) + where match(txt) against('alpha') and json_contains(j->'$.tags','"aaa"') +order by c; + +drop table t1; + +--echo # +--echo # CREATE TABLE ... LIKE +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +create table t2 like t1; +show create table t2; +--echo # one internal column, named from scratch in the new table +show index from t2; +insert into t2 select * from t1; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +--echo # the same rows without the index: +select * from t2 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +drop table t2, t1; + +--echo # +--echo # Renaming the base column +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +alter table t1 rename column j to jj; +show create table t1; +explain select * from t1 where json_contains(jj->'$.tags','"aaa"'); +select * from t1 where json_contains(jj->'$.tags','"aaa"') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(jj->'$.tags','"aaa"') order by c; + +--echo # ... and through CHANGE COLUMN, which renames and retypes at once +alter table t1 change jj j json; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +drop table t1; + +--echo # +--echo # Dropping the base column +--echo # +--echo # The hidden column is computed from it, so the drop has to be refused +--echo # while the index exists. It must not leave an index over a column +--echo # that is gone. + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'); + +--error ER_BAD_FIELD_ERROR +alter table t1 drop column j; +--echo # the table is unchanged and still usable +show create table t1; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo # dropping the index first makes it possible +alter table t1 drop key idx, drop column j; +show create table t1; + +drop table t1; + +--echo # +--echo # ALGORITHM=INSTANT / COPY / INPLACE +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'); + +--echo ## adding a multi-valued index costs the table instant ADD COLUMN +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 add column x int, algorithm=instant; +show create table t1; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +alter table t1 add column y int, algorithm=inplace; +show create table t1; +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +--echo ## the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +alter table t1 add column z int, algorithm=copy; +show create table t1; +explain +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; +select * from t1 where json_contains(j->'$.tags','"aaa"') order by c; + +--echo ## adding the index in place to a table that already has rows +create table t2 (c int, j json) engine=innodb; +insert into t2 select c, j from t1; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=inplace; + +--echo ## ... it needs a rebuild, which ALGORITHM=COPY does +alter table t2 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=copy; +show create table t2; +explain select * from t2 where json_contains(j->'$.tags','"aaa"'); +select * from t2 where json_contains(j->'$.tags','"aaa"') order by c; +--echo ## the same rows without the index: +select * from t2 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') order by c; + +--echo ## ... rebuild still needed needed even when a table already has a fulltext index +create table t3 (c int, txt text, j json, fulltext key ft (txt))engine=innodb; +insert into t3 select c, 'alpha', j from t1; +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=inplace; +alter table t3 add key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + algorithm=copy; +show create table t3; +select * from t3 where json_contains(j->'$.tags','"aaa"') order by c; +drop table t3; +drop table t1, t2; + +--echo # +--echo # Merging the accesses of several predicates on one index +--echo # + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["aaa","bbb"]}'), + (4,'{"tags": ["aaa","ccc"]}'); + +--echo # Two JSON_CONTAINS on the same path are one index search for both +--echo # keys, not a search for one of them with the other left to the +--echo # WHERE clause. +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"bbb"') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"bbb"') order by c; + +--echo # a key we already search for is not searched for twice +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.tags','"aaa"') order by c; +select * from t1 where json_contains(j->'$.tags','["aaa","aaa"]') order by c; + +--echo # JSON_OVERLAPS does not merge: (a OR b) AND (c OR d) is not a +--echo # boolean-mode query we can build, so the two stay separate +--echo # candidates and the cheaper one is used. +select * from t1 + where json_overlaps(j->'$.tags','["aaa","bbb"]') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_overlaps(j->'$.tags','["aaa","bbb"]') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; + +--echo # ... and a conjunctive and a disjunctive access on one index do not +--echo # merge either +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; +--echo # the same rows without the index: +select * from t1 ignore index(idx) + where json_contains(j->'$.tags','"aaa"') + and json_overlaps(j->'$.tags','["bbb","ccc"]') order by c; + +drop table t1; + +--echo # Accesses on two different indexes cannot be merged into one +--echo # fulltext search, so the cheaper one is chosen and the other +--echo # predicate is left to the WHERE clause. + +create table t1 (c int, j json, + key idx_t ((CAST(j->'$.tags' AS CHAR(6) ARRAY))), + key idx_n ((CAST(j->'$.nums' AS UNSIGNED ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"], "nums": [1,2]}'), + (2,'{"tags": ["bbb"], "nums": [2,3]}'), + (3,'{"tags": ["aaa"], "nums": [3,4]}'); +show create table t1; + +select * from t1 + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.nums','3') order by c; +--echo # the same rows without the indexes: +select * from t1 ignore index(idx_t,idx_n) + where json_contains(j->'$.tags','"aaa"') + and json_contains(j->'$.nums','3') order by c; + +drop table t1; + +--echo # direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +select mvi_encode('[1, 42, "3"]', unsigned); +select mvi_encode('[1, 42, "3 "]', char(6)); +select mvi_encode('[1, 42, " "]', char(6)); +select mvi_encode('[1, 42, "3 "]', binary(6)); +select mvi_encode('[1, 42]', char(6)); +select mvi_encode('[1, 42, {}}]', char(6)); +select mvi_encode('[[1], 42]', int); +# trailing junk after outer array is accepted, consistent with normal +# JSON_CONTAINS behaviour. NOTE that mysql would fail `select +# JSON_CONTAINS('[1]]', '1');` but not mariadb +select mvi_encode('[1]]', int); + +# A key image longer than 42 bytes does not fit in a fulltext token, so +# it is cut down to 42 and the key is a prefix key. Only a binary +# collation gets there: strnxfrm() is asked for 42 bytes and cannot +# return more. +select mvi_encode(concat('["', repeat('b',42), '"]'), char(60)); +# the same key as above +select mvi_encode(concat('["', repeat('b',43), '"]'), char(60)); +select mvi_encode(concat('["', repeat('b',100), '"]'), char(60)); +select mvi_encode(concat('["aaa", "', repeat('b',43), '"]'), char(60)); diff --git a/mysql-test/main/multi_valued_index_notembedded.result b/mysql-test/main/multi_valued_index_notembedded.result new file mode 100644 index 0000000000000..d9cfb56d503be --- /dev/null +++ b/mysql-test/main/multi_valued_index_notembedded.result @@ -0,0 +1,145 @@ +create table t1 (c int, j json, +key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb","ccc"]}'); +# The trace prints the element keys in their encoded form, which is not +# readable. These are the tokens the index is built from, so the values +# the trace prints below must be found here: +select c, mvi_encode(j->'$.tags', char(6)) from t1; +c mvi_encode(j->'$.tags', char(6)) +1 616161 +2 626262 636363 +set optimizer_trace=1; +# +# One element key +# +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "match": "all", + "rows": 1, + "cost": 0.00171364, + "ranges": + ["616161"] + } +] +# +# Several element keys. A row has to have all of them, so the rarest +# one alone already reads every row we may return: only that key is +# left in "ranges", and the JSON predicate in the WHERE clause discards +# whatever else the shorter search finds. +# +create table t3 (c int, j json, +key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), +(3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +select c, mvi_encode(j->'$.tags', char(6)) from t3; +c mvi_encode(j->'$.tags', char(6)) +1 616161 626262 +2 626262 +3 626262 +4 626262 636363 +# "bbb" is in every row and "aaa" in one, so "aaa" is what we search for +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t3 range idx3 idx3 0 NULL 1 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t3", + "index": "idx3", + "match": "all", + "rows": 1, + "cost": 0.00171364, + "ranges": + ["616161"] + } +] +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +# the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +c j +1 {"tags": ["aaa","bbb"]} +drop table t3; +# +# Two tables: each entry names the table it belongs to +# +create table t2 (c int, j json, +key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t2 values (1,'{"tags": ["zzz"]}'); +explain select * from t1,t2 where json_contains(t1.j->'$.tags','"aaa"') +and json_contains(t2.j->'$.tags','"zzz"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 1 Using where +1 SIMPLE t2 range idx2 idx2 0 NULL 1 Using where; Using join buffer (flat, BNL join) +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "match": "all", + "rows": 1, + "cost": 0.00171364, + "ranges": + ["616161"] + }, + { + "table": "t2", + "index": "idx2", + "match": "all", + "rows": 1, + "cost": 0.00171364, + "ranges": + ["7a7a7a"] + } +] +# +# JSON_OVERLAPS: the keys are ORed, so "match" is "any". A row matches +# through any one of them, so the estimates add up and every key stays +# in the query. +# +explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx idx 0 NULL 2 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +[ + { + "table": "t1", + "index": "idx", + "match": "any", + "rows": 2, + "cost": 0.00260808, + "ranges": + [ + "626262", + "636363" + ] + } +] +# +# A predicate under a top-level OR gives no access, so nothing is printed +# +explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 Using where +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +jd +NULL +set optimizer_trace=default; +drop table t1,t2; diff --git a/mysql-test/main/multi_valued_index_notembedded.test b/mysql-test/main/multi_valued_index_notembedded.test new file mode 100644 index 0000000000000..a8f3a2de96918 --- /dev/null +++ b/mysql-test/main/multi_valued_index_notembedded.test @@ -0,0 +1,90 @@ +--source include/have_innodb.inc +# The test uses the optimizer trace: +--source include/not_embedded.inc + + +create table t1 (c int, j json, + key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t1 values (1,'{"tags": ["aaa"]}'),(2,'{"tags": ["bbb","ccc"]}'); + +--echo # The trace prints the element keys in their encoded form, which is not +--echo # readable. These are the tokens the index is built from, so the values +--echo # the trace prints below must be found here: +select c, mvi_encode(j->'$.tags', char(6)) from t1; + +set optimizer_trace=1; + +--echo # +--echo # One element key +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_contains(j->'$.tags','"aaa"'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # Several element keys. A row has to have all of them, so the rarest +--echo # one alone already reads every row we may return: only that key is +--echo # left in "ranges", and the JSON predicate in the WHERE clause discards +--echo # whatever else the shorter search finds. +--echo # +create table t3 (c int, j json, + key idx3 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t3 values (1,'{"tags": ["aaa","bbb"]}'),(2,'{"tags": ["bbb"]}'), + (3,'{"tags": ["bbb"]}'),(4,'{"tags": ["bbb","ccc"]}'); +select c, mvi_encode(j->'$.tags', char(6)) from t3; + +--echo # "bbb" is in every row and "aaa" in one, so "aaa" is what we search for +--disable_replay next_query Need to preserve optimizer trace +explain select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol +select * from t3 where json_contains(j->'$.tags','["aaa","bbb"]') order by c; +--echo # the same rows without the index: +select * from t3 ignore index(idx3) +where json_contains(j->'$.tags','["aaa","bbb"]') order by c; + +drop table t3; + +--echo # +--echo # Two tables: each entry names the table it belongs to +--echo # +create table t2 (c int, j json, + key idx2 ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +insert into t2 values (1,'{"tags": ["zzz"]}'); +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1,t2 where json_contains(t1.j->'$.tags','"aaa"') + and json_contains(t2.j->'$.tags','"zzz"'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # JSON_OVERLAPS: the keys are ORed, so "match" is "any". A row matches +--echo # through any one of them, so the estimates add up and every key stays +--echo # in the query. +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_overlaps(j->'$.tags','["bbb","ccc"]'); +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +--echo # +--echo # A predicate under a top-level OR gives no access, so nothing is printed +--echo # +--disable_replay next_query Need to preserve optimizer trace +explain select * from t1 where json_contains(t1.j->'$.tags','"aaa"') or c=2; +--disable_view_protocol +select json_detailed(json_extract(trace, '$**.multi_value_index_use')) as jd +from information_schema.optimizer_trace; +--enable_view_protocol + +set optimizer_trace=default; +drop table t1,t2; diff --git a/mysql-test/suite/innodb_fts/r/estimate.result b/mysql-test/suite/innodb_fts/r/estimate.result new file mode 100644 index 0000000000000..32e8ecedd1a15 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate.result @@ -0,0 +1,181 @@ +SET @optimize= @@GLOBAL.innodb_optimize_fulltext_only; +SET GLOBAL innodb_optimize_fulltext_only= 1; +CREATE TABLE t1 ( +id INT PRIMARY KEY, +a TEXT, +FULLTEXT(a) +) ENGINE=InnoDB; +INSERT INTO t1 VALUES +(1,'alpha beta gamma'), (2,'beta gamma'), (3,'beta gamma'), +(4,'gamma'), (5,'gamma'), (6,'gamma'), (7,'gamma'), (8,'gamma'); +# +# Nothing has been SYNCed yet, so the words live only in the in-memory +# FTS cache. The estimate consults it, and since every cache node +# carries its own doc_count the answer is exact. +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('beta')= 3 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('gamma')= 8 +# +# A word that is in neither the cache nor the auxiliary table, while the +# auxiliary table is still empty: that is no information at all rather +# than "no matching rows", so the answer is "unknown". The cache on its +# own cannot prove a word absent, because it is only complete once +# fts_init_index() has run, and an estimate must not run it. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('zzzzz')= unknown +SET debug_dbug=''; +# +# Flush the cache into the auxiliary tables. +# +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +# +# Each word now occupies a single auxiliary record, so the walk covers +# the whole of its key range and the estimate is exact. +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('beta')= 3 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('gamma')= 8 +# +# A word that is not indexed at all: floored to 1, not 0. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('zzzzz')= 1 +# +# The word is folded with the collation of the fulltext index, so case +# does not matter. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('GAMMA'); +COUNT(*) +8 +Warnings: +Note 1105 fulltext_estimate('GAMMA')= 8 +SET debug_dbug=''; +# +# Cross-check against the authoritative auxiliary table contents. +# +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +# One auxiliary record per word here, so DOC_COUNT is the exact answer. +SELECT DISTINCT WORD, DOC_COUNT FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE +ORDER BY WORD; +WORD DOC_COUNT +alpha 1 +beta 3 +gamma 8 +# +# Rows inserted after the SYNC are back in the cache, while the older +# ones are in the auxiliary table. The two counts are added, and no +# document is counted twice: gamma is 8 + 2, beta 3 + 1, alpha 1 + 0. +# +INSERT INTO t1 VALUES (9,'beta gamma'), (10,'gamma'); +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +1 +Warnings: +Note 1105 fulltext_estimate('alpha')= 1 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +4 +Warnings: +Note 1105 fulltext_estimate('beta')= 4 +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +10 +Warnings: +Note 1105 fulltext_estimate('gamma')= 10 +SET debug_dbug=''; +# +# Back to the state the rest of the test expects. +# +DELETE FROM t1 WHERE id > 8; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +# +# Deleted rows are deliberately still counted: their entries survive in +# the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 +# is then clamped to the number of rows left in the table, so 4 is +# reported -- the clamp, not the deletions, is what moved the number. +# +DELETE FROM t1 WHERE id > 4; +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze Warning Engine-independent statistics are not collected for column 'a' +test.t1 analyze status OK +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +4 +Warnings: +Note 1105 fulltext_estimate('gamma')= 4 +SET debug_dbug=''; +# +# An engine that does not implement fulltext_estimate() reports +# "unknown" via the handler default. +# +SET GLOBAL innodb_ft_aux_table= @aux; +ALTER TABLE t1 ENGINE=MyISAM; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +0 +Warnings: +Note 1105 fulltext_estimate('gamma')= unknown +SET debug_dbug=''; +DROP TABLE t1; +SET GLOBAL innodb_optimize_fulltext_only= @optimize; +# +# A restart empties the cache, and the estimate does not repopulate it: +# fts_init_index() has side effects that an estimate must not have. A +# clean shutdown has SYNCed the words, so the auxiliary table answers on +# its own. +# +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1,'delta'), (2,'delta'), (3,'delta'); +# restart +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('delta'); +COUNT(*) +3 +Warnings: +Note 1105 fulltext_estimate('delta')= 3 +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/r/estimate_big.result b/mysql-test/suite/innodb_fts/r/estimate_big.result new file mode 100644 index 0000000000000..16cc65e19579a --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate_big.result @@ -0,0 +1,46 @@ +# +# Case 1: the word's documents are evenly spread over its doc id range, +# which is what the extrapolation assumes. +# +CREATE TABLE t1 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET debug_dbug=''; +# One auxiliary record per insert, well past the FTS_EST_MAX_RECS +# budget of 64, so the answer below is extrapolated rather than counted. +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +SELECT COUNT(*) AS n_records FROM +(SELECT DISTINCT WORD, FIRST_DOC_ID, DOC_COUNT +FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE WHERE WORD='gamma') x; +n_records +80 +SET GLOBAL innodb_ft_aux_table= @aux; +# Evenly spread, so the extrapolation lands on the true count. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +80 +Warnings: +Note 1105 fulltext_estimate('gamma')= 80 +SET debug_dbug=''; +DROP TABLE t1; +# +# Case 2: the documented limitation. When a word occurs only at both +# ends of the doc id range, the density measured over the sampled prefix +# does not describe the gap in the middle, and the estimate is far too +# high. Two probes cannot tell this apart from a word that really does +# occur throughout, so the estimator errs on the high side. +# +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET debug_dbug=''; +# 71 rows really match, but the estimate is pulled up towards the size +# of the doc id span, and then clamped to the number of rows. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('gamma'); +COUNT(*) +71 +Warnings: +Note 1105 fulltext_estimate('gamma')= 471 +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/r/estimate_debug.result b/mysql-test/suite/innodb_fts/r/estimate_debug.result new file mode 100644 index 0000000000000..315ba583b1df2 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/estimate_debug.result @@ -0,0 +1,47 @@ +CREATE TABLE t1 ( +id INT AUTO_INCREMENT PRIMARY KEY, +a VARCHAR(64), +FULLTEXT(a) +) ENGINE=InnoDB; +INSERT INTO t1(a) SELECT 'alpha' FROM seq_1_to_40; +INSERT INTO t1(a) SELECT 'beta' FROM seq_1_to_60; +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connection con1; +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET DEBUG_SYNC='fts_write_node SIGNAL written WAIT_FOR go'; +INSERT INTO t1(a) VALUES('zeta'); +connection con2; +SET DEBUG_SYNC='now WAIT_FOR written'; +# +# alpha is now in the auxiliary table and still in the cache. 40, not +# 80. +# +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +COUNT(*) +40 +Warnings: +Note 1105 fulltext_estimate('alpha')= 40 +# +# beta has not been written yet, so it is counted from the cache alone. +# +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +COUNT(*) +60 +Warnings: +Note 1105 fulltext_estimate('beta')= 60 +SET debug_dbug=''; +connection default; +SET DEBUG_SYNC='now SIGNAL go'; +connection con1; +SET DEBUG_SYNC='RESET'; +SET debug_dbug=''; +connection default; +disconnect con1; +disconnect con2; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/estimate.opt b/mysql-test/suite/innodb_fts/t/estimate.opt new file mode 100644 index 0000000000000..444dfb0dcf530 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate.opt @@ -0,0 +1 @@ +--innodb-ft-index-table diff --git a/mysql-test/suite/innodb_fts/t/estimate.test b/mysql-test/suite/innodb_fts/t/estimate.test new file mode 100644 index 0000000000000..77f188dfae25c --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate.test @@ -0,0 +1,137 @@ +# +# handler::fulltext_estimate() -- the fulltext analogue of records_in_range(). +# +# The estimate is read back through a debug-only hook in +# Item_func_match::init_search(), which pushes it as a note. +# +--source include/have_innodb.inc +--source include/have_debug.inc + +SET @optimize= @@GLOBAL.innodb_optimize_fulltext_only; +SET GLOBAL innodb_optimize_fulltext_only= 1; + +CREATE TABLE t1 ( + id INT PRIMARY KEY, + a TEXT, + FULLTEXT(a) +) ENGINE=InnoDB; + +# gamma appears in all 8 rows, beta in 3, alpha in 1. +INSERT INTO t1 VALUES + (1,'alpha beta gamma'), (2,'beta gamma'), (3,'beta gamma'), + (4,'gamma'), (5,'gamma'), (6,'gamma'), (7,'gamma'), (8,'gamma'); + +--echo # +--echo # Nothing has been SYNCed yet, so the words live only in the in-memory +--echo # FTS cache. The estimate consults it, and since every cache node +--echo # carries its own doc_count the answer is exact. +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); + +--echo # +--echo # A word that is in neither the cache nor the auxiliary table, while the +--echo # auxiliary table is still empty: that is no information at all rather +--echo # than "no matching rows", so the answer is "unknown". The cache on its +--echo # own cannot prove a word absent, because it is only complete once +--echo # fts_init_index() has run, and an estimate must not run it. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); +SET debug_dbug=''; + +--echo # +--echo # Flush the cache into the auxiliary tables. +--echo # +OPTIMIZE TABLE t1; + +--echo # +--echo # Each word now occupies a single auxiliary record, so the walk covers +--echo # the whole of its key range and the estimate is exact. +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); + +--echo # +--echo # A word that is not indexed at all: floored to 1, not 0. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('zzzzz'); + +--echo # +--echo # The word is folded with the collation of the fulltext index, so case +--echo # does not matter. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('GAMMA'); +SET debug_dbug=''; + +--echo # +--echo # Cross-check against the authoritative auxiliary table contents. +--echo # +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +--echo # One auxiliary record per word here, so DOC_COUNT is the exact answer. +SELECT DISTINCT WORD, DOC_COUNT FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE + ORDER BY WORD; + +--echo # +--echo # Rows inserted after the SYNC are back in the cache, while the older +--echo # ones are in the auxiliary table. The two counts are added, and no +--echo # document is counted twice: gamma is 8 + 2, beta 3 + 1, alpha 1 + 0. +--echo # +INSERT INTO t1 VALUES (9,'beta gamma'), (10,'gamma'); +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +--echo # +--echo # Back to the state the rest of the test expects. +--echo # +DELETE FROM t1 WHERE id > 8; +OPTIMIZE TABLE t1; + +--echo # +--echo # Deleted rows are deliberately still counted: their entries survive in +--echo # the ilists until OPTIMIZE TABLE purges them. Here that raw count of 8 +--echo # is then clamped to the number of rows left in the table, so 4 is +--echo # reported -- the clamp, not the deletions, is what moved the number. +--echo # +DELETE FROM t1 WHERE id > 4; +# The clamp uses the table statistics, which a background recalculation may +# still be about to redo after that DELETE. Recalculate them here instead, +# so the number below does not depend on when that happens. +ANALYZE TABLE t1; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +--echo # +--echo # An engine that does not implement fulltext_estimate() reports +--echo # "unknown" via the handler default. +--echo # +SET GLOBAL innodb_ft_aux_table= @aux; +ALTER TABLE t1 ENGINE=MyISAM; +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t1; +SET GLOBAL innodb_optimize_fulltext_only= @optimize; + +--echo # +--echo # A restart empties the cache, and the estimate does not repopulate it: +--echo # fts_init_index() has side effects that an estimate must not have. A +--echo # clean shutdown has SYNCed the words, so the auxiliary table answers on +--echo # its own. +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1,'delta'), (2,'delta'), (3,'delta'); +--source include/restart_mysqld.inc +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('delta'); +SET debug_dbug=''; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/t/estimate_big.opt b/mysql-test/suite/innodb_fts/t/estimate_big.opt new file mode 100644 index 0000000000000..444dfb0dcf530 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_big.opt @@ -0,0 +1 @@ +--innodb-ft-index-table diff --git a/mysql-test/suite/innodb_fts/t/estimate_big.test b/mysql-test/suite/innodb_fts/t/estimate_big.test new file mode 100644 index 0000000000000..bc60069edbba9 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_big.test @@ -0,0 +1,88 @@ +# +# handler::fulltext_estimate() when a word owns more auxiliary records than +# the estimator is willing to read, so it extrapolates instead of counting. +# +# Note: the numbers below depend on how many auxiliary records each word ends +# up occupying, so they have to be re-recorded if the FTS node layout or the +# FTS_EST_MAX_RECS / FTS_EST_MAX_PAGES budgets change. +# +--source include/have_innodb.inc +--source include/have_debug.inc + +--echo # +--echo # Case 1: the word's documents are evenly spread over its doc id range, +--echo # which is what the extrapolation assumes. +--echo # +CREATE TABLE t1 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; + +# Force a synchronous cache flush on every commit, so each single-row +# transaction leaves a node of its own in the auxiliary table. This is what a +# table that takes incremental inserts and is never OPTIMIZEd looks like. +SET debug_dbug='+d,fts_instrument_sync_debug'; +--disable_query_log +let $i= 0; +while ($i < 80) +{ + inc $i; + eval INSERT INTO t1 VALUES ($i, 'gamma'); +} +# 40 more rows without the word, so the table has clearly more rows than the +# word has documents. That keeps the row count clamp out of the picture and +# leaves the extrapolated value itself visible below. +while ($i < 120) +{ + inc $i; + eval INSERT INTO t1 VALUES ($i, 'delta'); +} +--enable_query_log +SET debug_dbug=''; + +--echo # One auxiliary record per insert, well past the FTS_EST_MAX_RECS +--echo # budget of 64, so the answer below is extrapolated rather than counted. +SET @aux= @@GLOBAL.innodb_ft_aux_table; +SET GLOBAL innodb_ft_aux_table='test/t1'; +SELECT COUNT(*) AS n_records FROM + (SELECT DISTINCT WORD, FIRST_DOC_ID, DOC_COUNT + FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE WHERE WORD='gamma') x; +SET GLOBAL innodb_ft_aux_table= @aux; + +--echo # Evenly spread, so the extrapolation lands on the true count. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t1; + +--echo # +--echo # Case 2: the documented limitation. When a word occurs only at both +--echo # ends of the doc id range, the density measured over the sampled prefix +--echo # does not describe the gap in the middle, and the estimate is far too +--echo # high. Two probes cannot tell this apart from a word that really does +--echo # occur throughout, so the estimator errs on the high side. +--echo # +CREATE TABLE t2 (id INT PRIMARY KEY, a TEXT, FULLTEXT(a)) ENGINE=InnoDB; + +SET debug_dbug='+d,fts_instrument_sync_debug'; +--disable_query_log +let $i= 0; +while ($i < 70) +{ + inc $i; + eval INSERT INTO t2 VALUES ($i, 'gamma'); +} +while ($i < 470) +{ + inc $i; + eval INSERT INTO t2 VALUES ($i, 'delta'); +} +INSERT INTO t2 VALUES (471, 'gamma'); +--enable_query_log +SET debug_dbug=''; + +--echo # 71 rows really match, but the estimate is pulled up towards the size +--echo # of the doc id span, and then clamped to the number of rows. +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t2 WHERE MATCH(a) AGAINST('gamma'); +SET debug_dbug=''; + +DROP TABLE t2; diff --git a/mysql-test/suite/innodb_fts/t/estimate_debug.test b/mysql-test/suite/innodb_fts/t/estimate_debug.test new file mode 100644 index 0000000000000..d3db1d0413900 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/estimate_debug.test @@ -0,0 +1,63 @@ +# +# handler::fulltext_estimate() while a SYNC is half way through writing the +# cache out to the auxiliary table. +# +# A node that SYNC has already written is flagged fts_node_t::synced but stays +# in the cache until the SYNC ends. The estimate reads the auxiliary B-tree +# without a read view, so it sees the record that was just written; it must +# skip the node as well, or it counts those documents twice. +# +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_sequence.inc + +CREATE TABLE t1 ( + id INT AUTO_INCREMENT PRIMARY KEY, + a VARCHAR(64), + FULLTEXT(a) +) ENGINE=InnoDB; + +# alpha in 40 rows, beta in the other 60. Neither is in a majority of the +# table, so a doubled count would still be below the row count and would not +# be hidden by the clamp in ha_innobase::fulltext_estimate(). +INSERT INTO t1(a) SELECT 'alpha' FROM seq_1_to_40; +INSERT INTO t1(a) SELECT 'beta' FROM seq_1_to_60; +ANALYZE TABLE t1; + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); + +connection con1; +# The words are visited in rb tree order, so the first node SYNC writes is +# alpha's, and that is where it stops. +SET debug_dbug='+d,fts_instrument_sync_debug'; +SET DEBUG_SYNC='fts_write_node SIGNAL written WAIT_FOR go'; +send INSERT INTO t1(a) VALUES('zeta'); + +connection con2; +SET DEBUG_SYNC='now WAIT_FOR written'; +--echo # +--echo # alpha is now in the auxiliary table and still in the cache. 40, not +--echo # 80. +--echo # +SET debug_dbug='+d,fulltext_estimate'; +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('alpha'); +--echo # +--echo # beta has not been written yet, so it is counted from the cache alone. +--echo # +SELECT COUNT(*) FROM t1 WHERE MATCH(a) AGAINST('beta'); +SET debug_dbug=''; + +connection default; +SET DEBUG_SYNC='now SIGNAL go'; + +connection con1; +--reap +SET DEBUG_SYNC='RESET'; +SET debug_dbug=''; + +connection default; +disconnect con1; +disconnect con2; +DROP TABLE t1; diff --git a/mysql-test/suite/perfschema/r/digest_view.result b/mysql-test/suite/perfschema/r/digest_view.result index 346b5e0c44e60..b7954b50a12ba 100644 --- a/mysql-test/suite/perfschema/r/digest_view.result +++ b/mysql-test/suite/perfschema/r/digest_view.result @@ -191,17 +191,17 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 1 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 1 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 1 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 1 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 1 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 1 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 1 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 1 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP TABLE test.v1; CREATE VIEW test.v1 AS SELECT * FROM test.t1; EXPLAIN SELECT * from test.v1; @@ -248,19 +248,19 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test ef8b10f452e117fa96af49687a93a10f CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 -test 6b99f4d2ad410f9fa4ee1d501b0db571 DROP TABLE `test` . `v1` 1 -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 2 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 2 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test 187a846fafe04b746eddaaab80b6a766 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 2 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 2 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test bd4716657502ce7922d2ed7e20043b76 CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 +test 2d14a4845ee825e3a1a21bd5799cba3c DROP TABLE `test` . `v1` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 2 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 2 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test aa329bcb29afc39d9793e4fd6e108722 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 2 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 2 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP VIEW test.v1; DROP TABLE test.t1; diff --git a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result index e4e2c8b294a3f..e78e21e19a9ca 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result +++ b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result @@ -8,5 +8,5 @@ SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 #################################### SELECT event_name, digest, digest_text, sql_text FROM events_statements_history_long; event_name digest digest_text sql_text -statement/sql/select c9e142fe40c43498607ca5310e11d2ce SELECT ? + ? + SELECT ... -statement/sql/truncate 506e3496d92689cd2367329dc7725165 TRUNCATE TABLE truncat... +statement/sql/select 3e19ece73d286977122b52fb027c180c SELECT ? + ? + SELECT ... +statement/sql/truncate 7ab36a94986dd98b93370b025166a946 TRUNCATE TABLE truncat... diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 429a85f9cac30..48eef3ca4bfac 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -122,7 +122,6 @@ SET (SQL_SOURCE opt_rewrite_remove_casefold.cc opt_sargable_left.cc opt_sum.cc - opt_vcol_substitution.cc ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc session_tracker.cc @@ -199,6 +198,9 @@ SET (SQL_SOURCE json_table.cc proxy_protocol.cc backup.cc xa.cc socketpair.c socketpair.h + opt_multi_valued_index.h + opt_multi_valued_index.cc + opt_mvi_jsonfuncs.cc opt_vcol_substitution.h opt_vcol_substitution.cc opt_hints_parser.cc opt_hints_parser.h scan_char.h diff --git a/sql/handler.h b/sql/handler.h index fa9196f189a8f..276bf091e1e63 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -4557,6 +4557,38 @@ class handler :public Sql_alloc const key_range *max_key, page_range *res) { return (ha_rows) 10; } + + /** + Estimate how many records a fulltext search for a single word will match. + + The fulltext analogue of records_in_range(): it probes fulltext index + @a index_nr cheaply, without performing the search. + + @param index_nr number of a fulltext index of this table + @param word a single literal word, in the character set of the + fulltext index. NOT a query: no boolean mode operators, + no wildcards, no phrases, no query expansion. + @param word_len length of @a word in bytes + + The result is an estimate. An engine may ignore index entries of rows + that were deleted but not yet purged, may ignore not yet flushed in-memory + index buffers, and may ignore stopword and token length rules. It is + therefore neither an upper nor a lower bound. + + @retval HA_POS_ERROR no estimate available: not a fulltext index, the + engine cannot estimate, an I/O or consistency + problem. The caller must fall back to its own + guess. This is called during optimization, so no + error is raised and no warning is pushed. + @return estimated number of matching records, >= 1. Never + 0: like records_in_range(), callers may treat 0 as + "provably empty", and this estimate may not make + that claim. + */ + virtual ha_rows fulltext_estimate(uint index_nr, const char *word, + uint word_len) + { return HA_POS_ERROR; } + /* If HA_PRIMARY_KEY_REQUIRED_FOR_POSITION is set, then it sets ref (reference to the row, aka position, with the primary key given in diff --git a/sql/item.h b/sql/item.h index dabe4ca1c8801..52114d6a8b140 100644 --- a/sql/item.h +++ b/sql/item.h @@ -822,6 +822,9 @@ const item_walk_flags WALK_NO_CACHE_PROCESS= (1<<1); const item_walk_flags WALK_NO_REF= (1<<2); +struct Mv_index; +struct Mvi_access; + class Item :public Value_source, public Type_all_attributes { @@ -2307,6 +2310,7 @@ class Item :public Value_source, invoked with this processor */ virtual bool get_context_for_vcol_processor(void *arg) { return 0; } + virtual bool mvi_analyze(void *arg) { return 0; } virtual bool enumerate_field_refs_processor(void *arg) { return 0; } virtual bool mark_as_eliminated_processor(void *arg) { return 0; } virtual bool eliminate_subselect_processor(void *arg) { return 0; } diff --git a/sql/item_func.cc b/sql/item_func.cc index 2b0bbbba70991..aa0df4f9df3d9 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -6343,6 +6343,19 @@ bool Item_func_match::init_search(THD *thd, bool no_order) if (join_key && !no_order) match_flags|=FT_SORTED; + DBUG_EXECUTE_IF("fulltext_estimate", + if (key != NO_SUCH_KEY) + { + char buff[22]; + ha_rows rows= table->file->fulltext_estimate(key, ft_tmp->ptr(), + (uint) ft_tmp->length()); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_UNKNOWN_ERROR, "fulltext_estimate('%.*s')= %s", + (int) ft_tmp->length(), ft_tmp->ptr(), + rows == HA_POS_ERROR ? "unknown" + : llstr((longlong) rows, buff)); + }); + if (key != NO_SUCH_KEY) THD_STAGE_INFO(table->in_use, stage_fulltext_initialization); diff --git a/sql/item_func.h b/sql/item_func.h index cc0985c8d53b7..59d02828ff8c5 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -109,7 +109,8 @@ class Item_func :public Item_func_or_sum JSON_EXTRACT_FUNC, JSON_VALID_FUNC, ROWNUM_FUNC, CASE_SEARCHED_FUNC, // Used by ColumnStore/Spider CASE_SIMPLE_FUNC, // Used by ColumnStore/spider, - DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC + DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC, + MVI_ENCODE_FUNC }; /* diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 3ec85d8c1a980..e7dc491f8e1b1 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -376,6 +376,8 @@ class Item_func_json_contains: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; + bool mvi_analyze(void *arg) override; + Mvi_access *get_mvi_access(THD *thd, List *indexes); protected: Item *shallow_copy(THD *thd) const override @@ -946,6 +948,8 @@ class Item_func_json_overlaps: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; + bool mvi_analyze(void *arg) override; + Mvi_access *get_mvi_access(THD *thd, List *indexes); Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } }; diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index a35749fc4dc1b..231fc51865df9 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2644,6 +2644,62 @@ class Item_temptable_rowid :public Item_str_func }; +/* + A function to support ARRAY indexes. When the user specifies an ARRAY index: + + CREATE INDEX idx1 ON + t1 ((CAST(JSON_EXTRACT(json_col, '$.arr') AS $datatype ARRAY))); + + We create a virtual column and a fulltext index over it: + + mvi_col_1 BLOB AS (MVI_ENCODE(JSON_EXTRACT(json_col, '$.arr'), $datatype)), + FULLTEXT INDEX idx (mvi_col_1) + + So, MVI_ENCODE has this signature: + + MVI_ENCODE(json_array, datatype) + + and it returns the JSON array elements represented in a form suitable for + putting into the fulltext index (without any custom fulltext parser atm) + + @seealso "multi_valued_key_part:" rule in sql_yacc.yy + + (TODO: move this item to opt_multi_valued_index, too) +*/ + +class Item_func_mvi_encode : public Item_str_ascii_func +{ + Lex_cast_type_st m_cast_type; + String tmp_js; + json_engine_t je; + /* Print the type the values are cast to, as CAST() spells it */ + void append_cast_type(String *str) const; +public: + void print(String *str, enum_query_type query_type) override; + /* + Print as the CAST(... AS ... ARRAY) the index was declared with, for + SHOW CREATE TABLE. print() cannot do this: what it produces goes into + the FRM, and that is parsed back as a call of this function. + */ + void print_as_array_cast(String *str); + Item_func_mvi_encode(THD* thd, Item *expr, const Lex_cast_type_st &cast_type): + Item_str_ascii_func(thd, expr), m_cast_type(cast_type) {} + String *val_str_ascii(String *buf) override; + enum Functype functype() const override { return MVI_ENCODE_FUNC; } + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("mvi_encode")}; + return name; + } + bool fix_length_and_dec(THD *thd) override; + Item *shallow_copy(THD *thd) const override + { + return get_item_copy(thd, this); + } + Lex_cast_type_st &cast_type() { return m_cast_type; } +}; + + class Item_func_format_pico_time : public Item_str_ascii_func { /* Format is 'AAAA.BB UUU' = 11 characters or 'AAA ps' = 6 characters. */ diff --git a/sql/lex.h b/sql/lex.h index 61774df019f0d..d14a4dd7c636b 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -419,6 +419,7 @@ SYMBOL symbols[] = { { "MONITOR", SYM(MONITOR_SYM)}, { "MONTH", SYM(MONTH_SYM)}, { "MUTEX", SYM(MUTEX_SYM)}, + { "MVI_ENCODE", SYM(MVI_ENCODE_SYM)}, { "MYSQL", SYM(MYSQL_SYM)}, { "MYSQL_ERRNO", SYM(MYSQL_ERRNO_SYM)}, { "NAME", SYM(NAME_SYM)}, diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc new file mode 100644 index 0000000000000..49ce77ecf5cae --- /dev/null +++ b/sql/opt_multi_valued_index.cc @@ -0,0 +1,1224 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "mariadb.h" +#include "sql_select.h" +#include "sql_table.h" /* make_internal_field_name */ +#include "item_func.h" +#include "my_json_writer.h" + +static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, + Mvi_access *access); + +/* + Append to *str string representation of m_cast_type. +*/ +void Item_func_mvi_encode::append_cast_type(String *str) const +{ + char buf[32]; + size_t length; + const Name name= m_cast_type.type_handler()->name(); + switch (m_cast_type.type_handler()->field_type()) + { + case MYSQL_TYPE_LONG_BLOB: + str->append(STRING_WITH_LEN("char")); + str->append('('); + length= (size_t) (longlong10_to_str(m_cast_type.length(), buf, -10) - buf); + str->append(buf, length); + str->append(')'); + break; + case MYSQL_TYPE_LONGLONG: + if (m_cast_type.type_handler()->is_unsigned()) + str->append(STRING_WITH_LEN("unsigned")); + else + str->append(STRING_WITH_LEN("int")); + break; + default: + str->append(name.ptr(), name.length()); + break; + } + /* TODO: this is copied from another print() implementation */ + if (decimals && decimals != NOT_FIXED_DEC) + { + str->append('('); + length= (size_t) (longlong10_to_str(decimals, buf, -10) - buf); + str->append(buf, length); + str->append(')'); + } +} + + +void Item_func_mvi_encode::print(String *str, enum_query_type query_type) +{ + str->append(func_name_cstring()); + str->append('('); + args[0]->print(str, query_type); + str->append(','); + append_cast_type(str); + str->append(')'); +} + + +/* + @brief + Print the index expression the way it was written: + + CAST( AS ARRAY) + + @detail + print() cannot do this. Its output is what pack_expression() writes into + the FRM, and that is read back as a call of mvi_encode(), which is the + only form the parser accepts outside an index definition. +*/ + +void Item_func_mvi_encode::print_as_array_cast(String *str) +{ + str->append(STRING_WITH_LEN("cast(")); + /* The same flags the other parts of a table definition are printed with */ + args[0]->print_for_table_def(str); + str->append(STRING_WITH_LEN(" as ")); + append_cast_type(str); + str->append(STRING_WITH_LEN(" array)")); +} + + +/* TODO: this duplicates logic in Item_func_json_extract::val_int */ +static longlong json_value_to_longlong(enum json_value_types type, + CHARSET_INFO *cs, + char* value, int value_len) +{ + switch (type) + { + case JSON_VALUE_NUMBER: + case JSON_VALUE_STRING: + { + char *end; + int err; + return cs->strntoll(value, value_len, 10, &end, &err); + } + case JSON_VALUE_TRUE: + return 1; + default: + return 0; + }; +} + +/* Copied from Type_handler::store_sort_key_longlong */ +static void store_sort_key_longlong(uchar *to, bool unsigned_flag, + longlong value) +{ + to[7]= (uchar) value; + to[6]= (uchar) (value >> 8); + to[5]= (uchar) (value >> 16); + to[4]= (uchar) (value >> 24); + to[3]= (uchar) (value >> 32); + to[2]= (uchar) (value >> 40); + to[1]= (uchar) (value >> 48); + to[0]= (uchar) (value >> 56) ^ (unsigned_flag ? 0 : 128); +} + + +/* + @brief + Encode the current JSON value in *je to either store or look it up in + Multi-Value Index. The index uses cast_th datatype. + + @detail + The encoded value shouldn't have space, punctuation or other similar + characters, as we're using the default Fulltext parser and want the + encoded value treated as one "term". + + If the value cannot be encoded this means it is not stored, also + searches won't find any matches for it. + + A key image longer than a fulltext token can be is cut short instead: + the engine drops a token that long, on the DML path and on the index + build path alike (fts_check_token()), and a value with no key in the + index is a value the index cannot be used for at all. Two values that + agree on the first MVI_KEY_IMAGE_MAX_LEN bytes of their image then + share a key, which costs false positives and nothing else -- the + predicate is rechecked on every row the index produces. The strnxfrm() + branch below has always worked that way; it asks for exactly that many + bytes of weights and cannot get more back. + + @return + false Encoded successfully, the key is appended to *buf + true The JSON value cannot be represented in the index datatype. + Nothing is appended. +*/ + +bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf) +{ + enum_field_types cast_ftype= cast_th->field_type(); + bool is_unsigned= cast_th->is_unsigned(); + StringBuffer sorted; + /* Skip encoding on type incompatibility */ + if (mvi_json_class(cast_ftype) != je->value_type) + return true; + /* 1. sort_string */ + sorted.length(0); + /* TODO: handle temporal types and decimal */ + switch(cast_ftype) + { + case MYSQL_TYPE_LONGLONG: + { + longlong val= json_value_to_longlong(je->value_type, cs, + (char *) je->value, + je->value_len); + sorted.length(8); + store_sort_key_longlong((uchar *) sorted.c_ptr(), + is_unsigned, val); + break; + } + /* TODO: unquote? */ + /* CHAR(n) => LONG BLOB */ + case MYSQL_TYPE_LONG_BLOB: + { + /* Trim trailing whitespaces if possible */ + if (!(cs->state & MY_CS_NOPAD)) + { + je->value_len= (int) cs->lengthsp((const char *) je->value, + je->value_len); + } + if (my_binary_compare(cs)) + { + sorted.set((char *) je->value, je->value_len, + &my_charset_latin1_bin); + } + else + { + // TODO: Is this ever used outside of "SELECT MVI_ENCODE()" ? + my_strnxfrm_ret_t rc= + cs->strnxfrm((uchar *) sorted.c_ptr(), + /*buffer_size*/ MVI_KEY_IMAGE_MAX_LEN, + /*n_weights*/ MVI_KEY_IMAGE_MAX_LEN, + je->value, je->value_len, 0); + sorted.length(rc.m_result_length); + } + break; + } + default: + return true; + } + + /* 2. cut what the engine would not index down to what it will, see above */ + if (sorted.length() > MVI_KEY_IMAGE_MAX_LEN) + sorted.length(MVI_KEY_IMAGE_MAX_LEN); + + /* 3. hex */ + buf->append_hex(sorted.c_ptr(), sorted.length()); + + /* 4. pad */ + if (sorted.length() == 0) + buf->append(STRING_WITH_LEN("xxxx")); + else if (sorted.length() == 1) + buf->append(STRING_WITH_LEN("xx")); + + return false; +} + + +/* + @brief + Walk a JSON array and report its elements, encoded for a multi-valued + index, to `visitor'. + + @detail + An element that is an array is walked too, so that the keys of a nested + array are the keys of its elements. An element that is an object, and + one that cannot be encoded in the index datatype, has no key of its own; + the visitor decides what that means for it. + + TODO: deduplicate, so that ["34567", "34567"] yield only one key + + @return + See Mvi_walk_result. Only MVI_WALK_OK means the array was walked to its + end. +*/ + +Mvi_walk_result walk_mvi_json_array(json_engine_t *je, CHARSET_INFO *cs, + const uchar *start, const uchar *end, + const Type_handler *cast_th, + Mvi_json_array_visitor *visitor) +{ + int depth= 0; + + if (json_scan_start(je, cs, start, end) || json_read_value(je)) + return MVI_WALK_JSON_ERROR; + + if (je->value_type != JSON_VALUE_ARRAY) + return MVI_WALK_NOT_ARRAY; + + do { + switch (je->state) + { + case JST_ARRAY_START: + /* The array we are walking. A nested one is opened below */ + depth++; + break; + case JST_ARRAY_END: + if (--depth == 0) + return MVI_WALK_OK; /* Trailing junk ignored */ + if (visitor->on_nested_array_end(depth + 1)) + return MVI_WALK_ABORTED; + break; + case JST_VALUE: + { + if (json_read_value(je)) + return MVI_WALK_JSON_ERROR; + if (je->value_type == JSON_VALUE_ARRAY) + { + if (visitor->on_nested_array_start(++depth)) + return MVI_WALK_ABORTED; + break; + } + if (je->value_type == JSON_VALUE_OBJECT) + { + if (json_skip_level(je)) + return MVI_WALK_JSON_ERROR; + if (visitor->on_element_without_key()) + return MVI_WALK_ABORTED; + break; + } + String *key= visitor->key_buffer(); + uint32 key_start= key->length(); + if (encode_mvi_key(je, cast_th, cs, key)) + { + key->length(key_start); + if (visitor->on_element_without_key()) + return MVI_WALK_ABORTED; + break; + } + if (visitor->on_key(key)) + return MVI_WALK_ABORTED; + break; + } + default: + return MVI_WALK_BAD_FORMAT; + } + } while (json_scan_next(je) == 0); + + /* The scan ended before the array was closed */ + return MVI_WALK_JSON_ERROR; +} + + +/* + The visitor MVI_ENCODE walks the array with: the keys, separated by a + space, are the fulltext document of the row. They are encoded straight + into the document, so there is nothing to copy afterwards. + + A separator follows every key, including the last one; finish() takes + that one back off. Doing it that way, rather than putting the separator + in front of every key but the first, keeps the buffer in a state the walk + can truncate an unencodable element out of. +*/ + +class Mvi_key_appender : public Mvi_json_array_visitor +{ + String * const out; +public: + Mvi_key_appender(String *out_arg) : out(out_arg) {} + String *key_buffer() override { return out; } + /* + The key is already in place from the call to encode_mvi_key, only + the separator is left to add + */ + bool on_key(String *key) override { return key->append(' '); } + void finish() + { + if (out->length()) + out->length(out->length() - 1); + } +}; + + +/* + @brief + Parse the JSON array argument and return a string that will be fed to the + fulltext index. +*/ + +String *Item_func_mvi_encode::val_str_ascii(String *buf) +{ + String *value= args[0]->val_json(&tmp_js); + Mvi_key_appender appender(buf); + DBUG_ASSERT(fixed()); + if ((null_value= !value)) + return nullptr; + buf->length(0); + buf->set_charset(&my_charset_latin1_bin); + + switch (walk_mvi_json_array(&je, value->charset(), + reinterpret_cast(value->ptr()), + reinterpret_cast(value->end()), + m_cast_type.type_handler(), &appender)) + { + case MVI_WALK_OK: + break; + case MVI_WALK_NOT_ARRAY: + case MVI_WALK_BAD_FORMAT: + goto error_format; + case MVI_WALK_JSON_ERROR: + goto json_error; + case MVI_WALK_ABORTED: + /* Out of memory, and already reported. Better no row than a wrong one */ + null_value= true; + return nullptr; + } + appender.finish(); + + /* + TODO: do something different when an empty string is + returned, i.e. the document has no key at all, to avoid wasting + index space? + */ + return buf; + +error_format: + { + int position= (int) ((const char *) je.s.c_str - value->ptr()); + /* TODO: fix error */ + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_VECTOR_FORMAT_INVALID, ER(ER_VECTOR_FORMAT_INVALID), + position, value->c_ptr_safe()); + null_value= true; + return nullptr; + } + +json_error: + report_json_error_ex(value->ptr(), &je, func_name(), + 0, Sql_condition::WARN_LEVEL_WARN); + null_value= true; + return nullptr; +} + +bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) +{ + /* TODO: validate args[0] is a json array */ + mem_root_dynamic_array_init(thd->mem_root, PSI_INSTRUMENT_MEM, + &je.stack, sizeof(int), NULL, + JSON_DEPTH_DEFAULT, JSON_DEPTH_INC, MYF(0)); + decimals= 0; + fix_length_and_charset(args[0]->max_char_length() * 2, + &my_charset_latin1_bin); + set_maybe_null(); + return false; +} + + +/* + @brief + If `field' is the internal column that holds the keys of a multi-valued + index, return the mvi_encode() call that computes them. + + @detail + Only the multi-valued index DDL creates a hidden column computed by + MVI_ENCODE(), so this identifies one for certain. +*/ + +static Item_func_mvi_encode *mvi_expr(field_visibility_t invisible, + const Virtual_column_info *vcol_info) +{ + Item *expr; + if (invisible != INVISIBLE_FULL || !vcol_info || + !(expr= vcol_info->expr) || + expr->type() != Item::FUNC_ITEM || + ((Item_func *) expr)->functype() != Item_func::MVI_ENCODE_FUNC) + return NULL; + return (Item_func_mvi_encode *) expr; +} + + +bool is_mvi_vcol(const Field *field) +{ + return mvi_expr(field->invisible, field->vcol_info) != NULL; +} + + +/* The same, on the way in: for a column that is being created */ +bool is_mvi_vcol(const Create_field *field) +{ + return mvi_expr(field->invisible, field->vcol_info) != NULL; +} + + +/* + @brief + Is key #keyno of `table' a multi-valued index, that is, a fulltext key + over one internal MVI column? + + @detail + init_key_part_spec() does not allow such a key to have more than one key + part. The check is here as well because a table created before it was + added may still have one, and there is no single expression to show for + it. The optimizer does use each of its parts, see + collect_mvi_indexes_for_table(). +*/ + +static Item_func_mvi_encode *mvi_key_expr(const TABLE *table, uint keyno) +{ + KEY *key= table->s->key_info + keyno; + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy) || key->user_defined_key_parts != 1) + return NULL; + /* + Take the field from the TABLE and not from the key part: the share's + Field objects have no expression, parse_vcol_defs() builds one for each + TABLE of the share. + */ + Field *field= table->field[key->key_part[0].fieldnr - 1]; + return mvi_expr(field->invisible, field->vcol_info); +} + + +bool is_mvi_key(const TABLE *table, uint keyno) +{ + return mvi_key_expr(table, keyno) != NULL; +} + + +void print_mvi_key_expr(String *str, const TABLE *table, uint keyno) +{ + Item_func_mvi_encode *mvi= mvi_key_expr(table, keyno); + DBUG_ASSERT(mvi); + mvi->print_as_array_cast(str); +} + + +/* + @brief + Can an index over an ARRAY be of the type `key' was declared with? + + @detail + Only a plain KEY can. What the server builds is a fulltext index over + the encoded elements of the array, which does not implement what any of + the other types would promise: UNIQUE and PRIMARY KEY would not be + enforced, and MATCH() against a FULLTEXT one would find nothing. They + used to be accepted and quietly turned into a plain index. + + @return + true No, and an error is raised +*/ + +static bool check_mvi_key_type(const Key *key) +{ + const char *type= NULL; + switch (key->type) { + case Key::PRIMARY: type= "PRIMARY KEY"; break; + case Key::UNIQUE: type= "UNIQUE"; break; + case Key::FULLTEXT: type= "FULLTEXT"; break; + case Key::SPATIAL: type= "SPATIAL"; break; + case Key::VECTOR: type= "VECTOR"; break; + case Key::MULTIPLE: /* A plain KEY: the only type an ARRAY can have */ + case Key::FOREIGN_KEY: /* Both of these are built with Key::MULTIPLE, so */ + case Key::IGNORE_KEY: /* they never reach us under their own name */ + break; + } + if (!type) + return false; + my_error(ER_WRONG_USAGE, MYF(0), type, "ARRAY"); + return true; +} + + +/* + @brief + Handle a `(CAST(expr AS type ARRAY))' key part: turn the key being + defined into a multi-valued index over a new internal column. + + @detail + There is no field to index directly, so the DDL builds one: a hidden + stored column computed by MVI_ENCODE(), holding the encoded elements of + the array, and a fulltext index over it. That pairing is what a + multi-valued index is, see is_mvi_key(). + + Both the column and the key are invisible: there is no syntax that would + name the column, and SHOW CREATE TABLE prints the key with the expression + it was declared with instead, see print_mvi_key_expr(). + + @return + The key part naming the new column, or NULL if an error was raised +*/ + +Key_part_spec *add_mvi_key_part(THD *thd, Item *expr, + const Lex_cast_type_st &cast_type) +{ + LEX *lex= thd->lex; + Key *key= lex->last_key; + + /* + An index over an ARRAY has exactly one key part. Catch a second one here, + before the type of the key is overwritten below and check_mvi_key_type() + starts seeing FULLTEXT instead of what the user wrote. A part that comes + *after* the ARRAY one is caught in init_key_part_spec(). + */ + if (unlikely(key->columns.elements)) + { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + return NULL; + } + if (unlikely(check_mvi_key_type(key))) + return NULL; + + /* TODO: check fts_min_token_size is 4, warn if not */ + Create_field *f= new (thd->mem_root) Create_field(); + Item *vcol_expr= + new (thd->mem_root) Item_func_mvi_encode(thd, expr, cast_type); + if (unlikely(!f || !vcol_expr)) + return NULL; + + /* Has to run before `f' joins the list it looks for a free name in */ + const Lex_ident_column fname= + make_internal_field_name(thd, "DB_MVI_", &lex->alter_info.create_list); + + Virtual_column_info *v= add_virtual_expression(thd, vcol_expr); + if (unlikely(!v)) + return NULL; + v->set_vcol_type(VCOL_GENERATED_STORED); + + f->invisible= INVISIBLE_FULL; + f->set_handler(&type_handler_blob); + f->charset= &my_charset_latin1_bin; + f->vcol_info= v; + lex->init_last_field(f, &fname); + lex->alter_info.create_list.push_back(f, thd->mem_root); + + key->type= Key::FULLTEXT; + key->invisible= true; + + return new (thd->mem_root) Key_part_spec(&fname, 0, /*gen=*/true); +} + + +/* Collect all the MVI indexes of `table' */ +static +bool collect_mvi_indexes_for_table(THD *thd, TABLE *table, + List *indexes) +{ + for (uint i=0; i < table->s->keys; i++) + { + if (!table->keys_in_use_for_query.is_set(i)) + continue; + + KEY *key= &table->key_info[i]; + /* TODO: "legacy" */ + if (!(key->flags & HA_FULLTEXT_legacy)) + continue; + for (uint kp=0; kp < key->user_defined_key_parts; kp++) + { + Field *field= key->key_part[kp].field; + if (!is_mvi_vcol(field)) + continue; + Mv_index *index= new (thd->mem_root) Mv_index(field, i); + if (indexes->push_back(index)) + return TRUE; // Out of memory + } + } + return FALSE; // Ok +} + + +/* + Add one encoded element key to the access. + + TODO: String object live on MEM_ROOT and their destructor is never called + (fix that or switch to something like LEX_STRINGs) +*/ + +bool Mvi_access::add_key(MEM_ROOT *mem_root, const String *key) +{ + List_iterator it(encoded); + String *have; + /* + A key we already search for adds nothing: '+ka +ka' matches what '+ka' + matches, and so does 'ka ka'. The lists are a handful of elements, so + the scan is cheaper than the extra fulltext term would be. + */ + while ((have= it++)) + { + if (have->length() == key->length() && + !memcmp(have->ptr(), key->ptr(), key->length())) + return false; + } + + String *s= new (mem_root) String; + const char *copy= (const char *) memdup_root(mem_root, key->ptr(), + key->length()); + if (!s || !copy) + return true; + s->set(copy, key->length(), &my_charset_latin1_bin); + return encoded.push_back(s, mem_root); +} + + +/* + @brief + Fold another access on the same index into this one. + + @detail + collect_mvi_accesses() only takes the top-level AND-parts of the + condition, so every access it produces has to be true for every row of + the result. Two conjunctive accesses on one index therefore require the + union of their keys, and one search for '+ka +kb' finds what two separate + searches would - more selectively than either, at the price of one. + + Only conjunctive accesses merge. Two disjunctive ones would need + '(ka kb) (kc kd)' to mean (a OR b) AND (c OR d), which the boolean-mode + query build_ft_query() puts together has no syntax for. They stay + separate candidates and get_best_mvi_access() picks between them. +*/ + +bool Mvi_access::merge(MEM_ROOT *mem_root, Mvi_access *other) +{ + List_iterator it(other->encoded); + String *key; + DBUG_ASSERT(can_merge(other)); + /* Merging changes the key set, so it has to happen before we cost it */ + DBUG_ASSERT(records == HA_POS_ERROR); + while ((key= it++)) + { + if (add_key(mem_root, key)) + return true; + } + return false; +} + + +/* + @brief + Estimate how many records this access will read. + + @detail + The engine gives us an estimate for one element key at a time (the + fulltext analogue of records_in_range()). We combine the estimates the + way the query combines the keys: + + - Disjunctive access (JSON_OVERLAPS) reads the rows of every key, so the + estimates add up. A key the engine cannot estimate leaves us with no + idea of what the scan costs, and we cannot leave that key out: dropping + it from an OR loses the rows that only have that key. Price the access + out of the plan instead. + + - Conjunctive access (JSON_CONTAINS) reads the rows that have all of the + keys. The engine estimates one key at a time and cannot intersect them + for us, so assume the keys are independent: + + rows ~ N * PROD(r_i / N) + + clamped to the rarest key, which is a hard upper bound. The assumption + under-estimates correlated keys - the elements of a tag array often + are - but the rarest key alone over-estimates by orders of magnitude + as soon as the keys are at all selective, and every term we keep in + the query is a term the engine intersects instead of us fetching the + row and having the WHERE clause discard it. + + Keys the engine cannot estimate take no part in the estimate but stay + in the query: a longer AND only narrows the scan, and the JSON + predicate does the exact filtering either way. If it could not + estimate a single one of them we know nothing at all, so the access is + priced out just like a disjunctive one. + + TODO: read_time only accounts for reading the rows, not for the fulltext + search that produces their rowids. +*/ + +void Mvi_access::estimate_records() +{ + TABLE *table= index->vcol->table; + handler *file= table->file; + List_iterator it(encoded); + String *key; + const double n_rows= rows2double(table->stat_records()); + double sum= 0.0, isect= n_rows; + ha_rows min_rows= HA_POS_ERROR; + uint n_estimated= 0; + bool have_unknown_estimate= false; + + while ((key= it++)) + { + ha_rows rows= file->fulltext_estimate(index->keyno, key->ptr(), + (uint) key->length()); + if (rows == HA_POS_ERROR) + { + have_unknown_estimate= true; + continue; + } + n_estimated++; + sum+= rows2double(rows); + set_if_smaller(min_rows, rows); + if (n_rows >= 1.0) + isect*= rows2double(rows) / n_rows; + } + + if (!conjunctive && have_unknown_estimate) + { + /* + Disjunctive means we have to read all keys. For at least one, we have no idea + how many matches it has. Fall back to full scan. + */ + records= table->stat_records(); + read_time= DBL_MAX; + return; + } + if (conjunctive && !n_estimated) + { + /* Nothing was estimated. Fall back to full table scan */ + records= table->stat_records(); + read_time= DBL_MAX; + return; + } + + if (conjunctive) + { + /* The rows that have all of the keys, see above */ + records= n_rows >= 1.0 ? (ha_rows) isect : (ha_rows) 1; + set_if_smaller(records, min_rows); + } + else + records= (ha_rows) sum; + + set_if_smaller(records, table->stat_records()); + set_if_bigger(records, (ha_rows) 1); + read_time= file->cost(file->ha_rnd_pos_call_and_compare_time(records)); +} + + +/* + @brief + Build the boolean-mode fulltext query to find rows of interest. + For conjunctive access it is + + '+encoded_foo +encoded_bar ...' + + For disjunctive access, it is + + 'encoded_foo encoded_bar' +*/ + +bool Mvi_access::build_ft_query(String *out) +{ + List_iterator it(encoded); + String *key; + out->length(0); + out->set_charset(&my_charset_latin1_bin); + while ((key= it++)) + { + if ((out->length() && out->append(' ')) || + (conjunctive && out->append('+')) || + out->append(key->ptr(), key->length())) + return true; + } + return !out->length(); +} + + +/* + @brief + Print the index this access uses and the element keys it will search that + index for into the optimizer trace. + + @detail + The keys are printed in their encoded form. That is what is stored in the + index and what we search for, but it is not readable. + + "match" tells whether a row has to have all of the keys (JSON_CONTAINS) + or just one of them (JSON_OVERLAPS). +*/ + +void Mvi_access::print_json(THD *thd, Json_writer_object *trace_object) +{ + KEY *key_info= index->vcol->table->key_info + index->keyno; + List_iterator it(encoded); + String *key; + trace_object->add("index", key_info->name). + add("match", conjunctive ? "all" : "any"); + if (cost_is_known()) + trace_object->add("rows", records).add("cost", read_time); + else + trace_object->add("usable", false). + add("cause", "the engine cannot estimate one of the keys"); + Json_writer_array trace_ranges(thd, "ranges"); + while ((key= it++)) + trace_ranges.add(key->ptr(), key->length()); +} + + +/* + @brief + Collect the MVI accesses allowed by the top-level AND-parts of `conds'. + + @detail + An MVI access only reads the rows the index scan matches, so we can only + use it for a predicate that has to be true for every row of the result. + That means the top-level conjuncts and nothing else: for + + json_contains(j1->'$.tags', '"a"') OR json_contains(j2->'$.tags', '"a"') + + a scan of either index would drop the rows that only match the other + branch. +*/ + +static bool collect_mvi_accesses(Mvi_context *ctx, Item *conds) +{ + Item *cond; + if (conds->type() != Item::COND_ITEM) + return conds->mvi_analyze(ctx); + if (((Item_cond *) conds)->functype() != Item_func::COND_AND_FUNC) + return false; + List_iterator it(*((Item_cond *) conds)->argument_list()); + while ((cond= it++)) + { + /* + No recursion: a nested Item_cond is either an already-flattened AND or + an OR, and Item::mvi_analyze() ignores both. + */ + if (cond->mvi_analyze(ctx)) + return true; + } + return false; +} + + +/* + @brief + Analyze `cond' and pick the MVI access `tab' will use, if any, and let + the range analysis see it. + + @param cond The condition the rows of this table have to satisfy: the + WHERE clause, or the ON expression when the table is on the + inner side of an outer join. That is what the range analysis + of this table uses, too. + + @detail + The analysis itself is scratch state: what we leave behind is the list of + accesses in tab->mvi_accesses. They and the Mv_index objects they refer + to live on the MEM_ROOT, so they outlive `ctx'. + + Accesses on one index that both require all of their keys are merged + here, see Mvi_access::merge(). What is left is one candidate per index + and kind, and get_best_mvi_access() prices those and picks one. We put + no price on anything here: the estimate probes the engine's fulltext + index, and this runs for every table of the join. + + A fulltext key never gets a bit in const_keys or keys, so we set them + here. The const_keys bit is what makes the range analysis run for this + table, where get_best_mvi_access() turns the access into a quick select; + the keys bit puts the index into EXPLAIN's possible_keys. + + @return + true Out of memory + false Ok, tab->mvi_accesses is set if the table has any MVI access +*/ + +bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond) +{ + Mvi_context ctx(thd); + MEM_ROOT *mem_root= thd->mem_root; + List *kept; + + if (!cond) + return false; + if (collect_mvi_indexes_for_table(thd, tab->table, &ctx.indexes)) + return true; + /* Most tables have no MVI. Leave before we walk the condition */ + if (ctx.indexes.is_empty()) + return false; + if (collect_mvi_accesses(&ctx, cond)) + return true; + if (ctx.accesses.is_empty()) + return false; + + if (!(kept= new (mem_root) List)) + return true; + + List_iterator it(ctx.accesses); + while (Mvi_access *access= it++) + { + Mvi_access *into; + /* + An access can only be on this table: ctx.indexes holds this table's + indexes and get_mvi_index() matches the predicate against those. + */ + DBUG_ASSERT(access->index->vcol->table == tab->table); + + /* Fold it into an access we already keep, if the two are compatible */ + List_iterator kit(*kept); + while ((into= kit++)) + { + if (into->can_merge(access)) + break; + } + if (into) + { + if (into->merge(mem_root, access)) + return true; + continue; + } + + if (kept->push_back(access, mem_root)) + return true; + tab->const_keys.set_bit(access->index->keyno); + tab->keys.set_bit(access->index->keyno); + } + + tab->mvi_accesses= kept; + return false; +} + + +/* + @brief + Create a quick select for the MVI access to `tab', if there is one. + + @detail + The range optimizer cannot produce this access (it skips fulltext keys), + so the caller creates it here and compares its cost with whatever + test_quick_select() came up with. + + This is the only place an MVI access is priced. Where a table has more + than one - accesses on different indexes, which cannot be merged into a + single fulltext search - the cheapest one wins, on the same read_time + scale keep_cheaper_quick() then uses against the range access. +*/ + +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab) +{ + TABLE *table= tab->table; + Mvi_access *access, *best= NULL; + + if (!tab->mvi_accesses) + return NULL; + + List_iterator it(*tab->mvi_accesses); + while ((access= it++)) + { + /* estimate_records() probes the engine, so do it at most once */ + if (access->records == HA_POS_ERROR) + access->estimate_records(); + /* + best_access_path() takes a quick select to be cheaper than a table + scan without checking (the range optimizer only proposes a quick when + it is), so an access we could not put a price on is no use to us. + */ + if (!access->cost_is_known()) + continue; + if (!best || access->read_time < best->read_time) + best= access; + } + + if (unlikely(thd->trace_started())) + { + /* + We are inside the "rows_estimation" array, so we need an object of our + own before we can add anything by name. Without it the writer hits an + assertion in Single_line_formatting_helper::on_add_member(). + */ + Json_writer_object trace_wrapper(thd); + Json_writer_object trace_mvi(thd, "multi_value_index_use"); + trace_mvi.add_table_name(table); + Json_writer_array trace_candidates(thd, "candidates"); + it.rewind(); + while ((access= it++)) + { + Json_writer_object trace_one(thd); + access->print_json(thd, &trace_one); + if (access == best) + trace_one.add("chosen", true); + } + } + + if (!best) + return NULL; + return create_quick_mvi_select(thd, table, best); +} + + +/**************************************************************************** + QUICK_MVI_SELECT - reading a multi-valued index +****************************************************************************/ +struct Mvi_access; + +/* + Quick select that reads a multi-valued index. + + It runs a boolean-mode fulltext search over the index's hidden vcol, looking + for the encoded element keys of the JSON predicate this access was built + from. The scan is a necessary, not a sufficient condition: the JSON + predicate stays in the WHERE clause and does the exact filtering. + + Unlike FT_SELECT, there is no Item_func_match to have created the FT_INFO + for us, so we create it ourselves in reset() and own it. + + The methods are implemented in opt_multi_valued_index.cc. +*/ + +class QUICK_MVI_SELECT: public QUICK_SELECT_I +{ + Mvi_access *access; + FT_INFO *ft_handler; + StringBuffer<256> query; /* the boolean-mode ft query */ +public: + QUICK_MVI_SELECT(THD *thd, TABLE *table, Mvi_access *access_arg); + ~QUICK_MVI_SELECT(); + int init() override { return 0; } + int reset() override; + int get_next() override; + bool reverse_sorted() override { return false; } + /* + Fulltext results come back ordered by relevance, not by key, so there is + no sorted output to offer. QS_TYPE_MVI is not one of the types the + ORDER BY-by-index code paths consider, so they never ask. + */ + void need_sorted_output() override {} + int get_type() override { return QS_TYPE_MVI; } + void add_keys_and_lengths(String *key_names, String *used_lengths) override; + void add_used_key_part_to_set() override {} + Explain_quick_select *get_explain(MEM_ROOT *alloc) override; +#ifndef DBUG_OFF + void dbug_dump(int indent, bool verbose) override; +#endif +}; + +static QUICK_SELECT_I *create_quick_mvi_select(THD *thd, TABLE *table, Mvi_access *access) +{ + return new QUICK_MVI_SELECT(thd, table, access); +} + +QUICK_MVI_SELECT::QUICK_MVI_SELECT(THD *thd, TABLE *table, + Mvi_access *access_arg) + : access(access_arg), ft_handler(NULL) +{ + head= table; + index= access->index->keyno; + record= head->record[0]; + records= access->records; + read_time= access->read_time; +} + + +QUICK_MVI_SELECT::~QUICK_MVI_SELECT() +{ + handler *file= head->file; + if (ft_handler) + { + file->ha_ft_end(); /* ft_end() + file->ft_handler= NULL */ + /* + We created the FT_INFO, so we free it. For an Item_func_match this is + done by Item_func_match::cleanup(). + */ + ft_handler->please->close_search(ft_handler); + ft_handler= NULL; + } + if (file->inited != handler::NONE) + file->ha_index_or_rnd_end(); +} + + +int QUICK_MVI_SELECT::reset() +{ + handler *file= head->file; + int error; + + if (!ft_handler) + { + if (access->build_ft_query(&query)) + return HA_ERR_OUT_OF_MEM; + if (!(ft_handler= file->ft_init_ext(FT_BOOL, index, &query))) + return HA_ERR_WRONG_COMMAND; /* the error is already reported */ + /* + ft_init() and ha_ft_read() both work off handler::ft_handler (and + ha_innobase::ft_init() dereferences it without checking), so it has to + be set before we go any further. + */ + file->ft_handler= ft_handler; + head->fulltext_searched= 1; + } + if (!file->inited && (error= file->ha_index_init(index, 1))) + return error; + /* This rewinds the search, so it is also right for a repeated reset() */ + return file->ft_init(); +} + + +int QUICK_MVI_SELECT::get_next() +{ + return head->file->ha_ft_read(record); +} + + +void QUICK_MVI_SELECT::add_keys_and_lengths(String *key_names, + String *used_lengths) +{ + bool first= TRUE; + + add_key_and_length(key_names, used_lengths, &first); +} + + +Explain_quick_select *QUICK_MVI_SELECT::get_explain(MEM_ROOT *local_alloc) +{ + Explain_quick_select *res; + if ((res= new (local_alloc) Explain_quick_select(QS_TYPE_MVI))) + res->range.set(local_alloc, &head->key_info[index], max_used_key_length); + return res; +} + + +#ifndef DBUG_OFF +void QUICK_MVI_SELECT::dbug_dump(int indent, bool verbose) +{ + fprintf(DBUG_FILE, "%*squick_mvi_select: index %s (%d)\n", + indent, "", head->key_info[index].name.str, index); +} +#endif + +enum json_value_types mvi_json_class(enum_field_types ftype) +{ + switch (ftype) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_DOUBLE: + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_NEWDECIMAL: + return JSON_VALUE_NUMBER; + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_YEAR: + case MYSQL_TYPE_NEWDATE: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_TIMESTAMP2: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIME2: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + return JSON_VALUE_STRING; + default: + return JSON_VALUE_UNINITIALIZED; + } +} diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h new file mode 100644 index 0000000000000..ffd3aac8804ec --- /dev/null +++ b/sql/opt_multi_valued_index.h @@ -0,0 +1,212 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +class Json_writer_object; + +/* An MVI index */ +struct Mv_index : public Sql_alloc +{ + Field *vcol; /* The hidden vcol of the index */ + uint keyno; /* The keyno of the index */ + Mv_index(Field *vcol_arg, uint keyno_arg) + : vcol(vcol_arg), keyno(keyno_arg) {} +}; + +/* Access descriptor for a predicate */ +struct Mvi_access : public Sql_alloc +{ + Mv_index *index; + List encoded; /* encoded element keys */ + bool conjunctive; /* CONTAINS -> AND, OVERLAPS -> OR */ + /* + The estimate for this access, produced by estimate_records(). + HA_POS_ERROR means we haven't estimated it yet. + */ + ha_rows records; + double read_time; + Mvi_access(Mv_index *idx, bool conj) + : index(idx), conjunctive(conj), records(HA_POS_ERROR), read_time(0.0) {} + + /* Build: Add one encoded element key */ + bool add_key(MEM_ROOT *mem_root, const String *key); + + /* + Build: can `other' be folded into this access? Two accesses on the same + index that both require all of their keys are the same thing as one + access requiring the union of the keys. + */ + bool can_merge(const Mvi_access *other) const + { return index == other->index && conjunctive && other->conjunctive; } + + /* Build: fold `other' into this access. can_merge() must hold */ + bool merge(MEM_ROOT *mem_root, Mvi_access *other); + + /* Usage: Estimate how many records this access will read */ + void estimate_records(); + + /* + Usage: false when estimate_records() could not put a price on the access. + Such an access must not be used: we have no idea what it costs. + */ + bool cost_is_known() const { return read_time != DBL_MAX; } + + /* Usage: Build the fulltext query searching for the element keys */ + bool build_ft_query(String *out); + + /* Usage: describe this access in the optimizer trace */ + void print_json(THD *thd, Json_writer_object *trace_object); +}; + + +/* + The state of the MVI analysis of one table. It only lives for the duration + of setup_mvi_access_for_table(): the accesses that analysis settles on are + what outlive it. +*/ +class Mvi_context : public Sql_alloc +{ + public: + THD *thd; + /* The MV indexes of the table */ + List indexes; + /* MVI accesses for all eligible predicates on the table */ + List accesses; + + Mvi_context(THD *thd_arg) : thd(thd_arg) {} +}; + +/* Return the compatible json type */ +enum json_value_types mvi_json_class(enum_field_types ftype); + +/* + A key is one fulltext token, so it cannot be longer than the maximum + token size the engine will index: 84 characters (HA_FT_MAXCHARLEN, which + is also the default and the maximum of innodb_ft_max_token_size). The key + is the hex of the key image, so that image is at most half of it. An + image longer than that is cut down to it, making the key a prefix key: + see encode_mvi_key(). + + TODO: innodb_ft_max_token_size can be set lower than its default, and + innodb_ft_min_token_size higher, and then the engine drops keys we + consider valid. Validate both against the index at DDL time. +*/ +#define MVI_KEY_IMAGE_MAX_LEN 42 +#define MVI_ENCODED_KEY_MAX_LEN (MVI_KEY_IMAGE_MAX_LEN * 2) + +/* + Encode one JSON value into the form it has in the index, appending it to + `buf'. Returns true if the value cannot be encoded for this index and has + to be skipped, in which case nothing is appended. + Shared with opt_mvi_jsonfuncs.cc. +*/ +bool encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf); + +/* + What walk_mvi_json_array() found. Anything but MVI_WALK_OK means it did + not reach the end of the array. +*/ +enum Mvi_walk_result +{ + MVI_WALK_OK, /* The whole array was walked */ + MVI_WALK_ABORTED, /* A visitor callback asked to stop */ + MVI_WALK_NOT_ARRAY, /* The document is not an array. *je holds the value */ + MVI_WALK_BAD_FORMAT, /* The document is not a JSON we can make sense of */ + MVI_WALK_JSON_ERROR /* Malformed JSON. The error is in je->s.error */ +}; + +/* + What walk_mvi_json_array() reports about the array it is walking. Every + callback returns true to stop the walk, which makes the walk return + MVI_WALK_ABORTED. +*/ +class Mvi_json_array_visitor +{ +public: + virtual ~Mvi_json_array_visitor() {} + + /* + The buffer to encode the next element into. encode_mvi_key() appends to + it, so a visitor that wants the keys one at a time hands out a buffer it + has emptied, and one that is building a document out of them hands out + the document itself and gets the key appended in place. Either way the + walk truncates it back to the length it had if the element turns out to + have no key. + */ + virtual String *key_buffer()= 0; + + /* + An element of the array that is in the index. Its key has been appended + to what key_buffer() returned, which is `key'. + */ + virtual bool on_key(String *key)= 0; + + /* + An element that has no key in the index: an object, or a value that + cannot be encoded in the index datatype. + */ + virtual bool on_element_without_key() { return false; } + + /* + An element that is an array was opened / closed. `depth' is the depth of + that array: 2 for an element of the array being walked, more for one + nested deeper. + */ + virtual bool on_nested_array_start(int) { return false; } + virtual bool on_nested_array_end(int) { return false; } +}; + +/* + Walk a JSON array, encoding its elements for a multi-valued index of the + cast_th datatype, and report them to `visitor'. Shared by the two sides of + the index: MVI_ENCODE, which turns them into the tokens of a row, and the + optimizer, which turns them into the keys to search for. +*/ +Mvi_walk_result walk_mvi_json_array(json_engine_t *je, CHARSET_INFO *cs, + const uchar *start, const uchar *end, + const Type_handler *cast_th, + Mvi_json_array_visitor *visitor); + +/* + Is `field' the internal column that holds the keys of a multi-valued index? +*/ +bool is_mvi_vcol(const Field *field); +bool is_mvi_vcol(const Create_field *field); + +/* Is key #keyno of `table' a multi-valued index? */ +bool is_mvi_key(const TABLE *table, uint keyno); + +/* + Print the expression key #keyno was declared with, in the CAST(... ARRAY) + form, for SHOW CREATE TABLE +*/ +void print_mvi_key_expr(String *str, const TABLE *table, uint keyno); + +/* + DDL: handle a `(CAST(expr AS type ARRAY))' key part of the key being + defined. Returns NULL if an error was raised. +*/ +Key_part_spec *add_mvi_key_part(THD *thd, Item *expr, + const Lex_cast_type_st &cast_type); + +/* + Analyze `cond' and pick the MVI access `tab' will use, if any, and let the + range analysis see it +*/ +bool setup_mvi_access_for_table(THD *thd, JOIN_TAB *tab, Item *cond); + +/* Create a quick select for the MVI access to `tab', if there is one */ +QUICK_SELECT_I *get_best_mvi_access(THD *thd, JOIN_TAB *tab); diff --git a/sql/opt_mvi_jsonfuncs.cc b/sql/opt_mvi_jsonfuncs.cc new file mode 100644 index 0000000000000..f50a803ee51c8 --- /dev/null +++ b/sql/opt_mvi_jsonfuncs.cc @@ -0,0 +1,320 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Making the JSON functions use a Multi-Value Index. + + This is the part of the MVI support that knows about the JSON predicates: + which of them can be computed from an index, which argument holds the + indexed expression, and what to search the index for. The index side of it + lives in opt_multi_valued_index.cc. +*/ + +#include "mariadb.h" +#include "sql_select.h" +#include "item_func.h" + +static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, + CHARSET_INFO *cs, const String *json, + bool conjunctive, json_engine_t *je); + + +/* + @brief + Find Multi-Value Index created over array_indexed_expr. + + @detail + Search the table for an index declared as + + INDEX idx ((CAST(array_indexed_expr AS ARRAY)); + + NOTE: we currently we only locate one such index. What if there are + multiple? +*/ + +static Mv_index *get_mvi_index(List *indexes, + Item *array_indexed_expr) +{ + Mv_index *index; + List_iterator it(*indexes); + Item_func_mvi_encode *mvitem; + while ((index= it++)) + { + Field *vcol_field= index->vcol; + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == Item::FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(array_indexed_expr, true)) + { + return index; + } + } + return NULL; +} + + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + Check if this item is a + + JSON_CONTAINS(array_indexed_expr, '[foo, bar, ... ]') + + which is true when ALL of the elements have a match, so the keys are + ANDed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_contains::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + DBUG_ASSERT(fixed()); + + if (arg_count > 2 || !a2_constant) + return NULL; + /* Find the MVI that matches the first argument */ + if (!(index= get_mvi_index(indexes, args[0]))) + return NULL; + + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + + return collect_mvi_keys(thd, index, args[0]->collation.collation, val, + true, &je); +} + +/* + @brief + Check if we can use Multi-Value Index access to read rows for this + predicate, if yes create an access descriptor. + + @detail + We can use MVI index when the predicate has either of the forms: + + JSON_OVERLAPS(array_indexed_expr, '[foo, bar, ... ]') + JSON_OVERLAPS('[foo, bar, ... ]', array_indexed_expr) + + JSON_OVERLAPS is true when ANY of the elements has a match, so the keys + are ORed. + + @return + The access descriptor, or NULL if the predicate cannot use an MVI. +*/ + +Mvi_access *Item_func_json_overlaps::get_mvi_access(THD *thd, + List *indexes) +{ + Mv_index *index; + uint literal_arg; + String *json; + StringBuffer<256> tmp; + DBUG_ASSERT(fixed()); + + if ((index= get_mvi_index(indexes, args[0]))) + literal_arg= 1; + else if ((index= get_mvi_index(indexes, args[1]))) + literal_arg= 0; + else + return NULL; + + if (!args[literal_arg]->const_item()) + return NULL; + if (!(json= args[literal_arg]->val_json(&tmp))) + return NULL; + + /* + TODO: is this really so: + encode_mvi_key() must see the collation of the indexed expression: that + is what decides how MVI_ENCODE built the keys that are in the index. + */ + return collect_mvi_keys(thd, index, + args[1 - literal_arg]->collation.collation, json, + false, &je); +} + + +/* + The visitor collect_mvi_keys() walks the JSON literal with: it turns the + elements into the keys of an access. Stopping the walk means giving up on + the access, see collect_mvi_keys() for when we have to. +*/ + +class Mvi_key_collector : public Mvi_json_array_visitor +{ + THD * const thd; + Mv_index * const index; + const bool conjunctive; + /* + The number of keys we had collected when the depth-2 array element we + are inside of was opened. Only used for an OR / JSON_OVERLAPS. + */ + uint keys_before_level2_array; + /* One key at a time: add_key() copies it onto the mem_root */ + StringBuffer buf; + + uint key_count() const { return access ? access->encoded.elements : 0; } +public: + /* What we are collecting into. NULL until the first key */ + Mvi_access *access; + + Mvi_key_collector(THD *thd_arg, Mv_index *index_arg, bool conjunctive_arg) + : thd(thd_arg), index(index_arg), conjunctive(conjunctive_arg), + keys_before_level2_array(0), access(NULL) + { buf.set_charset(&my_charset_latin1_bin); } + + String *key_buffer() override + { + buf.length(0); + return &buf; + } + + bool on_key(String *key) override + { + if (!access && + !(access= new (thd->mem_root) Mvi_access(index, conjunctive))) + return true; + return access->add_key(thd->mem_root, key); + } + + /* Only an AND of the keys tolerates a missing one */ + bool on_element_without_key() override { return !conjunctive; } + + bool on_nested_array_start(int depth) override + { + if (depth == 2) + keys_before_level2_array= key_count(); + return false; + } + + /* An element that is an array and yielded no key at all */ + bool on_nested_array_end(int depth) override + { + return depth == 2 && !conjunctive && + key_count() == keys_before_level2_array; + } +}; + + +/* + @brief + Collect the element keys to search `index' for from a JSON literal. + + @param cs Collation of the indexed expression + TODO why does that matter? + + @param json The JSON literal: an array, or a single scalar + @param conjunctive true when the keys are ANDed (JSON_CONTAINS), + false when they are ORed (JSON_OVERLAPS) + @param je A json_engine_t to scan with + + @detail + An element that cannot be encoded for this index (a type mismatch, say) + can only be skipped when the keys are ANDed. Dropping a key from an AND + makes the index scan less selective, so it still returns a superset of + the rows the predicate matches, and the predicate itself does the exact + filtering afterwards. + + For an OR we cannot do that. A row can satisfy the predicate through the + very element we failed to encode, and MVI_ENCODE skips such elements too, + so that row has no key in the index for us to find it by. Dropping the + key would lose it. Give up on the access instead. + + An element that is itself an array is flattened, the same way + MVI_ENCODE flattens the document. JSON_OVERLAPS does not flatten: + it only matches such an element against a document element that is + an array too, compared whole (json_compare_arrays_in_order()). The + flattening here is still safe as it will produce only false + positives that will be eliminated by a recheck. The only exception + is when the nested array yields no key at all i.e. [], [[]], + [[],[]], [[[]]], etc. Such an element may match a document element + that has no key of ours either, so nothing we could search for + would find that row. Give up in this case, as for a failed + encoding. + + Anything the walk itself does not like -- malformed JSON, an object + where an array should be -- means no access either. The predicate is + still there to parse the literal and raise whatever it raises. + + @return + The access descriptor, or NULL if the predicate cannot use this MVI. +*/ + +static Mvi_access *collect_mvi_keys(THD *thd, Mv_index *index, + CHARSET_INFO *cs, const String *json, + bool conjunctive, json_engine_t *je) +{ + Item_func_mvi_encode *mvitem= + (Item_func_mvi_encode *) index->vcol->vcol_info->expr; + const Type_handler *cast_th= mvitem->cast_type().type_handler(); + Mvi_key_collector collector(thd, index, conjunctive); + String *buf; + + switch (walk_mvi_json_array(je, cs, + reinterpret_cast(json->ptr()), + reinterpret_cast(json->end()), + cast_th, &collector)) + { + case MVI_WALK_OK: + return collector.access; + case MVI_WALK_NOT_ARRAY: + break; + default: + return NULL; + } + + /* A scalar: JSON_CONTAINS(expr, '123'). It is in *je */ + buf= collector.key_buffer(); + if (je->value_type == JSON_VALUE_UNINITIALIZED || + je->value_type == JSON_VALUE_OBJECT || + encode_mvi_key(je, cast_th, cs, buf) || + collector.on_key(buf)) + return NULL; + return collector.access; +} + + +/* Add `access' to the context, if there is one. Returns true on error */ + +static bool add_mvi_access(Mvi_context *ctx, Mvi_access *access) +{ + return access && ctx->accesses.push_back(access, ctx->thd->mem_root); +} + + +bool Item_func_json_contains::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); +} + + +bool Item_func_json_overlaps::mvi_analyze(void *arg) +{ + Mvi_context *ctx= (Mvi_context *) arg; + return add_mvi_access(ctx, get_mvi_access(ctx->thd, &ctx->indexes)); +} diff --git a/sql/opt_range.h b/sql/opt_range.h index bdac62099607a..b0c38213c9306 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -1216,7 +1216,8 @@ class QUICK_SELECT_I QS_TYPE_FULLTEXT = 4, QS_TYPE_ROR_INTERSECT = 5, QS_TYPE_ROR_UNION = 6, - QS_TYPE_GROUP_MIN_MAX = 7 + QS_TYPE_GROUP_MIN_MAX = 7, + QS_TYPE_MVI = 8 }; /* Get type of this quick select - one of the QS_TYPE_* values */ diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 47de84e48d131..7d488ff341ebf 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -170,7 +170,8 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root) columns(rhs.columns, mem_root), name(rhs.name), option_list(rhs.option_list), - generated(rhs.generated), invisible(false), + generated(rhs.generated), + invisible(rhs.invisible), without_overlaps(rhs.without_overlaps), old(rhs.old), length(rhs.length), period(rhs.period) { diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index 2e894deae5fce..eb8184bd5b9b9 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -2524,9 +2524,7 @@ void Explain_table_access::append_tag_name(String *str, enum explain_extra_tag t void Explain_quick_select::print_extra(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { /* print nothing */ } @@ -2614,9 +2612,7 @@ const char * Explain_quick_select::get_name_by_type() void Explain_quick_select::print_key(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { if (str->length() > 0) str->append(','); @@ -2640,9 +2636,7 @@ void Explain_quick_select::print_key(String *str) void Explain_quick_select::print_key_len(String *str) { - if (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || - quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX) + if (is_basic()) { char buf[64]; size_t length; diff --git a/sql/sql_explain.h b/sql/sql_explain.h index 9b07721973b45..e1405aac4a8bc 100644 --- a/sql/sql_explain.h +++ b/sql/sql_explain.h @@ -707,9 +707,10 @@ class Explain_quick_select : public Sql_alloc bool is_basic() { - return (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || + return (quick_type == QUICK_SELECT_I::QS_TYPE_RANGE || quick_type == QUICK_SELECT_I::QS_TYPE_RANGE_DESC || - quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX); + quick_type == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX || + quick_type == QUICK_SELECT_I::QS_TYPE_MVI); } /* This is used when quick_type == QUICK_SELECT_I::QS_TYPE_RANGE */ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index dc9e7b6113333..bdc1f67862f42 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -141,7 +141,7 @@ static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables); static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables); static bool get_quick_record_count(THD *thd, SQL_SELECT *select, - TABLE *table, + JOIN_TAB *tab, const key_map *keys,ha_rows limit, ha_rows *quick_count); static void optimize_straight_join(JOIN *join, table_map join_tables); @@ -5463,13 +5463,51 @@ mysql_select(THD *thd, TABLE_LIST *tables, List &fields, COND *conds, } +/* + @brief + Keep the cheaper of *quick_ref and mvi_quick in *quick_ref, delete the + other one. + + @detail + The range optimizer skips fulltext keys, so it can never produce an MVI + access itself. Instead the caller creates one, hands it to us and we keep + it if test_quick_select() did not come up with anything better. + + mvi_quick may be NULL, which means "there is no MVI access". + + TODO: when the MVI access gets a real cost estimate, also compare it with + the cost of a table scan. Right now, if test_quick_select() produced no + quick select at all (because a table scan was cheaper than any range), we + take the MVI access without asking how much it costs. +*/ + +static void keep_cheaper_quick(TABLE *table, QUICK_SELECT_I **quick_ref, + QUICK_SELECT_I *mvi_quick) +{ + if (!mvi_quick) + return; + if (*quick_ref && (*quick_ref)->read_time <= mvi_quick->read_time) + { + delete mvi_quick; + return; + } + delete *quick_ref; + *quick_ref= mvi_quick; + /* + Callers assume (*quick_ref)->records >= opt_range_condition_rows. This is + a min-setter, so it can only lower the value. + */ + table->set_opt_range_condition_rows(mvi_quick->records); +} + + /** Approximate how many records are going to be returned by this table in this select with this key. @param thd Thread handle @param select Select to be examined - @param table The table of interest + @param tab The table of interest @param keys The keys of interest @param limit Maximum number of rows of interest @param quick_count Pointer to where we want the estimate written @@ -5480,11 +5518,12 @@ mysql_select(THD *thd, TABLE_LIST *tables, List &fields, COND *conds, */ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, - TABLE *table, + JOIN_TAB *tab, const key_map *keys,ha_rows limit, ha_rows *quick_count) { quick_select_return error; + TABLE *table= tab->table; DBUG_ENTER("get_quick_record_count"); uchar buff[STACK_BUFF_ALLOC]; if (unlikely(check_stack_overrun(thd, STACK_MIN_SIZE, buff))) @@ -5496,6 +5535,12 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, { select->head=table; table->reginfo.impossible_range=0; + /* + An MVI access is not something test_quick_select() can find. Create it + here and keep it across the call: test_quick_select() deletes + select->quick on entry. + */ + QUICK_SELECT_I *mvi_quick= get_best_mvi_access(thd, tab); /* EQ_FUNC and EQUAL_FUNC already sent unusable key notes (if any) during update_ref_and_keys(). Have only other functions raise notes @@ -5509,6 +5554,7 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, if (error == SQL_SELECT::OK) { + keep_cheaper_quick(table, &select->quick, mvi_quick); if (select->quick) { /* @@ -5523,6 +5569,8 @@ static bool get_quick_record_count(THD *thd, SQL_SELECT *select, } DBUG_RETURN(false); } + /* Impossible range or an error: the MVI access is of no use */ + delete mvi_quick; if (error == SQL_SELECT::IMPOSSIBLE_RANGE) { table->reginfo.impossible_range=1; @@ -6203,6 +6251,14 @@ make_join_statistics(JOIN *join, List &tables_list, */ add_group_and_distinct_keys(join, s); + /* + Same for the multi-valued index this table can be read through: a + fulltext key never gets a bit of its own. Use the same condition the + range analysis below will use. + */ + if (setup_mvi_access_for_table(thd, s, *get_sargable_cond(join, s->table))) + goto error; + /* This will be updated in calculate_cond_selectivity_for_table() */ s->table->set_cond_selectivity(1.0); DBUG_ASSERT(s->table->used_stat_records == 0 || @@ -6241,7 +6297,7 @@ make_join_statistics(JOIN *join, List &tables_list, (SORT_INFO*) 0, 1, &error); if (!select) goto error; - if (get_quick_record_count(join->thd, select, s->table, + if (get_quick_record_count(join->thd, select, s, &s->const_keys, join->row_limit, &records)) { /* There was an error in test_quick_select */ @@ -9875,7 +9931,13 @@ best_access_path(JOIN *join, } else { - type= JT_INDEX_MERGE; + if (s->quick->get_type() == QUICK_SELECT_I::QS_TYPE_MVI) + type= JT_RANGE; + else + { + type= JT_INDEX_MERGE; + force_plan= s->quick->force_index_merge; + } /* We don't know exactly from where the costs comes from. Let's store it in copy_cost. @@ -9884,7 +9946,6 @@ best_access_path(JOIN *join, */ cost.reset(); cost.copy_cost= s->quick->read_time; - force_plan= s->quick->force_index_merge; } loose_scan_opt.check_range_access(join, idx, s->quick); } @@ -14861,6 +14922,19 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) sel->cond->quick_fix_field(); quick_select_return res; + /* + Take an MVI quick select out of sel->quick before the call: + test_quick_select() deletes it on entry and cannot produce + another one, so it would be lost for good. + */ + QUICK_SELECT_I *mvi_quick= NULL; + if (sel->quick && + sel->quick->get_type() == QUICK_SELECT_I::QS_TYPE_MVI) + { + mvi_quick= sel->quick; + sel->quick= 0; + } + if ((res= sel->test_quick_select(thd, tab->keys, ((used_tables & ~ current_map) | OUTER_REF_TABLE_BIT), @@ -14888,13 +14962,21 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) 0, FALSE, FALSE, FALSE, Item_func::BITMAP_NONE)) == SQL_SELECT::IMPOSSIBLE_RANGE) + { + delete mvi_quick; DBUG_RETURN(1); // Impossible WHERE + } } else sel->cond=orig_cond; if (res == SQL_SELECT::ERROR) + { + delete mvi_quick; DBUG_RETURN(1); /* Some error in one of test_quick_select calls */ + } + + keep_cheaper_quick(sel->head, &sel->quick, mvi_quick); /* Fix for EXPLAIN */ if (sel->quick) diff --git a/sql/sql_select.h b/sql/sql_select.h index 23a927bfcb14e..0fe0bd8b7b97e 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -35,6 +35,7 @@ #include "cset_narrowing.h" typedef struct st_join_table JOIN_TAB; +struct Mvi_access; /* Values in optimize */ #define KEY_OPTIMIZE_EXISTS 1U #define KEY_OPTIMIZE_REF_OR_NULL 2U @@ -566,6 +567,12 @@ typedef struct st_join_table { key_map checked_keys; /**< Keys checked in find_best */ key_map needed_reg; key_map keys; /**< all keys with can be used */ + /* + The multi-valued index accesses this table's predicates allow, or NULL if + there are none. Set by setup_mvi_access_for_table(); get_best_mvi_access() + prices them and picks one. + */ + List *mvi_accesses; /* Either #rows in the table or 1 for const table. */ ha_rows records; @@ -3035,5 +3042,6 @@ void propagate_new_equalities(THD *thd, Item *cond, bool dbug_user_var_equals_str(THD *thd, const char *name, const char *value); #include "opt_vcol_substitution.h" +#include "opt_multi_valued_index.h" #endif /* SQL_SELECT_INCLUDED */ diff --git a/sql/sql_show.cc b/sql/sql_show.cc index f716dc94e2b39..9bb6a0c205d5d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2476,6 +2476,20 @@ int show_create_table_ex(THD *thd, TABLE_LIST *table_list, const char *force_db, { if (key_info->flags & HA_INVISIBLE_KEY) continue; + if (is_mvi_key(table, i)) + { + /* + A multi-valued index. The column that holds its keys is internal and + is not printed, so print the expression the index was declared with + instead - that is also the only form that can be read back. + */ + packet->append(STRING_WITH_LEN(",\n KEY ")); + append_identifier(thd, packet, &key_info->name); + packet->append(STRING_WITH_LEN(" ((")); + print_mvi_key_expr(packet, table, i); + packet->append(STRING_WITH_LEN("))")); + continue; + } KEY_PART_INFO *key_part= key_info->key_part; bool found_primary=0; packet->append(STRING_WITH_LEN(",\n ")); @@ -7480,6 +7494,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, TABLE *table, for (uint j=0 ; j < key_info->user_defined_key_parts ; j++,key_part++) { if (key_part->field->invisible >= INVISIBLE_SYSTEM && + !is_mvi_key(show_table, i) && !DBUG_IF("test_completely_invisible")) { /* diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 54a604e404d47..5655d6e83cf73 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -116,8 +116,6 @@ static Lex_ident_column make_unique_key_name(THD *, static bool make_unique_constraint_name(THD *, LEX_CSTRING *, const char *, List *, List *, uint *); -static Lex_ident_column make_internal_field_name(THD *, const char *, - List *); static int copy_data_between_tables(THD *, TABLE *,TABLE *, bool, uint, ORDER *, ha_rows *, ha_rows *, @@ -2775,18 +2773,24 @@ static int mysql_add_invisible_field(THD *thd, List * field_list, #define INTERNAL_FIELD_NAME_LENGTH 30 -static Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, List *create_list) { char buf[INTERNAL_FIELD_NAME_LENGTH]= {0}; LEX_CSTRING name= { buf, 0 }; - bool dup_found= true; - for (uint num= 1; dup_found; num++) + for (uint num= 1; ; num++) { + /* + Note this has to start at false: `create_list' can be empty, and then + the loop below does not run at all. + */ + bool dup_found= false; name.length= my_snprintf(buf, sizeof(buf), "%s%u", prefix, num); for (auto &dup_field : *create_list) if ((dup_found= dup_field.field_name.streq(name))) break; + if (!dup_found) + break; } return Lex_ident_column(thd->strmake_lex_cstring(name)); } @@ -2970,6 +2974,17 @@ my_bool init_key_part_spec(THD *thd, Alter_info *alter_info, DBUG_RETURN(TRUE); } + /* + An index over an ARRAY has exactly one key part. A key with several of + them has no defining expression to show in SHOW CREATE TABLE, and no + syntax of its own that would read it back in. + */ + if (is_mvi_vcol(column) && key.columns.elements != 1) + { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + DBUG_RETURN(TRUE); + } + const Type_handler *type_handler= column->type_handler(); switch(key.type) { @@ -8546,6 +8561,41 @@ void rename_field_in_list(Create_field *field, List *field_list) #endif +/* + @brief + Should `field', the internal column of a multi-valued index, survive this + ALTER TABLE? + + @detail + It only exists to hold the entries of one key, so it lives exactly as + long as that key does: a column whose key is being dropped goes with it, + and so does one that has no key left at all. +*/ + +static bool mvi_vcol_kept_by_alter(TABLE *table, Field *field, + Alter_info *alter_info) +{ + KEY *key_info= table->key_info; + if (!is_mvi_vcol(field)) + return false; + for (uint i= 0; i < table->s->total_keys; i++, key_info++) + { + if (!is_mvi_key(table, i) || key_info->key_part[0].field != field) + continue; + /* This is its key. Keep the column unless the key is going away */ + List_iterator drop_it(alter_info->drop_list); + while (Alter_drop *drop= drop_it++) + { + if (drop->type == Alter_drop::KEY && + Lex_ident_column(key_info->name).streq(drop->name)) + return false; + } + return true; + } + return false; +} + + /** Prepare column and key definitions for CREATE TABLE in ALTER TABLE. @@ -8712,7 +8762,14 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, bitmap_clear_all(&table->tmp_set); for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) { - if (field->invisible == INVISIBLE_FULL) + /* + Internal columns are re-created from scratch by the new table's DDL, + except the one that holds the keys of a multi-valued index: there is no + syntax that would re-create that one, so carry it over as it is, for as + long as its key is (see the key loop below). + */ + if (field->invisible == INVISIBLE_FULL && + !mvi_vcol_kept_by_alter(table, field, alter_info)) continue; Alter_drop *drop; if (field->type() == MYSQL_TYPE_VARCHAR) @@ -8896,7 +8953,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, } else { - DBUG_ASSERT(field->invisible == INVISIBLE_SYSTEM); + /* The internal column of a multi-valued index also goes last */ + DBUG_ASSERT(field->invisible == INVISIBLE_SYSTEM || is_mvi_vcol(field)); def= new (root) Create_field(thd, field, field); new_create_tail.push_back(def, root); } @@ -9404,6 +9462,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key->without_overlaps= key_info->without_overlaps; key->period= table->s->period.name; key->old= true; + /* + A multi-valued index: its only key part is an internal column. Let + the key keep it, see init_key_part_spec(). + */ + key->invisible= is_mvi_key(table, i); new_key_list.push_back(key, root); } if (long_hash_key) diff --git a/sql/sql_table.h b/sql/sql_table.h index 3a27204f2c45b..3d34515cc43f2 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -226,4 +226,7 @@ extern MYSQL_PLUGIN_IMPORT const Lex_ident_column primary_key_name; bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, + List *create_list); + #endif /* SQL_TABLE_INCLUDED */ diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c2fe9974c42df..31725d459c8f8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1011,6 +1011,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token MONITOR_SYM /* MariaDB privilege */ %token MONTH_SYM /* SQL-2003-R */ %token MUTEX_SYM +%token MVI_ENCODE_SYM %token MYSQL_SYM %token MYSQL_ERRNO_SYM %token NAMES_SYM /* SQL-2003-N */ @@ -1761,7 +1762,7 @@ rule: using_list opt_use_partition use_partition %type - key_part key_part_simple + key_part key_part_simple multi_valued_key_part %type join_table_list join_table @@ -7636,6 +7637,7 @@ opt_without_overlaps: key_part: key_part_simple + | multi_valued_key_part | ident '(' NUM ')' { int key_part_len= atoi($3.str); @@ -7647,6 +7649,14 @@ key_part: } ; +multi_valued_key_part: + '(' CAST_SYM '(' expr AS cast_type ARRAY_SYM ')' ')' + { + if (unlikely(!($$= add_mvi_key_part(thd, $4, $6)))) + MYSQL_YYABORT; + } + ; + key_part_simple: ident { @@ -11146,6 +11156,12 @@ function_call_nonkeyword: MYSQL_YYABORT; Lex->safe_to_cache_query= false; } + | MVI_ENCODE_SYM '(' expr ',' cast_type ')' + { + $$= new (thd->mem_root) Item_func_mvi_encode(thd, $3, $5); + if (unlikely($$ == NULL)) + MYSQL_YYABORT; + } | NOW_SYM opt_time_precision { $$= new (thd->mem_root) Item_func_current_timestamp(thd, $2); @@ -16907,6 +16923,7 @@ keyword_sp_var_not_label: | HELP_SYM | HOST_SYM | INSTALL_SYM + | MVI_ENCODE_SYM | OPTION | OPTIONS_SYM | OTHERS_MARIADB_SYM diff --git a/sql/table.cc b/sql/table.cc index acc6377de1b50..db3c037ba8d9e 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3215,9 +3215,19 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->incompatible_version|= HA_CREATE_USED_CHARSET; key_part->type= field->key_type(); - if (field->invisible > INVISIBLE_USER && !field->vers_sys_field()) - if (keyinfo->algorithm != HA_KEY_ALG_LONG_HASH) - keyinfo->flags |= HA_INVISIBLE_KEY; + /* + A key part the user cannot name normally hides the whole key. Two + kinds of key are built that way on purpose and are not hidden: + a long unique, and a multi-valued index - a fulltext key over one + internal column holding the index keys. We cannot use is_mvi_key() + to recognize the latter: the vcol expressions are not parsed until + parse_vcol_defs(), long after this. + */ + if (field->invisible > INVISIBLE_USER && !field->vers_sys_field() && + keyinfo->algorithm != HA_KEY_ALG_LONG_HASH && + !(keyinfo->algorithm == HA_KEY_ALG_FULLTEXT && + keyinfo->user_defined_key_parts == 1)) + keyinfo->flags |= HA_INVISIBLE_KEY; if (field->null_ptr) { key_part->null_offset=(uint) ((uchar*) field->null_ptr - diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index c1b4a5c9559fc..b9465456d0027 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -173,6 +173,7 @@ SET(INNOBASE_SOURCES fts/fts0blex.cc fts/fts0config.cc fts/fts0exec.cc + fts/fts0est.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc diff --git a/storage/innobase/fts/fts0est.cc b/storage/innobase/fts/fts0est.cc new file mode 100644 index 0000000000000..e1c0b334c513b --- /dev/null +++ b/storage/innobase/fts/fts0est.cc @@ -0,0 +1,545 @@ +/***************************************************************************** + +Copyright (c) 2026, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0est.cc +Estimating the size of a fulltext search result. + +fts_estimate_word_docs() is the fulltext analogue of records_in_range(): it +answers "how many documents contain this word?" cheaply enough to be called +while the optimizer is still choosing a plan. It probes the auxiliary +INDEX_[1..6] B-tree that holds the word and the in-memory FTS cache, takes no +record locks, opens no read view, creates no transaction, and never reads an +ilist. + +Created 2026/09/06 +*******************************************************/ + +#include "fts0fts.h" +#include "fts0priv.h" +#include "fts0types.h" +#include "fts0types.inl" +#include "dict0dict.h" +#include "btr0pcur.h" +#include "mtr0mtr.h" +#include "rem0cmp.h" +#include "page0page.h" + +/** Maximum number of auxiliary records fts_estimate_word_docs() samples +before it gives up on an exact answer and extrapolates instead. */ +static constexpr uint32_t FTS_EST_MAX_RECS = 64; +/** Maximum number of auxiliary leaf pages fts_estimate_word_docs() reads. */ +static constexpr uint32_t FTS_EST_MAX_PAGES = 4; + +/* Physical field numbers of an FTS auxiliary INDEX_[1..6] record. The +clustered index is UNIQUE(word, first_doc_id), so the record is +(word, first_doc_id, DB_TRX_ID, DB_ROLL_PTR, last_doc_id, doc_count, ilist); +see fts_create_one_index_table(). */ +static constexpr ulint FTS_AUX_FLD_WORD = 0; +static constexpr ulint FTS_AUX_FLD_FIRST_DOC_ID = 1; +static constexpr ulint FTS_AUX_FLD_LAST_DOC_ID = 4; +static constexpr ulint FTS_AUX_FLD_DOC_COUNT = 5; +/** Number of offsets the estimator needs. This deliberately stops short of +the ilist (field 6), which may be stored off-page: the estimator must never +read a BLOB. */ +static constexpr ulint FTS_AUX_EST_N_FIELDS = FTS_AUX_FLD_DOC_COUNT + 1; + +/** Build the search tuple (word) for an FTS auxiliary clustered index. +The index key is (word, first_doc_id), and because the tuple compares on its +first field only it compares equal to every record of the word. So +PAGE_CUR_GE positions on the word's first record and PAGE_CUR_LE on its last. +@param[in] heap heap to allocate the tuple from +@param[in] aux_index auxiliary clustered index +@param[in] word word to search for +@return the search tuple */ +static +dtuple_t* +fts_est_word_tuple( + mem_heap_t* heap, + dict_index_t* aux_index, + const fts_string_t* word) noexcept +{ + dtuple_t* tuple = dtuple_create(heap, 1); + + dict_index_copy_types(tuple, aux_index, 1); + dfield_set_data(dtuple_get_nth_field(tuple, 0), + word->f_str, word->f_len); + dtuple_set_n_fields_cmp(tuple, 1); + + return(tuple); +} + +/** Read the fields the estimator needs out of an FTS auxiliary INDEX_[1..6] +leaf record. Never touches the ilist. +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in] rec auxiliary table record +@param[in] aux_index auxiliary clustered index +@param[in] offsets rec_get_offsets(rec, aux_index, ...) +@param[out] first_doc_id first doc id in this record's ilist +@param[out] last_doc_id last doc id in this record's ilist +@param[out] doc_count number of doc ids in this record's ilist +@return whether the record belongs to the word tuple was built for */ +static +bool +fts_est_read_rec( + const dtuple_t* tuple, + const rec_t* rec, + const dict_index_t* aux_index, + const rec_offs* offsets, + doc_id_t* first_doc_id, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + /* tuple has n_fields_cmp == 1, so this compares the word only, in the + collation of the auxiliary table's word column. */ + if (cmp_dtuple_rec(tuple, rec, aux_index, offsets)) { + return(false); + } + + ulint len; + const byte* data; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_FIRST_DOC_ID, &len); + *first_doc_id = (data && len == sizeof *first_doc_id) + ? fts_read_doc_id(data) : 0; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_LAST_DOC_ID, &len); + *last_doc_id = (data && len == sizeof *last_doc_id) + ? fts_read_doc_id(data) : *first_doc_id; + + data = rec_get_nth_field(rec, offsets, FTS_AUX_FLD_DOC_COUNT, &len); + *doc_count = (data && len == 4) ? mach_read_from_4(data) : 0; + + return(true); +} + +/** What fts_est_sample_word() found. */ +struct fts_est_sample_t +{ + /** Sum of doc_count over the records sampled. */ + uint64_t docs; + /** Number of records sampled. */ + uint32_t n_recs; + /** First doc id of the word's first record. */ + doc_id_t first_doc_id; + /** Last doc id of the last record sampled. */ + doc_id_t last_doc_id; + /** Whether sampling stopped because a budget ran out rather than + because the word's key range ended. When this is false, docs is the + exact number of documents that contain the word. */ + bool truncated; +}; + +/** Dive to a word's first auxiliary record and walk forward from it, over at +most FTS_EST_MAX_RECS records and FTS_EST_MAX_PAGES leaf pages. The walk is +nearly free because the dive has already latched the leaf page, and if it +reaches a different word before a budget runs out the result is exact. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[out] out what the walk found +@return DB_SUCCESS, or DB_RECORD_NOT_FOUND if the auxiliary table is empty +(nothing has been SYNCed yet, so there is no information at all) */ +static +dberr_t +fts_est_sample_word( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + fts_est_sample_t* out) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + memset(out, 0, sizeof *out); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err != DB_SUCCESS) { + goto func_exit; + } + + if (!btr_pcur_is_on_user_rec(&pcur)) { + /* Nothing at or after the word. Tell an empty auxiliary + table, where we have no information at all, apart from a word + that simply sorts after everything that is indexed. */ + const page_t* page = btr_pcur_get_page(&pcur); + + if (!page_has_prev(page) && !page_get_n_recs(page)) { + err = DB_RECORD_NOT_FOUND; + } + + goto func_exit; + } + + { + uint32_t n_pages = 1; + page_id_t last_page + = btr_pcur_get_block(&pcur)->page.id(); + + do { + const rec_t* rec = btr_pcur_get_rec(&pcur); + const buf_block_t* block + = btr_pcur_get_block(&pcur); + const ulint offs + = ulint(rec - block->page.frame); + + if (page_rec_is_infimum_low(offs) + || page_rec_is_supremum_low(offs)) { + continue; + } + + if (block->page.id() != last_page) { + last_page = block->page.id(); + if (++n_pages > FTS_EST_MAX_PAGES) { + out->truncated = true; + break; + } + } + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + doc_id_t first; + doc_id_t last; + uint32_t doc_count; + + if (!fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &doc_count)) { + /* Walked past the word's key range, so we + have seen all of it. */ + break; + } + + if (!out->n_recs) { + out->first_doc_id = first; + } + + out->last_doc_id = last; + out->docs += doc_count; + + if (++out->n_recs >= FTS_EST_MAX_RECS) { + out->truncated = true; + break; + } + } while (btr_pcur_move_to_next(&pcur, &mtr)); + } + +func_exit: + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Dive to a word's last auxiliary record, to learn how far its doc ids +reach. +@param[in] trx transaction, used only for the mtr +@param[in] aux_index auxiliary clustered index +@param[in] tuple search tuple built by fts_est_word_tuple() +@param[in,out] last_doc_id last doc id of the word; left alone if the + record cannot be read +@param[in,out] doc_count doc_count of that record; left alone if the + record cannot be read +@return DB_SUCCESS or error code */ +static +dberr_t +fts_est_last_rec( + trx_t* trx, + dict_index_t* aux_index, + const dtuple_t* tuple, + doc_id_t* last_doc_id, + uint32_t* doc_count) noexcept +{ + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets = offsets_; + mem_heap_t* offs_heap = NULL; + btr_pcur_t pcur; + mtr_t mtr{trx}; + + rec_offs_init(offsets_); + + mtr.start(); + pcur.btr_cur.page_cur.index = aux_index; + + dberr_t err = btr_pcur_open(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, + &pcur, &mtr); + + if (err == DB_SUCCESS) { + /* PAGE_CUR_LE may leave the cursor on the page infimum, in + which case the record we want is the last one of the previous + page. */ + const bool positioned + = !btr_pcur_is_before_first_on_page(&pcur) + || btr_pcur_move_to_prev(&pcur, &mtr); + + if (positioned && btr_pcur_is_on_user_rec(&pcur)) { + const rec_t* rec = btr_pcur_get_rec(&pcur); + doc_id_t first; + doc_id_t last; + uint32_t count; + + offsets = rec_get_offsets(rec, aux_index, offsets, + aux_index->n_core_fields, + FTS_AUX_EST_N_FIELDS, + &offs_heap); + + if (fts_est_read_rec(tuple, rec, aux_index, offsets, + &first, &last, &count)) { + *last_doc_id = last; + *doc_count = count; + } + } + } + + mtr.commit(); + ut_free(pcur.old_rec_buf); + + if (UNIV_LIKELY_NULL(offs_heap)) { + mem_heap_free(offs_heap); + } + + return(err); +} + +/** Estimate how many documents contain a word, by probing an already opened +FTS auxiliary INDEX_[1..6] table. See fts_estimate_word_docs(). +@param[in] trx transaction, used only for the mtr +@param[in] aux auxiliary table that holds the word +@param[in] word word to look up +@param[out] n_docs estimated number of matching documents +@return DB_SUCCESS, DB_RECORD_NOT_FOUND or DB_CORRUPTION */ +static +dberr_t +fts_est_probe_aux( + trx_t* trx, + dict_table_t* aux, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + dict_index_t* aux_index = dict_table_get_first_index(aux); + + if (!aux->space || !aux->is_readable() || !aux_index + || aux_index->page == FIL_NULL || aux_index->is_corrupted()) { + return(DB_CORRUPTION); + } + + mem_heap_t* heap = mem_heap_create(256); + const dtuple_t* tuple = fts_est_word_tuple(heap, aux_index, + word); + fts_est_sample_t s; + dberr_t err = fts_est_sample_word(trx, aux_index, + tuple, &s); + + if (err != DB_SUCCESS) { + mem_heap_free(heap); + return(err); + } + + if (!s.n_recs || !s.truncated) { + /* Either the word is absent, or the walk covered its whole + key range, in which case the count is exact. */ + *n_docs = s.docs; + mem_heap_free(heap); + return(DB_SUCCESS); + } + + /* The word has more records than we are willing to read. Dive once + more, to its last record, and extrapolate the density we measured + across the word's whole doc id span. Doc ids only ever increase, so + the records we sampled are a prefix of that span. */ + doc_id_t last_doc_id = s.last_doc_id; + uint32_t last_count = 0; + + err = fts_est_last_rec(trx, aux_index, tuple, &last_doc_id, + &last_count); + + if (err == DB_SUCCESS) { + if (last_doc_id < s.last_doc_id) { + last_doc_id = s.last_doc_id; + } + + const uint64_t sampled_span + = s.last_doc_id > s.first_doc_id + ? s.last_doc_id - s.first_doc_id + 1 : 1; + const uint64_t total_span + = last_doc_id - s.first_doc_id + 1; + + /* In double, because docs * total_span overflows 64 bits for + large inputs. The caller clamps the result to the number of + rows in the table. */ + uint64_t est = uint64_t(double(s.docs) + * double(total_span) + / double(sampled_span)); + + /* Never below what we actually counted. */ + if (est < s.docs + last_count) { + est = s.docs + last_count; + } + + *n_docs = est; + } + + mem_heap_free(heap); + + return(err); +} + +/** Count the documents that contain a word and are still only in the +in-memory FTS cache, that is, have not been SYNCed to the auxiliary table yet. + +fts_node_t::doc_count already holds the number we want, so this decodes no +ilist and reads nothing from disk; it is one rb tree lookup. + +Nodes flagged as synced are skipped. An in-flight SYNC has already written +them to the auxiliary table, and fts_est_probe_aux() reads the B-tree without +a read view, so it sees those records; counting the node as well would count +its documents twice. + +@param[in] index fulltext index +@param[in] word word to look up, folded the same way the caller folds it + for the auxiliary table +@return number of matching documents found in the cache, or 0 if there are +none, if the cache is not initialized, or if another thread holds the cache +mutex (an estimate is not worth waiting for a SYNC to finish) */ +static +uint64_t +fts_est_cache_docs( + const dict_index_t* index, + const fts_string_t* word) noexcept +{ + const fts_t* fts = index->table->fts; + + if (!fts || !fts->cache) { + return(0); + } + + fts_cache_t* cache = fts->cache; + + if (mysql_mutex_trylock(&cache->lock)) { + return(0); + } + + uint64_t n_docs = 0; + + if (const fts_index_cache_t* index_cache + = fts_find_index_cache(cache, index)) { + /* fts_cache_clear() leaves words NULL until the following + fts_cache_init(). The query path never observes that, because + it runs after fts_init_index(); we may. */ + if (index_cache->words) { + const ib_vector_t* nodes + = fts_cache_find_word(index_cache, word); + + for (ulint i = 0; nodes && i < ib_vector_size(nodes); + ++i) { + const fts_node_t* node + = static_cast( + ib_vector_get_const(nodes, i)); + + if (!node->synced) { + n_docs += node->doc_count; + } + } + } + } + + mysql_mutex_unlock(&cache->lock); + + return(n_docs); +} + +dberr_t +fts_estimate_word_docs( + trx_t* trx, + dict_index_t* index, + const fts_string_t* word, + uint64_t* n_docs) noexcept +{ + ut_ad(index->type & DICT_FTS); + ut_ad(!dict_sys.locked()); + ut_ad(word->f_len); + + *n_docs = 0; + + /* Documents that have been inserted but not SYNCed yet are only in the + memory cache. Look there first: it costs no I/O, and on a table that + has never been SYNCed it is the only information there is. */ + const uint64_t cached = fts_est_cache_docs(index, word); + + /* A word lives in exactly one of INDEX_1..INDEX_6, so only that one + auxiliary table is ever opened -- unlike the query path, which opens + all six. */ + CHARSET_INFO* cs = fts_index_get_charset(index); + const uint8_t selected = fts_select_index(cs, word->f_str, + word->f_len); + fts_table_t fts_table; + + FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), + FTS_INDEX_TABLE, index); + + char aux_name[MAX_FULL_NAME_LEN]; + + fts_get_table_name(&fts_table, aux_name, false); + + dict_table_t* aux = dict_table_open_on_name( + aux_name, false, DICT_ERR_IGNORE_TABLESPACE); + + if (!aux) { + if (cached) { + *n_docs = cached; + return(DB_SUCCESS); + } + + return(DB_TABLE_NOT_FOUND); + } + + dberr_t err = fts_est_probe_aux(trx, aux, word, n_docs); + + aux->release(); + + if (err == DB_SUCCESS) { + /* The two populations are disjoint: fts_est_cache_docs() + skipped every node that the auxiliary table already holds. */ + *n_docs += cached; + } else if (cached && err == DB_RECORD_NOT_FOUND) { + /* The auxiliary table is empty, but the cache is not, so we + are no longer without information. The other way round -- + nothing in either -- stays DB_RECORD_NOT_FOUND: the cache is + only complete once fts_init_index() has run, and an estimate + must not run it. */ + *n_docs = cached; + err = DB_SUCCESS; + } + + return(err); +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b60155f995683..2178c2b0d24d0 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14741,6 +14741,133 @@ ha_innobase::records_in_range( goto cleanup; } +/*********************************************************************//** +Estimates the number of records a fulltext search for a single word will +match. The fulltext analogue of records_in_range(). + +Unlike records_in_range() this leaves all handler state alone (in particular +active_index and trx->op_info): the optimizer may well discard this access +path. And unlike ft_init_ext() it does not lazily run fts_init_index(): an +estimate must have no side effects. +@return estimated number of rows, or HA_POS_ERROR if not known */ + +ha_rows +ha_innobase::fulltext_estimate( +/*===========================*/ + uint index_nr, /*!< in: fulltext index number */ + const char* word, /*!< in: word to search for, in the + character set of the fulltext index */ + uint word_len) /*!< in: length of word in bytes */ +{ + DBUG_ENTER("ha_innobase::fulltext_estimate"); + + if (!word || !word_len || word_len > FTS_MAX_WORD_LEN) { + DBUG_RETURN(HA_POS_ERROR); + } + + ut_ad(m_prebuilt->trx == thd_to_trx(ha_thd())); + + dict_table_t* ft_table = m_prebuilt->table; + + /* Mirrors the validation of ft_init_ext(), but reports nothing: this + runs during optimization, where raising an error would corrupt the + statement. */ + if (!ft_table->fts + || ib_vector_is_empty(ft_table->fts->indexes) + || !ft_table->space) { /* tablespace discarded */ + DBUG_RETURN(HA_POS_ERROR); + } + + dict_index_t* index = innobase_get_index(index_nr); + + if (!index + || index->type != DICT_FTS + || index->is_corrupted() + || !row_merge_is_index_usable(m_prebuilt->trx, index)) { + DBUG_RETURN(HA_POS_ERROR); + } + + /* The auxiliary tables store words folded to lower case, so fold the + search word the same way fts_query() folds the query string. */ + CHARSET_INFO* cs = fts_index_get_charset(index); + + /* A utf16 or utf32 fulltext index would need the conversion that + ft_init_ext() does; not worth it for an estimate. */ + if (cs->mbminlen != 1) { + DBUG_RETURN(HA_POS_ERROR); + } + + byte buf[FTS_MAX_WORD_LEN * 2 + 1]; + fts_string_t w; + + w.f_n_char = 0; + + if (my_binary_compare(cs)) { + /* Binary collations are searched case sensitively. */ + w.f_str = reinterpret_cast(const_cast(word)); + w.f_len = word_len; + } else { + const size_t buf_len + = word_len * cs->casedn_multiply() + 1; + + if (buf_len > sizeof buf) { + DBUG_RETURN(HA_POS_ERROR); + } + + w.f_len = cs->casedn_z(word, word_len, + reinterpret_cast(buf), buf_len); + w.f_str = buf; + } + + if (!w.f_len) { + DBUG_RETURN(HA_POS_ERROR); + } + + uint64_t n_docs; + dberr_t err; + + { + /* Attribute the pages read to this handler, the way + records_in_range() does. No private transaction is needed: + fts_estimate_word_docs() only probes the B-tree, so it takes + no locks, opens no read view and never commits anything. */ + mariadb_set_stats temp(m_prebuilt->trx, handler_stats); + + err = fts_estimate_word_docs(m_prebuilt->trx, index, &w, + &n_docs); + } + + if (err != DB_SUCCESS) { + /* Includes DB_RECORD_NOT_FOUND, which fts_estimate_word_docs() + returns for an empty auxiliary table: nothing has been SYNCed + yet, so we know nothing at all. */ + DBUG_RETURN(HA_POS_ERROR); + } + + /* The auxiliary tables still hold entries for rows that were deleted + or updated since the last OPTIMIZE TABLE, so the count can exceed the + number of rows in the table. */ + if (ft_table->stat_initialized()) { + const uint64_t n_rows = dict_table_get_n_rows(ft_table); + + if (n_docs > n_rows) { + n_docs = n_rows; + } + } + + /* Never report 0. The in-memory FTS cache is consulted only on a best + effort basis (see fts_estimate_word_docs()), so "found nowhere" does + not mean "no matching rows", and callers may treat 0 as provably + empty. */ + if (!n_docs) { + n_docs = 1; + } + + /* HA_ROWS_MAX is HA_POS_ERROR, the "not known" value, so stay below + it. */ + DBUG_RETURN((ha_rows) std::min(n_docs, HA_ROWS_MAX - 1)); +} + /*********************************************************************//** Gives an UPPER BOUND to the number of rows in a table. This is used in filesort.cc. diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index d47a52e5061ea..08a61651adacd 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -196,6 +196,11 @@ class ha_innobase final : public handler const key_range* max_key, page_range* pages) override; + ha_rows fulltext_estimate( + uint index_nr, + const char* word, + uint word_len) override; + ha_rows estimate_rows_upper_bound() override; void update_create_info(HA_CREATE_INFO* create_info) override; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 32e4d2cb1e887..0ed3e25d65b4f 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -537,6 +537,46 @@ fts_query( fts_result_t** result) MY_ATTRIBUTE((warn_unused_result)); +/** Estimate the number of documents that contain a single word, by probing +the FTS auxiliary INDEX_[1..6] table that holds it and the in-memory FTS +cache. + +This is cheap, in the spirit of records_in_range(): one rb tree lookup in the +cache, plus one B-tree dive and a bounded walk over the leaf pages that dive +already latched. It takes no record locks, opens no read view and creates no +transaction. It is therefore deliberately approximate: + + - the cache is consulted only on a best effort basis: it is skipped when + another thread holds cache->lock, and it is incomplete until + fts_init_index() has run, which an estimate must not do. A word can + therefore still be reported as absent even though rows do match; + - FTS_..._DELETED and DELETED_CACHE are NOT consulted, so documents that + were deleted or updated since the last OPTIMIZE TABLE are still counted; + - delete-marked auxiliary records are counted, exactly the way + records_in_range() counts delete-marked index records; + - the word is matched literally: no wildcards, no stemming, and no stopword + or token length filtering. + +@param[in] trx transaction to attribute buffer pool statistics to; it + is neither started, modified nor committed +@param[in] index fulltext index (index->type & DICT_FTS) +@param[in] word word to look up, in index's charset, already folded to + lower case unless my_binary_compare(charset) +@param[out] n_docs estimated number of matching documents; 0 means the + word is present neither in the auxiliary table nor in + the cache +@return DB_SUCCESS, DB_TABLE_NOT_FOUND if the auxiliary table cannot be opened +and the cache knows nothing either, DB_RECORD_NOT_FOUND if both the auxiliary +table and the cache are empty (so there is no information at all), or +DB_CORRUPTION */ +dberr_t +fts_estimate_word_docs( + trx_t* trx, + dict_index_t* index, + const fts_string_t* word, + uint64_t* n_docs) + noexcept MY_ATTRIBUTE((nonnull, warn_unused_result)); + /******************************************************************//** Retrieve the FTS Relevance Ranking result for doc with doc_id @return the relevance ranking value. */