/* Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the Free Software Foundation. This program is also distributed with certain software (including but not limited to OpenSSL) that is licensed under separate terms, as designated in a particular file or component or in included license documentation. The authors of MySQL hereby grant you an additional permission to link the program and your derivative works with the separately licensed software that they have included with MySQL. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /** @file @brief Optimize query expressions: Make optimal table join order, select optimal access methods per table, apply grouping, sorting and limit processing. @defgroup Query_Optimizer Query Optimizer @{ */ #include "sql/sql_optimizer.h" #include #include #include #include #include #include "field_types.h" // enum_field_types #include "ft_global.h" #include "item_json_func.h" // get_json_wrapper() #include "m_ctype.h" #include "memory_debugging.h" #include "my_bit.h" // my_count_bits #include "my_bitmap.h" #include "my_dbug.h" #include "my_inttypes.h" #include "my_macros.h" #include "my_sqlcommand.h" #include "my_sys.h" #include "mysql/udf_registration_types.h" #include "mysql_com.h" #include "mysqld_error.h" #include "sql/abstract_query_plan.h" // Join_plan #include "sql/check_stack.h" #include "sql/debug_sync.h" // DEBUG_SYNC #include "sql/derror.h" // ER_THD #include "sql/enum_query_type.h" #include "sql/error_handler.h" // Functional_index_error_handler #include "sql/handler.h" #include "sql/item.h" #include "sql/item_cmpfunc.h" #include "sql/item_func.h" #include "sql/item_row.h" #include "sql/item_sum.h" // Item_sum #include "sql/key.h" #include "sql/key_spec.h" #include "sql/lock.h" // mysql_unlock_some_tables #include "sql/mysqld.h" // stage_optimizing #include "sql/nested_join.h" #include "sql/opt_costmodel.h" #include "sql/opt_explain.h" // join_type_str #include "sql/opt_hints.h" // hint_table_state #include "sql/opt_range.h" // QUICK_SELECT_I #include "sql/opt_trace.h" // Opt_trace_object #include "sql/opt_trace_context.h" #include "sql/query_options.h" #include "sql/query_result.h" #include "sql/sql_base.h" // init_ftfuncs #include "sql/sql_bitmap.h" #include "sql/sql_const.h" #include "sql/sql_const_folding.h" #include "sql/sql_error.h" #include "sql/sql_join_buffer.h" // JOIN_CACHE #include "sql/sql_planner.h" // calculate_condition_filter #include "sql/sql_test.h" // print_where #include "sql/sql_tmp_table.h" #include "sql/system_variables.h" #include "sql/table.h" #include "sql/timing_iterator.h" #include "sql/window.h" #include "sql_string.h" using std::max; using std::min; const char *antijoin_null_cond = ""; static bool optimize_semijoin_nests_for_materialization(JOIN *join); static void calculate_materialization_costs(JOIN *join, TABLE_LIST *sj_nest, uint n_tables, Semijoin_mat_optimize *sjm); static bool make_join_select(JOIN *join, Item *item); static bool list_contains_unique_index(JOIN_TAB *tab, bool (*find_func)(Field *, void *), void *data); static bool find_field_in_item_list(Field *field, void *data); static bool find_field_in_order_list(Field *field, void *data); static TABLE *get_sort_by_table(ORDER *a, ORDER *b, TABLE_LIST *tables); static void trace_table_dependencies(Opt_trace_context *trace, JOIN_TAB *join_tabs, uint table_count); static bool update_ref_and_keys(THD *thd, Key_use_array *keyuse, JOIN_TAB *join_tab, uint tables, Item *cond, table_map normal_tables, SELECT_LEX *select_lex, SARGABLE_PARAM **sargables); static bool pull_out_semijoin_tables(JOIN *join); static void add_loose_index_scan_and_skip_scan_keys(JOIN *join, JOIN_TAB *join_tab); static ha_rows get_quick_record_count(THD *thd, JOIN_TAB *tab, ha_rows limit); static bool only_eq_ref_tables(JOIN *join, ORDER *order, table_map tables, table_map *cached_eq_ref_tables, table_map *eq_ref_tables); static bool setup_join_buffering(JOIN_TAB *tab, JOIN *join, uint no_jbuf_after); static bool test_if_skip_sort_order(JOIN_TAB *tab, ORDER_with_src &order, ha_rows select_limit, const bool no_changes, const Key_map *map, int *best_idx); static Item_func_match *test_if_ft_index_order(ORDER *order); static uint32 get_key_length_tmp_table(Item *item); static bool can_switch_from_ref_to_range(THD *thd, JOIN_TAB *tab, enum_order ordering, bool recheck_range); static bool has_not_null_predicate(Item *cond, Item_field *not_null_item); bool JOIN::alloc_indirection_slices() { const uint card = REF_SLICE_WIN_1 + m_windows.elements * 2; DBUG_ASSERT(ref_items == nullptr); ref_items = (Ref_item_array *)(*THR_MALLOC)->Alloc(sizeof(Ref_item_array) * card); if (ref_items == nullptr) return true; tmp_all_fields = (List *)(*THR_MALLOC)->Alloc(sizeof(List) * card); if (tmp_all_fields == nullptr) return true; tmp_fields_list = (List *)(*THR_MALLOC)->Alloc(sizeof(List) * card); if (tmp_fields_list == nullptr) return true; for (uint i = 0; i < card; i++) { ref_items[i].reset(); tmp_all_fields[i].empty(); tmp_fields_list[i].empty(); } return false; } /** Optimizes one query block into a query execution plan (QEP.) This is the entry point to the query optimization phase. This phase applies both logical (equivalent) query rewrites, cost-based join optimization, and rule-based access path selection. Once an optimal plan is found, the member function creates/initializes all structures needed for query execution. The main optimization phases are outlined below: -# Logical transformations: - Outer to inner joins transformation. - Equality/constant propagation. - Partition pruning. - COUNT(*), MIN(), MAX() constant substitution in case of implicit grouping. - ORDER BY optimization. -# Perform cost-based optimization of table order and access path selection. See JOIN::make_join_plan() -# Post-join order optimization: - Create optimal table conditions from the where clause and the join conditions. - Inject outer-join guarding conditions. - Adjust data access methods after determining table condition (several times.) - Optimize ORDER BY/DISTINCT. -# Code generation - Set data access functions. - Try to optimize away sorting/distinct. - Setup temporary table usage for grouping and/or sorting. @retval false Success. @retval true Error, error code saved in member JOIN::error. */ bool JOIN::optimize() { DBUG_TRACE; uint no_jbuf_after = UINT_MAX; const bool has_windows = m_windows.elements != 0; DBUG_ASSERT(select_lex->leaf_table_count == 0 || thd->lex->is_query_tables_locked() || select_lex == unit->fake_select_lex); DBUG_ASSERT(tables == 0 && primary_tables == 0 && tables_list == (TABLE_LIST *)1); // to prevent double initialization on EXPLAIN if (optimized) return false; Prepare_error_tracker tracker(thd); DEBUG_SYNC(thd, "before_join_optimize"); THD_STAGE_INFO(thd, stage_optimizing); if (select_lex->first_execution) { /** @todo This query block didn't transform itself in SELECT_LEX::prepare(), so belongs to a parent query block. That parent, or its parents, had to transform us - it has not; maybe it is itself in prepare() and evaluating the present query block as an Item_subselect. Such evaluation in prepare() is expected to be a rare case to be eliminated in the future ("SET x=(subq)" is one such case; because it locks tables before prepare()). */ if (select_lex->apply_local_transforms(thd, false)) return (error = 1); } Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_optimize(trace, "join_optimization"); trace_optimize.add_select_number(select_lex->select_number); Opt_trace_array trace_steps(trace, "steps"); count_field_types(select_lex, &tmp_table_param, all_fields, false, false); DBUG_ASSERT(tmp_table_param.sum_func_count == 0 || group_list || implicit_grouping); if (select_lex->olap == ROLLUP_TYPE && optimize_rollup()) return true; /* purecov: inspected */ if (alloc_func_list()) return true; /* purecov: inspected */ if (select_lex->get_optimizable_conditions(thd, &where_cond, &having_cond)) return true; set_optimized(); tables_list = select_lex->leaf_tables; if (alloc_indirection_slices()) return true; // The base ref items from query block are assigned as JOIN's ref items ref_items[REF_SLICE_ACTIVE] = select_lex->base_ref_items; /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */ /* Run optimize phase for all derived tables/views used in this SELECT, including those in semi-joins. */ if (select_lex->materialized_derived_table_count) { for (TABLE_LIST *tl = select_lex->leaf_tables; tl; tl = tl->next_leaf) { if (tl->is_view_or_derived() && tl->optimize_derived(thd)) return true; } } has_lateral = false; /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */ row_limit = ((select_distinct || order || group_list) ? HA_POS_ERROR : unit->select_limit_cnt); // m_select_limit is used to decide if we are likely to scan the whole table. m_select_limit = unit->select_limit_cnt; if (unit->first_select()->active_options() & OPTION_FOUND_ROWS) { /* Calculate found rows (ie., keep counting rows even after we hit LIMIT) if - LIMIT is set, and - This is the outermost query block (for a UNION query, this is the fake query block that contains the limit applied on the final UNION evaluation). */ calc_found_rows = m_select_limit != HA_POS_ERROR && (!unit->is_union() || select_lex == unit->fake_select_lex); } if (having_cond || calc_found_rows) m_select_limit = HA_POS_ERROR; if (unit->select_limit_cnt == 0 && !calc_found_rows) { zero_result_cause = "Zero limit"; best_rowcount = 0; create_iterators_for_zero_rows(); goto setup_subq_exit; } if (where_cond || select_lex->outer_join) { if (optimize_cond(thd, &where_cond, &cond_equal, &select_lex->top_join_list, &select_lex->cond_value)) { error = 1; DBUG_PRINT("error", ("Error from optimize_cond")); return true; } if (select_lex->cond_value == Item::COND_FALSE) { zero_result_cause = "Impossible WHERE"; best_rowcount = 0; create_iterators_for_zero_rows(); goto setup_subq_exit; } } if (having_cond) { if (optimize_cond(thd, &having_cond, &cond_equal, NULL, &select_lex->having_value)) { error = 1; DBUG_PRINT("error", ("Error from optimize_cond")); return true; } if (select_lex->having_value == Item::COND_FALSE) { zero_result_cause = "Impossible HAVING"; best_rowcount = 0; create_iterators_for_zero_rows(); goto setup_subq_exit; } } if (select_lex->partitioned_table_count && prune_table_partitions()) { error = 1; DBUG_PRINT("error", ("Error from prune_partitions")); return true; } /* Try to optimize count(*), min() and max() to const fields if there is implicit grouping (aggregate functions but no group_list). In this case, the result set shall only contain one row. */ if (tables_list && implicit_grouping && !(select_lex->active_options() & OPTION_NO_CONST_TABLES)) { aggregate_evaluated outcome; if (optimize_aggregated_query(thd, select_lex, all_fields, where_cond, &outcome)) { error = 1; DBUG_PRINT("error", ("Error from optimize_aggregated_query")); return true; } switch (outcome) { case AGGR_REGULAR: // Query was not (fully) evaluated. Revert to regular optimization. break; case AGGR_DELAYED: // Query was not (fully) evaluated. Revert to regular optimization, // but indicate that storage engine supports HA_COUNT_ROWS_INSTANT. select_count = true; break; case AGGR_COMPLETE: // All SELECT expressions are fully evaluated DBUG_PRINT("info", ("Select tables optimized away")); zero_result_cause = "Select tables optimized away"; tables_list = nullptr; // All tables resolved best_rowcount = 1; const_tables = tables = primary_tables = select_lex->leaf_table_count; m_root_iterator = NewIterator(thd, &examined_rows); m_root_iterator = attach_iterators_for_having_and_limit(move(m_root_iterator)); /* There are no relevant conditions left from the WHERE; optimize_aggregated_query() will not return AGGR_COMPLETE if there are any table-independent conditions, and all other conditions have been optimized away by it. Thus, remove the condition, unless we have EXPLAIN (in which case we will keep it for printing). */ if (!thd->lex->is_explain()) { #ifndef DBUG_OFF // Verify, to be sure. if (where_cond != nullptr) { Item *table_independent_conds = make_cond_for_table( thd, where_cond, PSEUDO_TABLE_BITS, table_map(0), false); DBUG_ASSERT(table_independent_conds == nullptr); } #endif where_cond = nullptr; } goto setup_subq_exit; case AGGR_EMPTY: // It was detected that the result tables are empty DBUG_PRINT("info", ("No matching min/max row")); zero_result_cause = "No matching min/max row"; create_iterators_for_zero_rows(); goto setup_subq_exit; } } if (tables_list == nullptr) { DBUG_PRINT("info", ("No tables")); best_rowcount = 1; error = 0; if (make_tmp_tables_info()) return true; count_field_types(select_lex, &tmp_table_param, all_fields, false, false); // Make plan visible for EXPLAIN set_plan_state(NO_TABLES); create_iterators(); return false; } error = -1; // Error is sent to client { m_windowing_steps = false; // initialization m_windows_sort = false; List_iterator li(m_windows); Window *w; while ((w = li++)) if (w->needs_sorting()) { m_windows_sort = true; break; } } sort_by_table = get_sort_by_table(order, group_list, select_lex->leaf_tables); if ((where_cond || group_list || order) && substitute_gc(thd, select_lex, where_cond, group_list, order)) { // We added hidden fields to the all_fields list, count them. count_field_types(select_lex, &tmp_table_param, select_lex->all_fields, false, false); } // Ensure there are no errors prior making query plan if (thd->is_error()) return true; // Set up join order and initial access paths THD_STAGE_INFO(thd, stage_statistics); if (make_join_plan()) { if (thd->killed) thd->send_kill_message(); DBUG_PRINT("error", ("Error: JOIN::make_join_plan() failed")); return true; } // At this stage, join_tab==NULL, JOIN_TABs are listed in order by best_ref. ASSERT_BEST_REF_IN_JOIN_ORDER(this); if (zero_result_cause != nullptr) { // Can be set by make_join_plan(). create_iterators_for_zero_rows(); goto setup_subq_exit; } if (rollup.state == ROLLUP::STATE_NONE) { /* Remove distinct if only const tables */ select_distinct &= !plan_is_const(); } if (const_tables && !thd->locked_tables_mode && !(select_lex->active_options() & SELECT_NO_UNLOCK)) { TABLE *ct[MAX_TABLES]; for (uint i = 0; i < const_tables; i++) ct[i] = best_ref[i]->table(); mysql_unlock_some_tables(thd, ct, const_tables); } if (!where_cond && select_lex->outer_join) { /* Handle the case where we have an OUTER JOIN without a WHERE */ where_cond = new Item_func_true(); // Always true } error = 0; /* Among the equal fields belonging to the same multiple equality choose the one that is to be retrieved first and substitute all references to these in where condition for a reference for the selected field. */ if (where_cond) { where_cond = substitute_for_best_equal_field(thd, where_cond, cond_equal, map2table); if (thd->is_error()) { error = 1; DBUG_PRINT("error", ("Error from substitute_for_best_equal")); return true; } where_cond->update_used_tables(); DBUG_EXECUTE("where", print_where(thd, where_cond, "after substitute_best_equal", QT_ORDINARY);); } /* Perform the same optimization on field evaluation for all join conditions. */ for (uint i = const_tables; i < tables; ++i) { JOIN_TAB *const tab = best_ref[i]; if (tab->position() && tab->join_cond()) { tab->set_join_cond(substitute_for_best_equal_field( thd, tab->join_cond(), tab->cond_equal, map2table)); if (thd->is_error()) { error = 1; DBUG_PRINT("error", ("Error from substitute_for_best_equal")); return true; } tab->join_cond()->update_used_tables(); if (tab->join_cond()) tab->join_cond()->walk(&Item::cast_incompatible_args, enum_walk::POSTFIX, nullptr); } } if (init_ref_access()) { error = 1; DBUG_PRINT("error", ("Error from init_ref_access")); return true; } // Update table dependencies after assigning ref access fields update_depend_map(); THD_STAGE_INFO(thd, stage_preparing); if (make_join_select(this, where_cond)) { if (thd->is_error()) return true; zero_result_cause = "Impossible WHERE noticed after reading const tables"; create_iterators_for_zero_rows(); goto setup_subq_exit; } if (select_lex->query_result()->optimize()) { DBUG_PRINT("error", ("Error: Query_result::optimize() failed")); return true; // error == -1 } // Inject cast nodes into the WHERE conditions if (where_cond) where_cond->walk(&Item::cast_incompatible_args, enum_walk::POSTFIX, nullptr); error = -1; /* if goto err */ if (optimize_distinct_group_order()) return true; if ((select_lex->active_options() & SELECT_NO_JOIN_CACHE) || select_lex->ftfunc_list->elements) no_jbuf_after = 0; // If we _must_ use the iterator executor, turn off BNL/BKA, since those // are the only features that it can't deal with now. // See SELECT_LEX::find_common_table_expr() for more information. if (select_lex->parent_lex->force_iterator_executor) no_jbuf_after = 0; /* Perform FULLTEXT search before all regular searches */ if (select_lex->has_ft_funcs() && optimize_fts_query()) return true; /* By setting child_subquery_can_materialize so late we gain the following: JOIN::compare_costs_of_subquery_strategies() can test this variable to know if we are have finished evaluating constant conditions, which itself helps determining fanouts. */ child_subquery_can_materialize = true; /* It's necessary to check const part of HAVING cond as there is a chance that some cond parts may become const items after make_join_plan() (for example when Item is a reference to const table field from outer join). This check is performed only for those conditions which do not use aggregate functions. In such case temporary table may not be used and const condition elements may be lost during further having condition transformation in JOIN::exec. */ if (having_cond && !having_cond->has_aggregation() && (const_tables > 0)) { having_cond->update_used_tables(); if (remove_eq_conds(thd, having_cond, &having_cond, &select_lex->having_value)) { error = 1; DBUG_PRINT("error", ("Error from remove_eq_conds")); return true; } if (select_lex->having_value == Item::COND_FALSE) { having_cond = new Item_func_false(); zero_result_cause = "Impossible HAVING noticed after reading const tables"; create_iterators_for_zero_rows(); goto setup_subq_exit; } } // Inject cast nodes into the HAVING conditions if (having_cond) having_cond->walk(&Item::cast_incompatible_args, enum_walk::POSTFIX, nullptr); // Traverse the expressions and inject cast nodes to compatible data types, // if needed. { List_iterator select_expression_it(all_fields); Item *item; while ((item = select_expression_it++)) item->walk(&Item::cast_incompatible_args, enum_walk::POSTFIX, nullptr); } if (rollup.state != ROLLUP::STATE_NONE) { if (rollup_process_const_fields()) { DBUG_PRINT("error", ("Error: rollup_process_fields() failed")); return true; } /* Fields may have been replaced by Item_func_rollup_const, so recalculate the number of fields and functions for this query block. */ // JOIN::optimize_rollup() may set allow_group_via_temp_table = false, // and we must not undo that. const bool save_allow_group_via_temp_table = tmp_table_param.allow_group_via_temp_table; count_field_types(select_lex, &tmp_table_param, all_fields, false, false); tmp_table_param.allow_group_via_temp_table = save_allow_group_via_temp_table; } // See if this subquery can be evaluated with subselect_indexsubquery_engine if (const int ret = replace_index_subquery()) { set_plan_state(PLAN_READY); /* We leave optimize() because the rest of it is only about order/group which those subqueries don't have and about setting up plan which we're not going to use due to different execution method. */ return ret < 0; } { /* If the hint FORCE INDEX FOR ORDER BY/GROUP BY is used for the first table (it does not make sense for other tables) then we cannot do join buffering. */ if (!plan_is_const()) { const TABLE *const first = best_ref[const_tables]->table(); if ((first->force_index_order && order) || (first->force_index_group && group_list)) no_jbuf_after = 0; } bool simple_sort = true; Deps_of_remaining_lateral_derived_tables deps_lateral(this, all_table_map); // Check whether join cache could be used for (uint i = const_tables; i < tables; i++) { JOIN_TAB *const tab = best_ref[i]; if (!tab->position()) continue; if (setup_join_buffering(tab, this, no_jbuf_after)) return true; if (tab->use_join_cache() != JOIN_CACHE::ALG_NONE) simple_sort = false; DBUG_ASSERT(tab->type() != JT_FT || tab->use_join_cache() == JOIN_CACHE::ALG_NONE); deps_lateral.recalculate(tab, i + 1); } if (!simple_sort) { /* A join buffer is used for this table. We here inform the optimizer that it should not rely on rows of the first non-const table being in order thanks to an index scan; indeed join buffering of the present table subsequently changes the order of rows. */ simple_order = simple_group = false; } } if (!plan_is_const() && order) { /* Force using of tmp table if sorting by a SP or UDF function due to their expensive and probably non-deterministic nature. */ for (ORDER *tmp_order = order; tmp_order; tmp_order = tmp_order->next) { Item *item = *tmp_order->item; if (item->is_expensive()) { /* Force tmp table without sort */ simple_order = simple_group = false; break; } } } /* Check if we need to create a temporary table prior to any windowing. (1) If there is ROLLUP, which happens before DISTINCT, windowing and ORDER BY, any of those clauses needs the result of ROLLUP in a tmp table. Rows which ROLLUP adds to the result are visible only to DISTINCT, windowing and ORDER BY which we handled above. So for the rest of conditions ((2), etc), we can do as if there were no ROLLUP. (2) If all tables are constant, the query's result is guaranteed to have 0 or 1 row only, so all SQL clauses discussed below (DISTINCT, ORDER BY, GROUP BY, windowing, SQL_BUFFER_RESULT) are useless and need no tmp table. (3) If there is GROUP BY which isn't resolved by using an index or sorting the first table, we need a tmp table to compute the grouped rows. GROUP BY happens before windowing; so it is a pre-windowing tmp table. (4) (5) If there is DISTINCT, or ORDER BY which isn't resolved by using an index or sorting the first table, those clauses need an input tmp table. If we have windowing, as those clauses are used after windowing, they can use the last window's tmp table. (6) If there are different ORDER BY and GROUP BY orders, ORDER BY needs an input tmp table, so it's like (5). (7) If the user wants us to buffer the result, we need a tmp table. But windowing creates one anyway, and so does the materialization of a derived table. See also the computation of Temp_table_param::m_window_short_circuit, where we make sure to create a tmp table if the clauses above want one. (8) If the first windowing step needs sorting, filesort() will be used; it can sort one table but not a join of tables, so we need a tmp table then. If GROUP BY was optimized away, the pre-windowing result is 0 or 1 row so doesn't need sorting. */ if (rollup.state != ROLLUP::STATE_NONE && // (1) (select_distinct || has_windows || order)) need_tmp_before_win = true; if (!plan_is_const()) // (2) { if ((group_list && !simple_group) || // (3) (!has_windows && (select_distinct || // (4) (order && !simple_order) || // (5) (group_list && order))) || // (6) ((select_lex->active_options() & OPTION_BUFFER_RESULT) && !has_windows && !(unit->derived_table && unit->derived_table->uses_materialization())) || // (7) (has_windows && (primary_tables - const_tables) > 1 && // (8) m_windows[0]->needs_sorting() && !group_optimized_away)) need_tmp_before_win = true; } DBUG_EXECUTE("info", TEST_join(this);); if (!plan_is_const()) { // Test if we can use an index instead of sorting test_skip_sort(); if (finalize_table_conditions()) return true; } if (alloc_qep(tables)) return (error = 1); /* purecov: inspected */ if (make_join_readinfo(this, no_jbuf_after)) return true; /* purecov: inspected */ if (make_tmp_tables_info()) return true; /* At this stage, we have fully set QEP_TABs; JOIN_TABs are unaccessible, Query parts being offloaded to the engines(below) may still change the 'plan', affecting which type of Iterator we should create. Thus no Iterators should be set up until after push_to_engine() has completed. */ if (push_to_engines()) return true; /* If we decided to not sort after all, update the cost of the JOIN. Windowing sorts are handled elsewhere */ if (sort_cost > 0.0 && !explain_flags.any(ESP_USING_FILESORT, ESC_WINDOWING)) { best_read -= sort_cost; sort_cost = 0.0; } count_field_types(select_lex, &tmp_table_param, all_fields, false, false); // Create the basic table Iterators, and composite Iterators where supported. create_iterators(); // Make plan visible for EXPLAIN set_plan_state(PLAN_READY); DEBUG_SYNC(thd, "after_join_optimize"); error = 0; return false; setup_subq_exit: DBUG_ASSERT(zero_result_cause != nullptr); DBUG_ASSERT(m_root_iterator != nullptr); /* Even with zero matching rows, subqueries in the HAVING clause may need to be evaluated if there are aggregate functions in the query. If this JOIN is part of an outer query, subqueries in HAVING may be evaluated several times in total; so subquery materialization makes sense. */ child_subquery_can_materialize = true; trace_steps.end(); // because all steps are done Opt_trace_object(trace, "empty_result").add_alnum("cause", zero_result_cause); having_for_explain = having_cond; error = 0; if (!qep_tab && best_ref) { /* After creation of JOIN_TABs in make_join_plan(), we have shortcut due to some zero_result_cause. For simplification, if we have JOIN_TABs we want QEP_TABs too. */ if (alloc_qep(tables)) return true; /* purecov: inspected */ unplug_join_tabs(); } set_plan_state(ZERO_RESULT); return false; } void JOIN::create_iterators_for_zero_rows() { if (send_row_on_empty_set()) { // Aggregate no rows into an aggregate row. m_root_iterator = NewIterator( thd, zero_result_cause, select_lex->join, &examined_rows); m_root_iterator = attach_iterators_for_having_and_limit(move(m_root_iterator)); } else { // Send no row at all (so also no need to check HAVING or LIMIT). m_root_iterator = NewIterator(thd, zero_result_cause); } } /** Push (parts of) the query execution down to the storage engines if they can provide faster execution of the query, or part of it. @return 1 in case of error, 0 otherwise. */ int JOIN::push_to_engines() { DBUG_TRACE; /* Push joins to handlerton(s) The handlerton(s) will inspect the QEP through the AQP (Abstract Query Plan) and extract from it whatever it might implement of pushed execution. It is the responsibility of the handler: - to store any information it need for later execution of pushed queries. - to call appropriate AQP functions which modifies the QEP to use the special 'linked' read functions for those parts of the join which have been pushed. Currently pushed joins are only implemented by NDB. It only make sense to try pushing if > 1 non-const tables. */ if (!plan_is_single_table() && !plan_is_const()) { const AQP::Join_plan plan(this); if (ha_make_pushed_joins(thd, &plan)) return 1; } /* If enabled by optimizer settings, and implemented by handler, (parts of) the table condition may be pushed down to the SE-engine for evaluation. */ if (thd->optimizer_switch_flag(OPTIMIZER_SWITCH_ENGINE_CONDITION_PUSHDOWN)) { for (uint i = const_tables; i < tables; i++) { const join_type jt = qep_tab[i].type(); if ((jt == JT_EQ_REF || jt == JT_CONST || jt == JT_SYSTEM) && !qep_tab[i].table()->file->member_of_pushed_join()) { /* It is of limited value to push a condition to a single row access method, so we skip cond_push() for these. The exception is if we are member of a pushed join, where execution of entire join branches may be eliminated. */ continue; } const Item *cond = qep_tab[i].condition(); if (cond != nullptr) { const bool using_join_cache = (qep_tab[i].op != nullptr && qep_tab[i].op->type() == QEP_operation::OT_CACHE); /* If a join cache is referred by this table, there is not a single specific row from the 'other tables' to compare rows from this table against. Thus, other tables can not be referred in this case. */ const bool other_tbls_ok = !using_join_cache && thd->lex->sql_command != SQLCOM_UPDATE_MULTI && thd->lex->sql_command != SQLCOM_DELETE_MULTI; const Item *remainder = qep_tab[i].table()->file->cond_push(cond, other_tbls_ok); qep_tab[i].set_condition(const_cast(remainder)); } } } return 0; } /** Substitute all expressions in the WHERE condition and ORDER/GROUP lists that match generated columns (GC) expressions with GC fields, if any. @details This function does 3 things: 1) Creates list of all GC fields that are a part of a key and the GC expression is a function. All query tables are scanned. If there's no such fields, function exits. 2) By means of Item::compile() WHERE clause is transformed. @see Item_func::gc_subst_transformer() for details. 3) If there's ORDER/GROUP BY clauses, this function tries to substitute expressions in these lists with GC too. It removes from the list of indexed GC all elements which index blocked by hints. This is done to reduce amount of further work. Next it goes through ORDER/GROUP BY list and matches the expression in it against GC expressions in indexed GC list. When a match is found, the expression is replaced with a new Item_field for the matched GC field. Also, this new field is added to the hidden part of all_fields list. @param thd thread handle @param select_lex the current select @param where_cond the WHERE condition, possibly NULL @param group_list the GROUP BY clause, possibly NULL @param order the ORDER BY clause, possibly NULL @return true if the GROUP BY clause or the ORDER BY clause was changed, false otherwise */ bool substitute_gc(THD *thd, SELECT_LEX *select_lex, Item *where_cond, ORDER *group_list, ORDER *order) { List indexed_gc; Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object subst_gc(trace, "substitute_generated_columns"); // Collect all GCs that are a part of a key for (TABLE_LIST *tl = select_lex->leaf_tables; tl; tl = tl->next_leaf) { if (tl->table->s->keys == 0) continue; for (uint i = 0; i < tl->table->s->fields; i++) { Field *fld = tl->table->field[i]; if (fld->is_gcol() && !(fld->part_of_key.is_clear_all() && fld->part_of_prefixkey.is_clear_all()) && fld->gcol_info->expr_item->can_be_substituted_for_gc()) { // Don't check allowed keys here as conditions/group/order use // different keymaps for that. indexed_gc.push_back(fld); } } } // No GC in the tables used in the query if (indexed_gc.elements == 0) return false; if (where_cond) { // Item_func::compile will dereference this pointer, provide valid value. uchar i, *dummy = &i; where_cond->compile(&Item::gc_subst_analyzer, &dummy, &Item::gc_subst_transformer, (uchar *)&indexed_gc); subst_gc.add("resulting_condition", where_cond); } // An error occur during substitution. Let caller handle it. if (thd->is_error()) return false; if (!(group_list || order)) return false; // Filter out GCs that do not have index usable for GROUP/ORDER Field *gc; List_iterator li(indexed_gc); while ((gc = li++)) { Key_map tkm = gc->part_of_key; tkm.intersect(group_list ? gc->table->keys_in_use_for_group_by : gc->table->keys_in_use_for_order_by); if (tkm.is_clear_all()) li.remove(); } if (!indexed_gc.elements) return false; // Index could be used for ORDER only if there is no GROUP ORDER *list = group_list ? group_list : order; bool changed = false; for (ORDER *ord = list; ord; ord = ord->next) { li.rewind(); if (!(*ord->item)->can_be_substituted_for_gc()) continue; while ((gc = li++)) { Item_func *tmp = pointer_cast(*ord->item); Item_field *field; if ((field = get_gc_for_expr(&tmp, gc, gc->result_type()))) { changed = true; /* Add new field to field list. */ ord->item = select_lex->add_hidden_item(field); break; } } } // An error occur during substitution. Let caller handle it. if (thd->is_error()) return false; if (changed && trace->is_started()) { String str; SELECT_LEX::print_order( thd, &str, list, enum_query_type(QT_TO_SYSTEM_CHARSET | QT_SHOW_SELECT_NUMBER | QT_NO_DEFAULT_DB)); subst_gc.add_utf8(group_list ? "resulting_GROUP_BY" : "resulting_ORDER_BY", str.ptr(), str.length()); } return changed; } /** Sets the plan's state of the JOIN. This is always the final step of optimization; starting from this call, we expose the plan to other connections (via EXPLAIN CONNECTION) so the plan has to be final. QEP_TAB's quick_optim, condition_optim and keyread_optim are set here. */ void JOIN::set_plan_state(enum_plan_state plan_state_arg) { // A plan should not change to another plan: DBUG_ASSERT(plan_state_arg == NO_PLAN || plan_state == NO_PLAN); if (plan_state == NO_PLAN && plan_state_arg != NO_PLAN) { if (qep_tab != NULL) { /* We want to cover primary tables, tmp tables (they may have a sort, so their "quick" and "condition" may change when execution runs the sort), and sj-mat inner tables. Note that make_tmp_tables_info() may have added a sort to the first non-const primary table, so it's important to do those assignments after make_tmp_tables_info(). */ for (uint i = const_tables; i < tables; ++i) { qep_tab[i].set_quick_optim(); qep_tab[i].set_condition_optim(); qep_tab[i].set_keyread_optim(); } } } DEBUG_SYNC(thd, "before_set_plan"); // If SQLCOM_END, no thread is explaining our statement anymore. const bool need_lock = thd->query_plan.get_command() != SQLCOM_END; if (need_lock) thd->lock_query_plan(); plan_state = plan_state_arg; if (need_lock) thd->unlock_query_plan(); } bool JOIN::alloc_qep(uint n) { static_assert(MAX_TABLES <= INT_MAX8, "plan_idx needs to be wide enough."); ASSERT_BEST_REF_IN_JOIN_ORDER(this); qep_tab = new (thd->mem_root) QEP_TAB[n]; if (!qep_tab) return true; /* purecov: inspected */ for (uint i = 0; i < n; ++i) qep_tab[i].init(best_ref[i]); return false; } void QEP_TAB::init(JOIN_TAB *jt) { jt->share_qs(this); set_table(table()); // to update table()->reginfo.qep_tab table_ref = jt->table_ref; } /// @returns semijoin strategy for this table. uint QEP_TAB::get_sj_strategy() const { if (first_sj_inner() == NO_PLAN_IDX) return SJ_OPT_NONE; const uint s = join()->qep_tab[first_sj_inner()].position()->sj_strategy; DBUG_ASSERT(s != SJ_OPT_NONE); return s; } /** Return the index used for a table in a QEP The various access methods have different places where the index/key number is stored, so this function is needed to return the correct value. @returns index number, or MAX_KEY if not applicable. JT_SYSTEM and JT_ALL does not use an index, and will always return MAX_KEY. JT_INDEX_MERGE supports more than one index. Hence MAX_KEY is returned and a further inspection is needed. */ uint QEP_TAB::effective_index() const { switch (type()) { case JT_SYSTEM: DBUG_ASSERT(ref().key == -1); return MAX_KEY; case JT_CONST: case JT_EQ_REF: case JT_REF_OR_NULL: case JT_REF: DBUG_ASSERT(ref().key != -1); return uint(ref().key); case JT_INDEX_SCAN: case JT_FT: return index(); case JT_INDEX_MERGE: DBUG_ASSERT(quick()->index == MAX_KEY); return MAX_KEY; case JT_RANGE: return quick()->index; case JT_ALL: default: // @todo Check why JT_UNKNOWN is a valid value here. DBUG_ASSERT(type() == JT_ALL || type() == JT_UNKNOWN); return MAX_KEY; } } uint JOIN_TAB::get_sj_strategy() const { if (first_sj_inner() == NO_PLAN_IDX) return SJ_OPT_NONE; ASSERT_BEST_REF_IN_JOIN_ORDER(join()); JOIN_TAB *tab = join()->best_ref[first_sj_inner()]; uint s = tab->position()->sj_strategy; DBUG_ASSERT(s != SJ_OPT_NONE); return s; } int JOIN::replace_index_subquery() { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(this); if (group_list || !(unit->item && unit->item->substype() == Item_subselect::IN_SUBS) || primary_tables != 1 || !where_cond || unit->is_union()) return 0; // Guaranteed by remove_redundant_subquery_clauses(): DBUG_ASSERT(order == NULL && !select_distinct); subselect_engine *engine = NULL; Item_in_subselect *const in_subs = static_cast(unit->item); bool found_engine = false; JOIN_TAB *const first_join_tab = best_ref[0]; if (in_subs->exec_method == Item_exists_subselect::EXEC_MATERIALIZATION) { // We cannot have two engines at the same time } else if (first_join_tab->table_ref->is_view_or_derived() && first_join_tab->table_ref->derived_unit()->is_recursive()) { // The index subquery engine, which runs the derived table machinery // from the old executor, is not capable of materializing a WITH RECURSIVE // query from the iterator executor. Thus, be conservative here, so that the // case never happens. } else if (having_cond == NULL) { const join_type type = first_join_tab->type(); if ((type == JT_EQ_REF || type == JT_REF) && first_join_tab->ref().items[0]->item_name.ptr() == in_left_expr_name) { found_engine = true; } } else if (first_join_tab->type() == JT_REF_OR_NULL && first_join_tab->ref().items[0]->item_name.ptr() == in_left_expr_name && having_cond->created_by_in2exists()) { found_engine = true; } if (!found_engine) return 0; /* Remove redundant predicates and cache constant expressions */ if (finalize_table_conditions()) return -1; if (alloc_qep(tables)) return -1; /* purecov: inspected */ unplug_join_tabs(); error = 0; QEP_TAB *const first_qep_tab = &qep_tab[0]; if (first_qep_tab->table()->covering_keys.is_set(first_qep_tab->ref().key)) { DBUG_ASSERT(!first_qep_tab->table()->no_keyread); first_qep_tab->table()->set_keyread(true); } engine = new (thd->mem_root) subselect_indexsubquery_engine( first_qep_tab, unit->item, first_qep_tab->condition(), having_cond); if (!unit->item->change_engine(engine)) return 1; else // error: return -1; /* purecov: inspected */ } bool JOIN::optimize_distinct_group_order() { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(this); const bool windowing = m_windows.elements > 0; const bool may_trace = select_distinct || group_list || order || windowing || tmp_table_param.sum_func_count; Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_disable_I_S trace_disabled(trace, !may_trace); Opt_trace_object wrapper(trace); Opt_trace_object trace_opt(trace, "optimizing_distinct_group_by_order_by"); /* Optimize distinct away if possible */ { ORDER *org_order = order; order = ORDER_with_src( remove_const(order, where_cond, rollup.state == ROLLUP::STATE_NONE, &simple_order, false), order.src); if (thd->is_error()) { error = 1; DBUG_PRINT("error", ("Error from remove_const")); return true; } /* If we are using ORDER BY NULL or ORDER BY const_expression, return result in any order (even if we are using a GROUP BY) */ if (!order && org_order) skip_sort_order = 1; } /* Check if we can optimize away GROUP BY/DISTINCT. We can do that if there are no aggregate functions, the fields in DISTINCT clause (if present) and/or columns in GROUP BY (if present) contain direct references to all key parts of an unique index (in whatever order) and if the key parts of the unique index cannot contain NULLs. Note that the unique keys for DISTINCT and GROUP BY should not be the same (as long as they are unique). The FROM clause must contain a single non-constant table. @todo Apart from the LIS test, every condition depends only on facts which can be known in SELECT_LEX::prepare(), possibly this block should move there. */ JOIN_TAB *const tab = best_ref[const_tables]; if (plan_is_single_table() && (group_list || select_distinct) && !tmp_table_param.sum_func_count && (!tab->quick() || tab->quick()->get_type() != QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX)) { if (group_list && rollup.state == ROLLUP::STATE_NONE && list_contains_unique_index(tab, find_field_in_order_list, (void *)group_list)) { /* We have found that grouping can be removed since groups correspond to only one row anyway. */ group_list = 0; grouped = false; } if (select_distinct && list_contains_unique_index(tab, find_field_in_item_list, (void *)&fields_list)) { select_distinct = 0; trace_opt.add("distinct_is_on_unique", true) .add("removed_distinct", true); } } if (!(group_list || tmp_table_param.sum_func_count || windowing) && select_distinct && plan_is_single_table() && rollup.state == ROLLUP::STATE_NONE) { int order_idx = -1, group_idx = -1; /* We are only using one table. In this case we change DISTINCT to a GROUP BY query if: - The GROUP BY can be done through indexes (no sort) and the ORDER BY only uses selected fields. (In this case we can later optimize away GROUP BY and ORDER BY) - We are scanning the whole table without LIMIT This can happen if: - We are using CALC_FOUND_ROWS - We are using an ORDER BY that can't be optimized away. - Selected expressions are not set functions (those cannot be put into GROUP BY). We don't want to use this optimization when we are using LIMIT because in this case we can just create a temporary table that holds LIMIT rows and stop when this table is full. */ if (order) { skip_sort_order = test_if_skip_sort_order( tab, order, m_select_limit, true, // no_changes &tab->table()->keys_in_use_for_order_by, &order_idx); count_field_types(select_lex, &tmp_table_param, all_fields, false, false); } ORDER *o; bool all_order_fields_used; if ((o = create_order_from_distinct(thd, ref_items[REF_SLICE_ACTIVE], order, fields_list, /*skip_aggregates=*/true, /*convert_bit_fields_to_long=*/true, &all_order_fields_used))) { group_list = ORDER_with_src(o, ESC_DISTINCT); const bool skip_group = skip_sort_order && test_if_skip_sort_order(tab, group_list, m_select_limit, true, // no_changes &tab->table()->keys_in_use_for_group_by, &group_idx); count_field_types(select_lex, &tmp_table_param, all_fields, false, false); // ORDER BY and GROUP BY are using different indexes, can't skip sorting if (group_idx >= 0 && order_idx >= 0 && group_idx != order_idx) skip_sort_order = false; if ((skip_group && all_order_fields_used) || m_select_limit == HA_POS_ERROR || (order && !skip_sort_order)) { /* Change DISTINCT to GROUP BY */ select_distinct = 0; /* group_list was created with ORDER BY clause as prefix and replaces it. So it must respect ordering. If there is no ORDER BY, GROUP BY need not have to provide order. */ if (!order) { for (ORDER *group = group_list; group; group = group->next) group->direction = ORDER_NOT_RELEVANT; } if (all_order_fields_used && skip_sort_order && order) { /* Force MySQL to read the table in sorted order to get result in ORDER BY order. */ tmp_table_param.allow_group_via_temp_table = false; } grouped = true; // For end_write_group trace_opt.add("changed_distinct_to_group_by", true); } else group_list = 0; } else if (thd->is_fatal_error()) // End of memory return true; } simple_group = 0; ORDER *old_group_list = group_list; group_list = ORDER_with_src( remove_const(group_list, where_cond, rollup.state == ROLLUP::STATE_NONE, &simple_group, true), group_list.src); if (thd->is_error()) { error = 1; DBUG_PRINT("error", ("Error from remove_const")); return true; } if (old_group_list && !group_list) select_distinct = 0; if (!group_list && grouped) { order = 0; // The output has only one row simple_order = 1; select_distinct = 0; // No need in distinct for 1 row group_optimized_away = 1; } calc_group_buffer(this, group_list); send_group_parts = tmp_table_param.group_parts; /* Save org parts */ /* If ORDER BY is a prefix of GROUP BY and if windowing or ROLLUP doesn't change this order, ORDER BY can be removed and we can enforce GROUP BY to provide order. Also true if the result is one row. */ if ((test_if_subpart(group_list, order) && !m_windows_sort && select_lex->olap != ROLLUP_TYPE) || (!group_list && tmp_table_param.sum_func_count)) { if (order) { order = 0; trace_opt.add("removed_order_by", true); } if (is_indexed_agg_distinct(this, NULL)) streaming_aggregation = false; } return false; } void JOIN::test_skip_sort() { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(this); JOIN_TAB *const tab = best_ref[const_tables]; DBUG_ASSERT(m_ordered_index_usage == ORDERED_INDEX_VOID); if (group_list) // GROUP BY honoured first // (DISTINCT was rewritten to GROUP BY if skippable) { /* When there is SQL_BIG_RESULT or a JSON aggregation function, do not sort using index for GROUP BY, and thus force sorting on disk unless a group min-max optimization is going to be used as it is applied now only for one table queries with covering indexes. */ if (!(select_lex->active_options() & SELECT_BIG_RESULT || with_json_agg) || (tab->quick() && tab->quick()->get_type() == QUICK_SELECT_I::QS_TYPE_GROUP_MIN_MAX)) { if (simple_group && // GROUP BY is possibly skippable !select_distinct) // .. if not preceded by a DISTINCT { /* Calculate a possible 'limit' of table rows for 'GROUP BY': A specified 'LIMIT' is relative to the final resultset. 'need_tmp' implies that there will be more postprocessing so the specified 'limit' should not be enforced yet. */ const ha_rows limit = (need_tmp_before_win ? HA_POS_ERROR : m_select_limit); int dummy; if (test_if_skip_sort_order(tab, group_list, limit, false, &tab->table()->keys_in_use_for_group_by, &dummy)) { m_ordered_index_usage = ORDERED_INDEX_GROUP_BY; } } /* If we are going to use semi-join LooseScan, it will depend on the selected index scan to be used. If index is not used for the GROUP BY, we risk that sorting is put on the LooseScan table. In order to avoid this, force use of temporary table. TODO: Explain the allow_group_via_temp_table part of the test below. */ if ((m_ordered_index_usage != ORDERED_INDEX_GROUP_BY) && (tmp_table_param.allow_group_via_temp_table || (tab->emb_sj_nest && tab->position()->sj_strategy == SJ_OPT_LOOSE_SCAN))) { need_tmp_before_win = true; simple_order = simple_group = false; // Force tmp table without sort } } } else if (order && // ORDER BY wo/ preceding GROUP BY (simple_order || skip_sort_order) && // which is possibly skippable, !m_windows_sort) // and WFs will not shuffle rows { int dummy; if ((skip_sort_order = test_if_skip_sort_order( tab, order, m_select_limit, false, &tab->table()->keys_in_use_for_order_by, &dummy))) { m_ordered_index_usage = ORDERED_INDEX_ORDER_BY; } } } /** Test if ORDER BY is a single MATCH function(ORDER BY MATCH) and sort order is descending. @param order pointer to ORDER struct. @retval Pointer to MATCH function if order is 'ORDER BY MATCH() DESC' @retval NULL otherwise */ static Item_func_match *test_if_ft_index_order(ORDER *order) { if (order && order->next == NULL && order->direction == ORDER_DESC && (*order->item)->type() == Item::FUNC_ITEM && down_cast(*order->item)->functype() == Item_func::FT_FUNC) return down_cast(*order->item)->get_master(); return NULL; } /** Test if one can use the key to resolve ordering. @param order_src Sort order @param table Table to sort @param idx Index to check @param[out] used_key_parts NULL by default, otherwise return value for used key parts. @param[out] skip_quick Whether found index can be used for backward range scans @note used_key_parts is set to correct key parts used if return value != 0 (On other cases, used_key_part may be changed) Note that the value may actually be greater than the number of index key parts. This can happen for storage engines that have the primary key parts as a suffix for every secondary key. @retval 1 key is ok. @retval 0 Key can't be used @retval -1 Reverse key can be used */ int test_if_order_by_key(ORDER_with_src *order_src, TABLE *table, uint idx, uint *used_key_parts, bool *skip_quick) { DBUG_TRACE; KEY_PART_INFO *key_part, *key_part_end; key_part = table->key_info[idx].key_part; key_part_end = key_part + table->key_info[idx].user_defined_key_parts; key_part_map const_key_parts = table->const_key_parts[idx]; int reverse = 0; uint key_parts; bool on_pk_suffix = false; // Whether [extented] key has key parts with mixed ASC/DESC order bool mixed_order = false; // Order direction of the first key part bool reverse_sorted = (bool)(key_part->key_part_flag & HA_REVERSE_SORT); ORDER *order = *order_src; *skip_quick = false; for (; order; order = order->next, const_key_parts >>= 1) { /* Since only fields can be indexed, ORDER BY that is not a field cannot be resolved by using an index. */ Item *real_itm = (*order->item)->real_item(); if (real_itm->type() != Item::FIELD_ITEM) return 0; Field *field = static_cast(real_itm)->field; /* Skip key parts that are constants in the WHERE clause. These are already skipped in the ORDER BY by const_expression_in_where() */ for (; const_key_parts & 1 && key_part < key_part_end; const_key_parts >>= 1) key_part++; if (key_part == key_part_end) { /* We are at the end of the key. Check if the engine has the primary key as a suffix to the secondary keys. If it has continue to check the primary key as a suffix. */ if (!on_pk_suffix && (table->file->ha_table_flags() & HA_PRIMARY_KEY_IN_READ_INDEX) && table->s->primary_key != MAX_KEY && table->s->primary_key != idx) { on_pk_suffix = true; key_part = table->key_info[table->s->primary_key].key_part; key_part_end = key_part + table->key_info[table->s->primary_key].user_defined_key_parts; const_key_parts = table->const_key_parts[table->s->primary_key]; for (; const_key_parts & 1 && key_part < key_part_end; const_key_parts >>= 1) key_part++; /* The primary and secondary key parts were all const (i.e. there's one row). The sorting doesn't matter. */ if (key_part == key_part_end && reverse == 0) { key_parts = 0; reverse = 1; goto ok; } } else return 0; } if (key_part->field != field || !field->part_of_sortkey.is_set(idx)) return 0; if (order->direction != ORDER_NOT_RELEVANT) { const enum_order keypart_order = (key_part->key_part_flag & HA_REVERSE_SORT) ? ORDER_DESC : ORDER_ASC; /* set flag to 1 if we can use read-next on key, else to -1 */ int cur_scan_dir = (order->direction == keypart_order) ? 1 : -1; if (reverse && cur_scan_dir != reverse) return 0; reverse = cur_scan_dir; // Remember if reverse } mixed_order |= (reverse_sorted != (bool)((key_part)->key_part_flag & HA_REVERSE_SORT)); key_part++; } /* The index picked here might be used for range scans with multiple ranges. This will require tricky reordering in case of ranges would have to be scanned backward and index consists of mixed ASC/DESC key parts. Due to that backward scans on such indexes are disabled. */ if (mixed_order && reverse < 0) *skip_quick = true; if (!reverse) { /* We get here when the key is suitable and we don't care about it's order, i.e. GROUP BY/DISTINCT. Use forward scan. */ reverse = 1; } if (on_pk_suffix) { uint used_key_parts_secondary = table->key_info[idx].user_defined_key_parts; uint used_key_parts_pk = (uint)(key_part - table->key_info[table->s->primary_key].key_part); key_parts = used_key_parts_pk + used_key_parts_secondary; if (reverse == -1 && (!(table->file->index_flags(idx, used_key_parts_secondary - 1, 1) & HA_READ_PREV) || !(table->file->index_flags(table->s->primary_key, used_key_parts_pk - 1, 1) & HA_READ_PREV))) reverse = 0; // Index can't be used } else { key_parts = (uint)(key_part - table->key_info[idx].key_part); if (reverse == -1 && !(table->file->index_flags(idx, key_parts - 1, 1) & HA_READ_PREV)) reverse = 0; // Index can't be used } ok: if (used_key_parts != NULL) *used_key_parts = key_parts; return reverse; } /** Find shortest key suitable for full table scan. @param table Table to scan @param usable_keys Allowed keys @note As far as 1) clustered primary key entry data set is a set of all record fields (key fields and not key fields) and 2) secondary index entry data is a union of its key fields and primary key fields (at least InnoDB and its derivatives don't duplicate primary key fields there, even if the primary and the secondary keys have a common subset of key fields), then secondary index entry data is always a subset of primary key entry. Unfortunately, key_info[nr].key_length doesn't show the length of key/pointer pair but a sum of key field lengths only, thus we can't estimate index IO volume comparing only this key_length value of secondary keys and clustered PK. So, try secondary keys first, and choose PK only if there are no usable secondary covering keys or found best secondary key include all table fields (i.e. same as PK): @return MAX_KEY no suitable key found key index otherwise */ uint find_shortest_key(TABLE *table, const Key_map *usable_keys) { uint best = MAX_KEY; uint usable_clustered_pk = (table->file->primary_key_is_clustered() && table->s->primary_key != MAX_KEY && usable_keys->is_set(table->s->primary_key)) ? table->s->primary_key : MAX_KEY; if (!usable_keys->is_clear_all()) { uint min_length = (uint)~0; for (uint nr = 0; nr < table->s->keys; nr++) { if (nr == usable_clustered_pk) continue; if (usable_keys->is_set(nr)) { /* Can not do full index scan on rtree index because it is not supported by Innodb, probably not supported by others either. A multi-valued key requires unique filter, and won't be the most fast option even if it will be the shortest one. */ const KEY &key_ref = table->key_info[nr]; DBUG_ASSERT(!(key_ref.flags & HA_MULTI_VALUED_KEY)); if (key_ref.key_length < min_length && !(key_ref.flags & HA_SPATIAL)) { min_length = key_ref.key_length; best = nr; } } } } if (usable_clustered_pk != MAX_KEY) { /* If the primary key is clustered and found shorter key covers all table fields then primary key scan normally would be faster because amount of data to scan is the same but PK is clustered. It's safe to compare key parts with table fields since duplicate key parts aren't allowed. */ if (best == MAX_KEY || table->key_info[best].user_defined_key_parts >= table->s->fields) best = usable_clustered_pk; } return best; } /** Test if a second key is the subkey of the first one. @param key_part First key parts @param ref_key_part Second key parts @param ref_key_part_end Last+1 part of the second key @note Second key MUST be shorter than the first one. @retval 1 is a subkey @retval 0 no sub key */ inline bool is_subkey(KEY_PART_INFO *key_part, KEY_PART_INFO *ref_key_part, KEY_PART_INFO *ref_key_part_end) { for (; ref_key_part < ref_key_part_end; key_part++, ref_key_part++) if (!key_part->field->eq(ref_key_part->field)) return 0; return 1; } /** Test if REF_OR_NULL optimization will be used if the specified ref_key is used for REF-access to 'tab' @retval true JT_REF_OR_NULL will be used @retval false no JT_REF_OR_NULL access */ static bool is_ref_or_null_optimized(const JOIN_TAB *tab, uint ref_key) { if (tab->keyuse()) { const Key_use *keyuse = tab->keyuse(); while (keyuse->key != ref_key && keyuse->table_ref == tab->table_ref) keyuse++; const table_map const_tables = tab->join()->const_table_map; while (keyuse->key == ref_key && keyuse->table_ref == tab->table_ref) { if (!(keyuse->used_tables & ~const_tables)) { if (keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL) return true; } keyuse++; } } return false; } /** Test if we can use one of the 'usable_keys' instead of 'ref' key for sorting. @param order The query block's order clause. @param tab Current JOIN_TAB. @param ref Number of key, used for WHERE clause @param ref_key_parts Index columns used for ref lookup. @param usable_keys Keys for testing @return - MAX_KEY If we can't use other key - the number of found key Otherwise */ static uint test_if_subkey(ORDER_with_src *order, JOIN_TAB *tab, uint ref, uint ref_key_parts, const Key_map *usable_keys) { uint nr; uint min_length = (uint)~0; uint best = MAX_KEY; TABLE *table = tab->table(); KEY_PART_INFO *ref_key_part = table->key_info[ref].key_part; KEY_PART_INFO *ref_key_part_end = ref_key_part + ref_key_parts; for (nr = 0; nr < table->s->keys; nr++) { bool skip_quick; if (usable_keys->is_set(nr) && table->key_info[nr].key_length < min_length && table->key_info[nr].user_defined_key_parts >= ref_key_parts && is_subkey(table->key_info[nr].key_part, ref_key_part, ref_key_part_end) && !is_ref_or_null_optimized(tab, nr) && test_if_order_by_key(order, table, nr, NULL, &skip_quick) && !skip_quick) { min_length = table->key_info[nr].key_length; best = nr; } } return best; } /** It is not obvious to see that test_if_skip_sort_order() never changes the plan if no_changes is true. So we double-check: creating an instance of this class saves some important access-path-related information of the current table; when the instance is destroyed, the latest access-path information is compared with saved data. */ class Plan_change_watchdog { #ifndef DBUG_OFF public: /** @param tab_arg table whose access path is being determined @param no_changes_arg whether a change to the access path is allowed */ Plan_change_watchdog(const JOIN_TAB *tab_arg, const bool no_changes_arg) { // Only to keep gcc 4.1.2-44 silent about uninitialized variables quick = NULL; quick_index = 0; if (no_changes_arg) { tab = tab_arg; type = tab->type(); if ((quick = tab->quick())) quick_index = quick->index; use_quick = tab->use_quick; ref_key = tab->ref().key; ref_key_parts = tab->ref().key_parts; index = tab->index(); } else { tab = NULL; // Only to keep gcc 4.1.2-44 silent about uninitialized variables type = JT_UNKNOWN; quick = NULL; ref_key = ref_key_parts = index = 0; use_quick = QS_NONE; } } ~Plan_change_watchdog() { if (tab == NULL) return; // changes are not allowed, we verify: DBUG_ASSERT(tab->type() == type); DBUG_ASSERT(tab->quick() == quick); DBUG_ASSERT((quick == NULL) || tab->quick()->index == quick_index); DBUG_ASSERT(tab->use_quick == use_quick); DBUG_ASSERT(tab->ref().key == ref_key); DBUG_ASSERT(tab->ref().key_parts == ref_key_parts); DBUG_ASSERT(tab->index() == index); } private: const JOIN_TAB *tab; ///< table, or NULL if changes are allowed enum join_type type; ///< copy of tab->type() // "Range / index merge" info const QUICK_SELECT_I *quick; ///< copy of tab->select->quick uint quick_index; ///< copy of tab->select->quick->index enum quick_type use_quick; ///< copy of tab->use_quick // "ref access" info int ref_key; ///< copy of tab->ref().key uint ref_key_parts; /// copy of tab->ref().key_parts // Other index-related info uint index; ///< copy of tab->index #else // in non-debug build, empty class public: Plan_change_watchdog(const JOIN_TAB *, const bool) {} #endif }; /** Test if we can skip ordering by using an index. If the current plan is to use an index that provides ordering, the plan will not be changed. Otherwise, if an index can be used, the JOIN_TAB / tab->select struct is changed to use the index. The index must cover all fields in @, or it will not be considered. @param tab NULL or JOIN_TAB of the accessed table @param order Linked list of ORDER BY arguments @param select_limit LIMIT value, or HA_POS_ERROR if no limit @param no_changes No changes will be made to the query plan. @param map Key_map of applicable indexes. @param [out] order_idx Number of index selected, -1 if no applicable index found @todo - sergeyp: Results of all index merge selects actually are ordered by clustered PK values. @note This function may change tmp_table_param.precomputed_group_by. This affects how create_tmp_table() treats aggregation functions, so count_field_types() must be called again to make sure this is taken into consideration. @retval 0 We have to use filesort to do the sorting @retval 1 We can use an index. */ static bool test_if_skip_sort_order(JOIN_TAB *tab, ORDER_with_src &order, ha_rows select_limit, const bool no_changes, const Key_map *map, int *order_idx) { DBUG_TRACE; int ref_key; uint ref_key_parts = 0; int order_direction = 0; uint used_key_parts = 0; TABLE *const table = tab->table(); JOIN *const join = tab->join(); THD *const thd = join->thd; QUICK_SELECT_I *const save_quick = tab->quick(); int best_key = -1; bool set_up_ref_access_to_key = false; bool can_skip_sorting = false; // used as return value int changed_key = -1; /* Check that we are always called with first non-const table */ DBUG_ASSERT((uint)tab->idx() == join->const_tables); Plan_change_watchdog watchdog(tab, no_changes); *order_idx = -1; /* Sorting a single row can always be skipped */ if (tab->type() == JT_EQ_REF || tab->type() == JT_CONST || tab->type() == JT_SYSTEM) { return 1; } /* Check if FT index can be used to retrieve result in the required order. It is possible if ordering is on the first non-constant table. */ if (join->order && join->simple_order) { /* Check if ORDER is DESC, ORDER BY is a single MATCH function. */ Item_func_match *ft_func = test_if_ft_index_order(order); /* Two possible cases when we can skip sort order: 1. FT_SORTED must be set(Natural mode, no ORDER BY). 2. If FT_SORTED flag is not set then the engine should support deferred sorting. Deferred sorting means that sorting is postponed utill the start of index reading(InnoDB). In this case we set FT_SORTED flag here to let the engine know that internal sorting is needed. */ if (ft_func && ft_func->ft_handler && ft_func->ordered_result()) { /* FT index scan is used, so the only additional requirement is that ORDER BY MATCH function is the same as the function that is used for FT index. */ if (tab->type() == JT_FT && ft_func->eq(tab->position()->key->val, true)) { ft_func->set_hints(join, FT_SORTED, select_limit, false); return true; } /* No index is used, it's possible to use FT index for ORDER BY if LIMIT is present and does not exceed count of the records in FT index and there is no WHERE condition since a condition may potentially require more rows to be fetch from FT index. */ else if (!tab->condition() && select_limit != HA_POS_ERROR && select_limit <= ft_func->get_count()) { /* test_if_ft_index_order() always returns master MATCH function. */ DBUG_ASSERT(!ft_func->master); /* ref is not set since there is no WHERE condition */ DBUG_ASSERT(tab->ref().key == -1); /*Make EXPLAIN happy */ tab->set_type(JT_FT); tab->ref().key = ft_func->key; tab->ref().key_parts = 0; tab->set_index(ft_func->key); tab->set_ft_func(ft_func); /* Setup FT handler */ ft_func->set_hints(join, FT_SORTED, select_limit, true); ft_func->join_key = true; table->file->ft_handler = ft_func->ft_handler; return true; } } } /* Keys disabled by ALTER TABLE ... DISABLE KEYS should have already been taken into account. */ Key_map usable_keys = *map; for (ORDER *tmp_order = order; tmp_order; tmp_order = tmp_order->next) { Item *item = (*tmp_order->item)->real_item(); if (item->type() != Item::FIELD_ITEM) { usable_keys.clear_all(); return 0; } usable_keys.intersect(((Item_field *)item)->field->part_of_sortkey); if (usable_keys.is_clear_all()) return 0; // No usable keys } if (tab->type() == JT_REF_OR_NULL || tab->type() == JT_FT) return 0; ref_key = -1; /* Test if constant range in WHERE */ if (tab->type() == JT_REF) { DBUG_ASSERT(tab->ref().key >= 0 && tab->ref().key_parts); ref_key = tab->ref().key; ref_key_parts = tab->ref().key_parts; } else if (tab->type() == JT_RANGE || tab->type() == JT_INDEX_MERGE) { // Range found by opt_range int quick_type = tab->quick()->get_type(); /* assume results are not ordered when index merge is used TODO: sergeyp: Results of all index merge selects actually are ordered by clustered PK values. */ if (quick_type == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE || quick_type == QUICK_SELECT_I::QS_TYPE_ROR_UNION || quick_type == QUICK_SELECT_I::QS_TYPE_ROR_INTERSECT) return 0; ref_key = tab->quick()->index; ref_key_parts = tab->quick()->used_key_parts; } else if (tab->type() == JT_INDEX_SCAN) { // The optimizer has decided to use an index scan. ref_key = tab->index(); ref_key_parts = actual_key_parts(&table->key_info[tab->index()]); } Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_skip_sort_order( trace, "reconsidering_access_paths_for_index_ordering"); trace_skip_sort_order.add_alnum( "clause", (order.src == ESC_ORDER_BY ? "ORDER BY" : "GROUP BY")); Opt_trace_array trace_steps(trace, "steps"); if (ref_key >= 0) { /* We come here when ref/index scan/range scan access has been set up for this table. Do not change access method if ordering is provided already. */ if (!usable_keys.is_set(ref_key)) { /* We come here when ref_key is not among usable_keys, try to find a usable prefix key of that key. */ uint new_ref_key; /* If using index only read, only consider other possible index only keys */ if (table->covering_keys.is_set(ref_key)) usable_keys.intersect(table->covering_keys); if ((new_ref_key = test_if_subkey(&order, tab, ref_key, ref_key_parts, &usable_keys)) < MAX_KEY) { /* Found key that can be used to retrieve data in sorted order */ if (tab->ref().key >= 0) { /* We'll use ref access method on key new_ref_key. The actual change is done further down in this function where we update the plan. */ set_up_ref_access_to_key = true; } else if (!no_changes) { /* The range optimizer constructed QUICK_RANGE for ref_key, and we want to use instead new_ref_key as the index. We can't just change the index of the quick select, because this may result in an incosistent QUICK_SELECT object. Below we create a new QUICK_SELECT from scratch so that all its parameres are set correctly by the range optimizer. Note that the range optimizer is NOT called if no_changes==true. This reason is that the range optimizer cannot find a QUICK that can return ordered result unless index access (ref or index scan) is also able to do so (which test_if_order_by_key () will tell). Admittedly, range access may be much more efficient than e.g. index scan, but the only thing that matters when no_change==true is the answer to the question: "Is it possible to avoid sorting if an index is used to access this table?". The answer does not depend on the outcome of the range optimizer. */ Key_map new_ref_key_map; // Force the creation of quick select new_ref_key_map.set_bit(new_ref_key); // only for new_ref_key. Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_recest(trace, "rows_estimation"); trace_recest.add_utf8_table(tab->table_ref) .add_utf8("index", table->key_info[new_ref_key].name); QUICK_SELECT_I *qck; const bool no_quick = test_quick_select(thd, new_ref_key_map, 0, // empty table_map join->calc_found_rows ? HA_POS_ERROR : join->unit->select_limit_cnt, false, // don't force quick range order->direction, tab, // we are after make_join_select(): tab->condition(), &tab->needed_reg, &qck, tab->table()->force_index) <= 0; DBUG_ASSERT(tab->quick() == save_quick); tab->set_quick(qck); if (no_quick) { can_skip_sorting = false; goto fix_ICP; } } ref_key = new_ref_key; changed_key = new_ref_key; } } bool dummy; /* Check if we get the rows in requested sorted order by using the key */ if (usable_keys.is_set(ref_key)) // Last parameter can be ignored as it'll be checked later, if needed order_direction = test_if_order_by_key(&order, table, ref_key, &used_key_parts, &dummy); } if (ref_key < 0 || order_direction <= 0) { /* There is no ref/index scan/range scan access set up for this table, or it does not provide the requested ordering, or it uses backward scan. Do a cost-based search on all keys. */ uint best_key_parts = 0; uint saved_best_key_parts = 0; int best_key_direction = 0; ha_rows table_records = table->file->stats.records; /* If an index scan that cannot provide ordering has been selected then do not use the index scan key as starting hint to test_if_cheaper_ordering() */ const int ref_key_hint = (order_direction == 0 && tab->type() == JT_INDEX_SCAN) ? -1 : ref_key; test_if_cheaper_ordering(tab, &order, table, usable_keys, ref_key_hint, select_limit, &best_key, &best_key_direction, &select_limit, &best_key_parts, &saved_best_key_parts); // Try backward scan for previously found key if (best_key < 0 && order_direction < 0) goto check_reverse_order; if (best_key < 0) { // No usable key has been found can_skip_sorting = false; goto fix_ICP; } /* If found index will use backward index scan, ref_key uses backward range/ref, pick the latter as it has better selectivity. */ if (order_direction < 0 && order_direction == best_key_direction) { best_key = -1; // reset found best key goto check_reverse_order; } /* Does the query have a "FORCE INDEX [FOR GROUP BY] (idx)" (if clause is group by) or a "FORCE INDEX [FOR ORDER BY] (idx)" (if clause is order by)? */ const bool is_group_by = join && join->grouped && order == join->group_list; const bool is_force_index = table->force_index || (is_group_by ? table->force_index_group : table->force_index_order); /* filesort() and join cache are usually faster than reading in index order and not using join cache. Don't use index scan unless: - the user specified FORCE INDEX [FOR {GROUP|ORDER} BY] (have to assume the user knows what's best) - the chosen index is clustered primary key (table scan is not cheaper) */ if (!is_force_index && (select_limit >= table_records) && (tab->type() == JT_ALL && join->primary_tables > join->const_tables + 1) && ((unsigned)best_key != table->s->primary_key || !table->file->primary_key_is_clustered())) { can_skip_sorting = false; goto fix_ICP; } if (table->quick_keys.is_set(best_key) && !tab->quick_order_tested.is_set(best_key) && best_key != ref_key) { tab->quick_order_tested.set_bit(best_key); Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_recest(trace, "rows_estimation"); trace_recest.add_utf8_table(tab->table_ref) .add_utf8("index", table->key_info[best_key].name); Key_map keys_to_use; // Force the creation of quick select keys_to_use.set_bit(best_key); // only best_key. QUICK_SELECT_I *qck; test_quick_select( thd, keys_to_use, 0, // empty table_map join->calc_found_rows ? HA_POS_ERROR : join->unit->select_limit_cnt, true, // force quick range order->direction, tab, tab->condition(), &tab->needed_reg, &qck, tab->table()->force_index); if (order_direction < 0 && tab->quick() != NULL && tab->quick() != save_quick) { /* We came here in case when 3 indexes are available for quick select: 1 - found by join order optimizer (greedy search) and saved in save_quick 2 - constructed far above, as better suited for order by, but it was found that it requires backward scan. 3 - constructed right above In this case we drop quick #2 as #3 is expected to be better. */ delete tab->quick(); tab->set_quick(NULL); } /* If tab->quick() pointed to another quick than save_quick, we would lose access to it and leak memory. */ DBUG_ASSERT(tab->quick() == save_quick || tab->quick() == NULL); tab->set_quick(qck); } order_direction = best_key_direction; /* saved_best_key_parts is actual number of used keyparts found by the test_if_order_by_key function. It could differ from keyinfo->key_parts, thus we have to restore it in case of desc order as it affects QUICK_SELECT_DESC behaviour. */ used_key_parts = (order_direction == -1) ? saved_best_key_parts : best_key_parts; changed_key = best_key; // We will use index scan or range scan: set_up_ref_access_to_key = false; } check_reverse_order: DBUG_ASSERT(order_direction != 0); if (order_direction == -1) // If ORDER BY ... DESC { if (tab->quick()) { /* Don't reverse the sort order, if it's already done. (In some cases test_if_order_by_key() can be called multiple times */ if (tab->quick()->reverse_sorted()) { can_skip_sorting = true; goto fix_ICP; } // test_if_cheaper_ordering() might disable range scan on current index if (tab->table()->quick_keys.is_set(tab->quick()->index) && tab->quick()->reverse_sort_possible()) can_skip_sorting = true; else { can_skip_sorting = false; goto fix_ICP; } } else { // Other index access (ref or scan) poses no problem can_skip_sorting = true; } } else { // ORDER BY ASC poses no problem can_skip_sorting = true; } DBUG_ASSERT(can_skip_sorting); /* Update query plan with access pattern for doing ordered access according to what we have decided above. */ if (!no_changes) // We are allowed to update QEP { if (set_up_ref_access_to_key) { /* We'll use ref access method on key changed_key. In general case the index search tuple for changed_ref_key will be different (e.g. when one index is defined as (part1, part2, ...) and another as (part1, part2(N), ...) and the WHERE clause contains "part1 = const1 AND part2=const2". So we build tab->ref() from scratch here. */ Key_use *keyuse = tab->keyuse(); while (keyuse->key != (uint)changed_key && keyuse->table_ref == tab->table_ref) keyuse++; if (create_ref_for_key(join, tab, keyuse, tab->prefix_tables())) { can_skip_sorting = false; goto fix_ICP; } DBUG_ASSERT(tab->type() != JT_REF_OR_NULL && tab->type() != JT_FT); // Changing the key makes filter_effect obsolete tab->position()->filter_effect = COND_FILTER_STALE; /* Check if it is possible to shift from ref to range. The index chosen for 'ref' has changed since the last time this function was called. */ if (can_switch_from_ref_to_range(thd, tab, order->direction, true)) { // Allow the code to fall-through to the next if condition. set_up_ref_access_to_key = false; best_key = changed_key; } } if (!set_up_ref_access_to_key && best_key >= 0) { // Cancel any ref-access previously set up tab->ref().key = -1; tab->ref().key_parts = 0; /* If ref_key used index tree reading only ('Using index' in EXPLAIN), and best_key doesn't, then revert the decision. */ if (!table->covering_keys.is_set(best_key)) table->set_keyread(false); if (!tab->quick() || tab->quick() == save_quick) // created no QUICK { // Avoid memory leak: DBUG_ASSERT(tab->quick() == save_quick || tab->quick() == NULL); tab->set_quick(NULL); tab->set_index(best_key); tab->set_type(JT_INDEX_SCAN); // Read with index_first(), index_next() /* There is a bug. When we change here, e.g. from group_min_max to index scan: loose index scan expected to read a small number of rows (jumping through the index), this small number was in position()->rows_fetched; index scan will read much more, so rows_fetched should be updated. So should the filtering effect. It is visible in main.distinct in trunk: explain SELECT distinct a from t3 order by a desc limit 2; id select_type table partitions type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t3 NULL index a a 5 NULL 40 25.00 Using index "rows=40" should be ~200 i.e. # of records in table. Filter should be 100.00 (no WHERE). */ table->file->ha_index_or_rnd_end(); tab->position()->filter_effect = COND_FILTER_STALE; } else if (tab->type() != JT_ALL) { /* We're about to use a quick access to the table. We need to change the access method so as the quick access method is actually used. */ DBUG_ASSERT(tab->quick()); DBUG_ASSERT(tab->quick()->index == (uint)best_key); tab->set_type(calc_join_type(tab->quick()->get_type())); tab->use_quick = QS_RANGE; if (tab->quick()->is_loose_index_scan()) join->tmp_table_param.precomputed_group_by = true; tab->position()->filter_effect = COND_FILTER_STALE; } } // best_key >= 0 if (order_direction == -1) // If ORDER BY ... DESC { if (tab->quick()) { /* ORDER BY range_key DESC */ QUICK_SELECT_I *tmp = tab->quick()->make_reverse(used_key_parts); if (!tmp) { /* purecov: begin inspected */ can_skip_sorting = false; // Reverse sort failed -> filesort goto fix_ICP; /* purecov: end */ } if (tab->quick() != tmp && tab->quick() != save_quick) delete tab->quick(); tab->set_quick(tmp); tab->set_type(calc_join_type(tmp->get_type())); tab->position()->filter_effect = COND_FILTER_STALE; } else if (tab->type() == JT_REF && tab->ref().key_parts <= used_key_parts) { /* SELECT * FROM t1 WHERE a=1 ORDER BY a DESC,b DESC Use a traversal function that starts by reading the last row with key part (A) and then traverse the index backwards. */ tab->reversed_access = true; /* The current implementation of the reverse RefIterator does not work well in combination with ICP and can lead to increased execution time. Setting changed_key to the current key (based on that we change the access order for the key) will ensure that a pushed index condition will be cancelled. */ changed_key = tab->ref().key; } else if (tab->type() == JT_INDEX_SCAN) tab->reversed_access = true; } else if (tab->quick()) tab->quick()->need_sorted_output(); } // QEP has been modified fix_ICP: /* Cleanup: We may have both a 'tab->quick()' and 'save_quick' (original) at this point. Delete the one that we won't use. */ if (can_skip_sorting && !no_changes) { if (tab->type() == JT_INDEX_SCAN && select_limit < table->file->stats.records) { DBUG_ASSERT(select_limit > 0); tab->position()->rows_fetched = select_limit; tab->position()->filter_effect = COND_FILTER_STALE_NO_CONST; } // Keep current (ordered) tab->quick() if (save_quick != tab->quick()) delete save_quick; } else { // Restore original save_quick if (tab->quick() != save_quick) { delete tab->quick(); tab->set_quick(save_quick); } } trace_steps.end(); Opt_trace_object trace_change_index(trace, "index_order_summary"); trace_change_index.add_utf8_table(tab->table_ref) .add("index_provides_order", can_skip_sorting) .add_alnum("order_direction", order_direction == 1 ? "asc" : ((order_direction == -1) ? "desc" : "undefined")); if (changed_key >= 0) { // switching to another index // Should be no pushed index conditions at this point DBUG_ASSERT(!table->file->pushed_idx_cond); if (unlikely(trace->is_started())) { trace_change_index.add_utf8("index", table->key_info[changed_key].name); trace_change_index.add("plan_changed", !no_changes); if (!no_changes) trace_change_index.add_alnum("access_type", join_type_str[tab->type()]); } } else if (unlikely(trace->is_started())) { trace_change_index.add_utf8( "index", ref_key >= 0 ? table->key_info[ref_key].name : "unknown"); trace_change_index.add("plan_changed", false); } *order_idx = best_key < 0 ? ref_key : best_key; return can_skip_sorting; } /** Prune partitions for all tables of a join (query block). Requires that tables have been locked. @returns false if success, true if error */ bool JOIN::prune_table_partitions() { DBUG_ASSERT(select_lex->partitioned_table_count); for (TABLE_LIST *tbl = select_lex->leaf_tables; tbl; tbl = tbl->next_leaf) { /* If tbl->embedding!=NULL that means that this table is in the inner part of the nested outer join, and we can't do partition pruning (TODO: check if this limitation can be lifted. This also excludes semi-joins. Is that intentional?) This will try to prune non-static conditions, which can be used after the tables are locked. */ if (!tbl->embedding) { Item *prune_cond = tbl->join_cond_optim() ? tbl->join_cond_optim() : where_cond; if (prune_partitions(thd, tbl->table, prune_cond)) return true; } } return false; } /** A helper function to check whether it's better to use range than ref. @details Heuristic: Switch from 'ref' to 'range' access if 'range' access can utilize more keyparts than 'ref' access. Conditions for doing switching: 1) Range access is possible 2) 'ref' access and 'range' access uses the same index 3) Used parts of key shouldn't have nullable parts & ref_or_null isn't used. 4) 'ref' access depends on a constant, not a value read from a table earlier in the join sequence. Rationale: if 'ref' depends on a value from another table, the join condition is not used to limit the rows read by 'range' access (that would require dynamic range - 'Range checked for each record'). In other words, if 'ref' depends on a value from another table, we have a query with conditions of the form this_table.idx_col1 = other_table.col AND <<- used by 'ref' this_table.idx_col1 OP @ AND <<- used by 'range' this_table.idx_col2 OP @ AND ... <<- used by 'range' and an index on (idx_col1,idx_col2,...). But the fact that 'range' access uses more keyparts does not mean that it is more selective than 'ref' access because these access types utilize different parts of the query condition. We therefore trust the cost based choice made by best_access_path() instead of forcing a heuristic choice here. 5) 'range' access uses more keyparts than 'ref' access 6) ORDER BY might make range better than table scan: Check possibility of range scan even if it was previously deemed unviable (for example when table scan was estimated to be cheaper). If yes, range-access should be chosen only for larger key length. @param thd To re-run range optimizer. @param tab JOIN_TAB to check @param ordering Used as a parameter to call test_quick_select. @param recheck_range Check possibility of range scan even if it is currently unviable. @return true Range is better than ref @return false Ref is better or switch isn't possible @todo: This decision should rather be made in best_access_path() */ static bool can_switch_from_ref_to_range(THD *thd, JOIN_TAB *tab, enum_order ordering, bool recheck_range) { if ((tab->quick() && // 1) tab->position()->key->key == tab->quick()->index) || // 2) recheck_range) { uint keyparts = 0, length = 0; table_map dep_map = 0; bool maybe_null = false; calc_length_and_keyparts(tab->position()->key, tab, tab->position()->key->key, tab->prefix_tables(), NULL, &length, &keyparts, &dep_map, &maybe_null); if (!maybe_null && // 3) !dep_map) // 4) { if (recheck_range) // 6) { Key_map new_ref_key_map; new_ref_key_map.set_bit(tab->ref().key); Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object can_switch( trace, "check_if_range_uses_more_keyparts_than_ref"); Opt_trace_object trace_cond( trace, "rerunning_range_optimizer_for_single_index"); QUICK_SELECT_I *qck; if (test_quick_select( thd, new_ref_key_map, 0, // empty table_map tab->join()->row_limit, false, ordering, tab, tab->join_cond() ? tab->join_cond() : tab->join()->where_cond, &tab->needed_reg, &qck, recheck_range) > 0) { if (length < qck->max_used_key_length) { delete tab->quick(); tab->set_quick(qck); return true; } else { Opt_trace_object(trace, "access_type_unchanged") .add("ref_key_length", length) .add("range_key_length", qck->max_used_key_length); delete qck; } } } else return length < tab->quick()->max_used_key_length; // 5) } } return false; } /** An utility function - apply heuristics and optimize access methods to tables. Currently this function can change REF to RANGE and ALL to INDEX scan if latter is considered to be better (not cost-based) than the former. @note Side effect - this function could set 'Impossible WHERE' zero result. */ void JOIN::adjust_access_methods() { ASSERT_BEST_REF_IN_JOIN_ORDER(this); for (uint i = const_tables; i < tables; i++) { JOIN_TAB *const tab = best_ref[i]; TABLE_LIST *const tl = tab->table_ref; if (tab->type() == JT_ALL) { /* It's possible to speedup query by switching from full table scan to the scan of covering index, due to less data being read. Prerequisites for this are: 1) Keyread (i.e index only scan) is allowed (table isn't updated/deleted from) 2) Covering indexes are available 3) This isn't a derived table/materialized view */ if (!tab->table()->no_keyread && // 1 !tab->table()->covering_keys.is_clear_all() && // 2 !tl->uses_materialization()) // 3 { /* It has turned out that the change commented out below, while speeding things up for disk-bound loads, slows them down for cases when the data is in disk cache (see BUG#35850): // See bug #26447: "Using the clustered index for a table scan // is always faster than using a secondary index". if (table->s->primary_key != MAX_KEY && table->file->primary_key_is_clustered()) tab->index= table->s->primary_key; else tab->index=find_shortest_key(table, & table->covering_keys); */ if (tab->position()->sj_strategy != SJ_OPT_LOOSE_SCAN) tab->set_index( find_shortest_key(tab->table(), &tab->table()->covering_keys)); tab->set_type(JT_INDEX_SCAN); // Read with index_first / index_next // From table scan to index scan, thus filter effect needs no recalc. } } else if (tab->type() == JT_REF) { if (can_switch_from_ref_to_range(thd, tab, ORDER_NOT_RELEVANT, false)) { tab->set_type(JT_RANGE); Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object wrapper(trace); Opt_trace_object(trace, "access_type_changed") .add_utf8_table(tl) .add_utf8("index", tab->table()->key_info[tab->position()->key->key].name) .add_alnum("old_type", "ref") .add_alnum("new_type", join_type_str[tab->type()]) .add_alnum("cause", "uses_more_keyparts"); tab->use_quick = QS_RANGE; tab->position()->filter_effect = COND_FILTER_STALE; } else { // Cleanup quick, REF/REF_OR_NULL/EQ_REF, will be clarified later delete tab->quick(); tab->set_quick(NULL); } } // Ensure AM consistency DBUG_ASSERT( !(tab->quick() && (tab->type() == JT_REF || tab->type() == JT_ALL))); DBUG_ASSERT((tab->type() != JT_RANGE && tab->type() != JT_INDEX_MERGE) || tab->quick()); if (!tab->const_keys.is_clear_all() && tab->table()->reginfo.impossible_range && ((i == const_tables && tab->type() == JT_REF) || ((tab->type() == JT_ALL || tab->type() == JT_RANGE || tab->type() == JT_INDEX_MERGE || tab->type() == JT_INDEX_SCAN) && tab->use_quick != QS_RANGE)) && !tab->table_ref->is_inner_table_of_outer_join()) zero_result_cause = "Impossible WHERE noticed after reading const tables"; } } static JOIN_TAB *alloc_jtab_array(THD *thd, uint table_count) { JOIN_TAB *t = new (thd->mem_root) JOIN_TAB[table_count]; if (!t) return NULL; /* purecov: inspected */ QEP_shared *qs = new (thd->mem_root) QEP_shared[table_count]; if (!qs) return NULL; /* purecov: inspected */ for (uint i = 0; i < table_count; ++i) t[i].set_qs(qs++); return t; } /** Set up JOIN_TAB structs according to the picked join order in best_positions. This allocates execution structures so may be called only after we have the very final plan. It must be called after Optimize_table_order::fix_semijoin_strategies(). @return False if success, True if error @details - create join->join_tab array and copy from existing JOIN_TABs in join order - create helper structs for materialized semi-join handling - finalize semi-join strategy choices - Number of intermediate tables "tmp_tables" is calculated. - "tables" and "primary_tables" are recalculated. - for full and index scans info of estimated # of records is updated. - in a helper function: - all heuristics are applied and the final access method type is picked for each join_tab (only test_if_skip_sortorder() could override it) - AM consistency is ensured (e.g only range and index merge are allowed to have quick select set). - if "Impossible WHERE" is detected - appropriate zero_result_cause is set. Notice that intermediate tables will not have a POSITION reference; and they will not have a TABLE reference before the final stages of code generation. @todo the block which sets tab->type should move to adjust_access_methods for unification. */ bool JOIN::get_best_combination() { DBUG_TRACE; // At this point "tables" and "primary"tables" represent the same: DBUG_ASSERT(tables == primary_tables); /* Allocate additional space for tmp tables. Number of plan nodes: # of regular input tables (including semi-joined ones) + # of semi-join nests for materialization + 1? + // For GROUP BY (or implicit grouping when we have windowing) 1? + // For DISTINCT 1? + // For aggregation functions aggregated in outer query // when used with distinct 1? + // For ORDER BY 1? // buffer result Up to 2 tmp tables + N window output tmp are allocated (NOTE: windows also have frame buffer tmp tables, but those are not relevant here). */ uint num_tmp_tables = (group_list || (implicit_grouping && m_windows.elements) > 0 ? 1 : 0) + (select_distinct ? (tmp_table_param.outer_sum_func_count ? 2 : 1) : 0) + (order ? 1 : 0) + (select_lex->active_options() & (SELECT_BIG_RESULT | OPTION_BUFFER_RESULT) ? 1 : 0) + m_windows.elements + 1; /* the presence of windows may increase need for grouping tmp tables, cf. de-optimization in make_tmp_tables_info */ if (num_tmp_tables > (2 + m_windows.elements)) num_tmp_tables = 2 + m_windows.elements; /* Rearrange queries with materialized semi-join nests so that the semi-join nest is replaced with a reference to a materialized temporary table and all materialized subquery tables are placed after the intermediate tables. After the following loop, "inner_target" is the position of the first subquery table (if any). "outer_target" is the position of first outer table, and will later be used to track the position of any materialized temporary tables. */ const bool has_semijoin = !select_lex->sj_nests.is_empty(); uint outer_target = 0; uint inner_target = primary_tables + num_tmp_tables; uint sjm_nests = 0; if (has_semijoin) { for (uint tableno = 0; tableno < primary_tables;) { if (sj_is_materialize_strategy(best_positions[tableno].sj_strategy)) { sjm_nests++; inner_target -= (best_positions[tableno].n_sj_tables - 1); tableno += best_positions[tableno].n_sj_tables; } else tableno++; } } JOIN_TAB *tmp_join_tabs = NULL; if (sjm_nests + num_tmp_tables) { // join_tab array only has "primary_tables" tables. We need those more: if (!(tmp_join_tabs = alloc_jtab_array(thd, sjm_nests + num_tmp_tables))) return true; /* purecov: inspected */ } // To check that we fill the array correctly: fill it with zeros first memset(best_ref, 0, sizeof(JOIN_TAB *) * (primary_tables + sjm_nests + num_tmp_tables)); int sjm_index = tables; // Number assigned to materialized temporary table int remaining_sjm_inner = 0; bool err = false; for (uint tableno = 0; tableno < tables; tableno++) { POSITION *const pos = best_positions + tableno; if (has_semijoin && sj_is_materialize_strategy(pos->sj_strategy)) { DBUG_ASSERT(outer_target < inner_target); TABLE_LIST *const sj_nest = pos->table->emb_sj_nest; // Handle this many inner tables of materialized semi-join remaining_sjm_inner = pos->n_sj_tables; /* If we fail in some allocation below, we cannot bail out immediately; that would put us in a difficult situation to clean up; imagine we have planned this layout: outer1 - sj_mat_tmp1 - outer2 - sj_mat_tmp2 - outer3 We have successfully filled a JOIN_TAB for sj_mat_tmp1, and are failing to fill a JOIN_TAB for sj_mat_tmp2 (OOM). So we want to quit this function, which will lead to cleanup functions. But sj_mat_tmp1 is in this->best_ref only, outer3 is in this->join_tab only: what is the array to traverse for cleaning up? What is the number of tables to loop over? So: if we fail in the present loop, we record the error but continue filling best_ref; when it's fully filled, bail out, because then best_ref can be used as reliable array for cleaning up. */ JOIN_TAB *const tab = tmp_join_tabs++; best_ref[outer_target] = tab; tab->set_join(this); tab->set_idx(outer_target); /* Up to this point there cannot be a failure. JOIN_TAB has been filled enough to be clean-able. */ Semijoin_mat_exec *const sjm_exec = new (thd->mem_root) Semijoin_mat_exec( sj_nest, (pos->sj_strategy == SJ_OPT_MATERIALIZE_SCAN), remaining_sjm_inner, outer_target, inner_target); tab->set_sj_mat_exec(sjm_exec); if (!sjm_exec || setup_semijoin_materialized_table( tab, sjm_index, pos, best_positions + sjm_index)) err = true; /* purecov: inspected */ outer_target++; sjm_index++; } /* Locate join_tab target for the table we are considering. (remaining_sjm_inner becomes negative for non-SJM tables, this can be safely ignored). */ const uint target = (remaining_sjm_inner--) > 0 ? inner_target++ : outer_target++; JOIN_TAB *const tab = pos->table; best_ref[target] = tab; tab->set_idx(target); tab->set_position(pos); TABLE *const table = tab->table(); if (tab->type() != JT_CONST && tab->type() != JT_SYSTEM) { if (pos->sj_strategy == SJ_OPT_LOOSE_SCAN && tab->quick() && tab->quick()->index != pos->loosescan_key) { /* We must use the duplicate-eliminating index, so this QUICK is not an option. */ delete tab->quick(); tab->set_quick(NULL); } if (!pos->key) { if (tab->quick()) tab->set_type(calc_join_type(tab->quick()->get_type())); else tab->set_type(JT_ALL); } else // REF or RANGE, clarify later when prefix tables are set for JOIN_TABs tab->set_type(JT_REF); } DBUG_ASSERT(tab->type() != JT_UNKNOWN); DBUG_ASSERT(table->reginfo.join_tab == tab); if (!tab->join_cond()) table->reginfo.not_exists_optimize = false; // Only with LEFT JOIN map2table[tab->table_ref->tableno()] = tab; } // Count the materialized semi-join tables as regular input tables tables += sjm_nests + num_tmp_tables; // Set the number of non-materialized tables: primary_tables = outer_target; /* Between the last outer table or sj-mat tmp table, and the first sj-mat inner table, there may be 2 slots for sort/group/etc tmp tables: */ for (uint i = 0; i < num_tmp_tables; ++i) { const uint idx = outer_target + i; tmp_join_tabs->set_join(this); tmp_join_tabs->set_idx(idx); DBUG_ASSERT(best_ref[idx] == NULL); // verify that not overwriting best_ref[idx] = tmp_join_tabs++; /* note that set_table() cannot be called yet. We may not even use this JOIN_TAB in the end, it's dummy at the moment. Which can be tested with "position()!=NULL". */ } // make array unreachable: should walk JOIN_TABs by best_ref now join_tab = NULL; if (err) return true; /* purecov: inspected */ if (has_semijoin) { set_semijoin_info(); // Update equalities and keyuses after having added SJ materialization if (update_equalities_for_sjm()) return true; } if (!plan_is_const()) { // Assign map of "available" tables to all tables belonging to query block set_prefix_tables(); adjust_access_methods(); } // Calculate outer join info if (select_lex->outer_join) make_outerjoin_info(); // sjm is no longer needed, trash it. To reuse it, reset its members! List_iterator sj_list_it(select_lex->sj_nests); TABLE_LIST *sj_nest; while ((sj_nest = sj_list_it++)) TRASH(&sj_nest->nested_join->sjm, sizeof(sj_nest->nested_join->sjm)); return false; } /** Updates JOIN::deps_of_remaining_lateral_derived_tables @param plan_tables map of all tables that the planner is processing (tables already in plan and tables to be added to plan) @param idx index of the table which the planner is currently considering */ void JOIN::recalculate_deps_of_remaining_lateral_derived_tables( table_map plan_tables, uint idx) { DBUG_ASSERT(has_lateral); deps_of_remaining_lateral_derived_tables = 0; auto last = best_ref + tables; for (auto **pos = best_ref + idx; pos < last; pos++) { if ((*pos)->table_ref && (*pos)->table_ref->is_derived() && ((*pos)->table_ref->map() & plan_tables)) deps_of_remaining_lateral_derived_tables |= (*pos)->table_ref->derived_unit()->m_lateral_deps; } } /* Revise usage of join buffer for the specified table and the whole nest SYNOPSIS revise_cache_usage() tab join table for which join buffer usage is to be revised DESCRIPTION The function revise the decision to use a join buffer for the table 'tab'. If this table happened to be among the inner tables of a nested outer join/ semi-join the functions denies usage of join buffers for all of them RETURN none */ static void revise_cache_usage(JOIN_TAB *join_tab) { plan_idx first_inner = join_tab->first_inner(); JOIN *const join = join_tab->join(); if (first_inner != NO_PLAN_IDX) { plan_idx end_tab = join_tab->idx(); for (first_inner = join_tab->first_inner(); first_inner != NO_PLAN_IDX; first_inner = join->best_ref[first_inner]->first_upper()) { for (plan_idx i = end_tab - 1; i >= first_inner; --i) join->best_ref[i]->set_use_join_cache(JOIN_CACHE::ALG_NONE); end_tab = first_inner; } } else if (join_tab->get_sj_strategy() == SJ_OPT_FIRST_MATCH) { plan_idx first_sj_inner = join_tab->first_sj_inner(); for (plan_idx i = join_tab->idx() - 1; i >= first_sj_inner; --i) { JOIN_TAB *tab = join->best_ref[i]; if (tab->first_sj_inner() == first_sj_inner) tab->set_use_join_cache(JOIN_CACHE::ALG_NONE); } } else join_tab->set_use_join_cache(JOIN_CACHE::ALG_NONE); DBUG_ASSERT(join->qep_tab == NULL); } /** Set up join buffering for a specified table, if possible. @param tab joined table to check join buffer usage for @param join join for which the check is performed @param no_jbuf_after don't use join buffering after table with this number @return false if successful, true if error. Currently, allocation errors for join cache objects are ignored, and regular execution is chosen silently. @details The function finds out whether the table 'tab' can be joined using a join buffer. This check is performed after the best execution plan for 'join' has been chosen. If the function decides that a join buffer can be employed then it selects the most appropriate join cache type, which later will be instantiated by init_join_cache(). If it has already been decided to not use join buffering for this table, no action is taken. Often it is already decided that join buffering will be used earlier in the optimization process, and this will also ensure that the most correct cost for the operation is calculated, and hence the probability of choosing an optimal join plan is higher. However, some join buffering decisions cannot currently be taken before this stage, hence we need this function to decide the most accurate join buffering strategy. @todo Long-term it is the goal that join buffering strategy is decided when the plan is selected. The result of the check and the type of the join buffer to be used depend on: - the access method to access rows of the joined table - whether the join table is an inner table of an outer join or semi-join - the optimizer_switch settings for join buffering - the join 'options'. In any case join buffer is not used if the number of the joined table is greater than 'no_jbuf_after'. If block_nested_loop is turned on, and if all other criteria for using join buffering is fulfilled (see below), then join buffer is used for any join operation (inner join, outer join, semi-join) with 'JT_ALL' access method. In that case, a JOIN_CACHE_BNL type is always employed. If an index is used to access rows of the joined table and batched_key_access is on, then a JOIN_CACHE_BKA type is employed. (Unless debug flag, test_bka unique, is set, then a JOIN_CACHE_BKA_UNIQUE type is employed instead.) If the function decides that a join buffer can be used to join the table 'tab' then it sets @c tab->use_join_cache to reflect the chosen algorithm. @note For a nested outer join/semi-join, currently, we either use join buffers for all inner tables or for none of them. @todo Support BKA inside SJ-Materialization nests. When doing this, we'll need to only store sj-inner tables in the join buffer. @verbatim JOIN_TAB *first_tab= join->join_tab+join->const_tables; uint n_tables= i-join->const_tables; / * We normally put all preceding tables into the join buffer, except for the constant tables. If we're inside a semi-join materialization nest, e.g. outer_tbl1 outer_tbl2 ( inner_tbl1, inner_tbl2 ) ... ^-- we're here then we need to put into the join buffer only the tables from within the nest. * / if (i >= first_sjm_table && i < last_sjm_table) { n_tables= i - first_sjm_table; // will be >0 if we got here first_tab= join->join_tab + first_sjm_table; } @endverbatim */ static bool setup_join_buffering(JOIN_TAB *tab, JOIN *join, uint no_jbuf_after) { ASSERT_BEST_REF_IN_JOIN_ORDER(join); Cost_estimate cost; ha_rows rows; uint bufsz = 4096; uint join_cache_flags = HA_MRR_NO_NULL_ENDPOINTS; const bool bnl_on = hint_table_state(join->thd, tab->table_ref, BNL_HINT_ENUM, OPTIMIZER_SWITCH_BNL); const bool bka_on = hint_table_state(join->thd, tab->table_ref, BKA_HINT_ENUM, OPTIMIZER_SWITCH_BKA); const uint tableno = tab->idx(); const uint tab_sj_strategy = tab->get_sj_strategy(); bool use_bka_unique = false; DBUG_EXECUTE_IF("test_bka_unique", use_bka_unique = true;); // Set preliminary join cache setting based on decision from greedy search if (!join->select_count) tab->set_use_join_cache(tab->position()->use_join_buffer ? JOIN_CACHE::ALG_BNL : JOIN_CACHE::ALG_NONE); if (tableno == join->const_tables) { DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); return false; } if (!(bnl_on || bka_on)) goto no_join_cache; /* psergey-todo: why the below when execution code seems to handle the "range checked for each record" case? */ if (tab->use_quick == QS_DYNAMIC_RANGE) goto no_join_cache; /* No join buffering if prevented by no_jbuf_after */ if (tableno > no_jbuf_after) goto no_join_cache; /* An inner table of an outer join nest must not use join buffering if the first inner table of that outer join nest does not use join buffering. This condition is not handled by earlier optimizer stages. */ if (tab->first_inner() != NO_PLAN_IDX && tab->first_inner() != tab->idx() && !join->best_ref[tab->first_inner()]->use_join_cache()) goto no_join_cache; /* The first inner table of an outer join nest must not use join buffering if the tables in the embedding outer join nest do not use join buffering. This condition is not handled by earlier optimizer stages. */ if (tab->first_upper() != NO_PLAN_IDX && !join->best_ref[tab->first_upper()]->use_join_cache()) goto no_join_cache; if (tab->table()->pos_in_table_list->is_table_function() && tab->dependent) goto no_join_cache; switch (tab_sj_strategy) { case SJ_OPT_FIRST_MATCH: /* Use join cache with FirstMatch semi-join strategy only when semi-join contains only one table. */ if (!tab->is_single_inner_of_semi_join()) { DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); goto no_join_cache; } break; case SJ_OPT_LOOSE_SCAN: /* No join buffering if this semijoin nest is handled by loosescan */ DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); goto no_join_cache; case SJ_OPT_MATERIALIZE_LOOKUP: case SJ_OPT_MATERIALIZE_SCAN: /* The Materialize strategies reuse the join_tab belonging to the first table that was materialized. Neither table can use join buffering: - The first table in a join never uses join buffering. - The join_tab used for looking up a row in the materialized table, or scanning the rows of a materialized table, cannot use join buffering. We allow join buffering for the remaining tables of the materialized semi-join nest. */ if (tab->first_sj_inner() == tab->idx()) { DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); goto no_join_cache; } break; case SJ_OPT_DUPS_WEEDOUT: // This strategy allows the same join buffering as a regular join would. case SJ_OPT_NONE: break; } /* The following code prevents use of join buffering when there is an outer join operation and first match semi-join strategy is used, because: Outer join needs a "match flag" to track that a row should be NULL-complemented, such flag being attached to first inner table's cache (tracks whether the cached row from outer table got a match, in which case no NULL-complemented row is needed). FirstMatch also needs a "match flag", such flag is attached to sj inner table's cache (tracks whether the cached row from outer table already got a first match in the sj-inner table, in which case we don't need to join this cached row again) - but a row in a cache has only one "match flag" - so if "sj inner table"=="first inner", there is a problem. */ if (tab_sj_strategy == SJ_OPT_FIRST_MATCH && tab->is_inner_table_of_outer_join()) goto no_join_cache; if (join->deps_of_remaining_lateral_derived_tables & (tab->prefix_tables() & ~tab->added_tables())) { /* Even though the planner said "no jbuf please", the switch below may force it. If first-dependency-of-lateral-table < table-we-plan-for <= lateral-table, disable join buffering. Reason for this rule: consider a plan t1-t2-dt where dt is LATERAL and depends only on t1, and imagine t2 could do join buffering: then we buffer many rows of t1, then read one row of t2, fetch row#1 of t1 from cache, then materialize "dt" (as it depends on t1) and send row to client; then fetch row#2 of t1 from cache, rematerialize "dt": it's very inefficient. So we forbid join buffering on t2; this way, the signal "row of t1 changed" is emitted at the level of t1's operator, i.e. much less often, as one row of t1 may serve N rows of t2 before changing. On the other hand, t1 can do join buffering. A nice side-effect is to disable join buffering for "dt" itself. If "dt" would do join buffering: "dt" buffers many rows from t1/t2, then in a second phase we read one row from "dt" and join it with the many rows from t1/t2; but we cannot read a row from "dt" without first choosing a row of t1/t2 as "dt" depends on t1. See similar code in best_access_path(). */ goto no_join_cache; } switch (tab->type()) { case JT_ALL: case JT_INDEX_SCAN: case JT_RANGE: case JT_INDEX_MERGE: if (!bnl_on) { DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); goto no_join_cache; } if (!join->select_count) tab->set_use_join_cache(JOIN_CACHE::ALG_BNL); return false; case JT_SYSTEM: case JT_CONST: case JT_REF: case JT_EQ_REF: if (!bka_on) { DBUG_ASSERT(tab->use_join_cache() == JOIN_CACHE::ALG_NONE); goto no_join_cache; } /* Disable BKA for materializable derived tables/views as they aren't instantiated yet. */ if (tab->table_ref->uses_materialization()) goto no_join_cache; /* Can't use BKA for subquery if dealing with a subquery that can turn a ref access into a "full scan on NULL key" table scan. @see Item_in_optimizer::val_int() @see subselect_iterator_engine::exec() @see TABLE_REF::cond_guards @see push_index_cond() @todo: This choice to not use BKA should be done before making cost estimates, e.g. in set_join_buffer_properties(). That happens before cond guards are set up, so instead of doing the check below, BKA should be disabled if - We are in an IN subquery, and - The IN predicate is not a top_level_item, and - The left_expr of the IN predicate may contain NULL values (left_expr->maybe_null) */ if (tab->has_guarded_conds()) goto no_join_cache; if (tab->table()->covering_keys.is_set(tab->ref().key)) join_cache_flags |= HA_MRR_INDEX_ONLY; rows = tab->table()->file->multi_range_read_info( tab->ref().key, 10, 20, &bufsz, &join_cache_flags, &cost); /* Cannot use BKA/BKA_UNIQUE if 1. MRR scan cannot be performed, or 2. MRR default implementation is used Cannot use BKA if 3. HA_MRR_NO_ASSOCIATION flag is set */ if ((rows == HA_POS_ERROR) || // 1 (join_cache_flags & HA_MRR_USE_DEFAULT_IMPL) || // 2 ((join_cache_flags & HA_MRR_NO_ASSOCIATION) && // 3 !use_bka_unique)) goto no_join_cache; if (use_bka_unique) tab->set_use_join_cache(JOIN_CACHE::ALG_BKA_UNIQUE); else tab->set_use_join_cache(JOIN_CACHE::ALG_BKA); tab->join_cache_flags = join_cache_flags; return false; default:; } no_join_cache: revise_cache_usage(tab); tab->set_use_join_cache(JOIN_CACHE::ALG_NONE); return false; } /***************************************************************************** Make some simple condition optimization: If there is a test 'field = const' change all refs to 'field' to 'const' Remove all dummy tests 'item = item', 'const op const'. Remove all 'item is NULL', when item can never be null! Return in cond_value false if condition is impossible (1 = 2) *****************************************************************************/ class COND_CMP : public ilink { public: static void *operator new(size_t size) { return (*THR_MALLOC)->Alloc(size); } static void operator delete(void *ptr MY_ATTRIBUTE((unused)), size_t size MY_ATTRIBUTE((unused))) { TRASH(ptr, size); } Item *and_level; Item_func *cmp_func; COND_CMP(Item *a, Item_func *b) : and_level(a), cmp_func(b) {} }; /** Find the multiple equality predicate containing a field. The function retrieves the multiple equalities accessed through the cond_equal structure from current level and up looking for an equality containing a field. It stops retrieval as soon as the equality is found and set up inherited_fl to true if it's found on upper levels. @param cond_equal multiple equalities to search in @param item_field field to look for @param[out] inherited_fl set up to true if multiple equality is found on upper levels (not on current level of cond_equal) @return - Item_equal for the found multiple equality predicate if a success; - NULL otherwise. */ static Item_equal *find_item_equal(COND_EQUAL *cond_equal, Item_field *item_field, bool *inherited_fl) { Item_equal *item = 0; bool in_upper_level = false; while (cond_equal) { List_iterator_fast li(cond_equal->current_level); while ((item = li++)) { if (item->contains(item_field->field)) goto finish; } in_upper_level = true; cond_equal = cond_equal->upper_levels; } in_upper_level = false; finish: *inherited_fl = in_upper_level; return item; } /** Get the best field substitution for a given field. If the field is member of a multiple equality, look up that equality and return the most appropriate field. Usually this is the equivalenced field belonging to the outer-most table in the join order, but @see Item_field::get_subst_item() for details. Otherwise, return the same field. @param item_field The field that we are seeking a substitution for. @param cond_equal multiple equalities to search in @return The substituted field. */ Item_field *get_best_field(Item_field *item_field, COND_EQUAL *cond_equal) { bool dummy; Item_equal *item_eq = find_item_equal(cond_equal, item_field, &dummy); if (!item_eq) return item_field; return item_eq->get_subst_item(item_field); } /** Check whether an equality can be used to build multiple equalities. This function first checks whether the equality (left_item=right_item) is a simple equality i.e. one that equates a field with another field or a constant (field=field_item or field=const_item). If this is the case the function looks for a multiple equality in the lists referenced directly or indirectly by cond_equal inferring the given simple equality. If it doesn't find any, it builds a multiple equality that covers the predicate, i.e. the predicate can be inferred from this multiple equality. The built multiple equality could be obtained in such a way: create a binary multiple equality equivalent to the predicate, then merge it, if possible, with one of old multiple equalities. This guarantees that the set of multiple equalities covering equality predicates will be minimal. EXAMPLE: For the where condition @code WHERE a=b AND b=c AND (b=2 OR f=e) @endcode the check_equality will be called for the following equality predicates a=b, b=c, b=2 and f=e. - For a=b it will be called with *cond_equal=(0,[]) and will transform *cond_equal into (0,[Item_equal(a,b)]). - For b=c it will be called with *cond_equal=(0,[Item_equal(a,b)]) and will transform *cond_equal into CE=(0,[Item_equal(a,b,c)]). - For b=2 it will be called with *cond_equal=(ptr(CE),[]) and will transform *cond_equal into (ptr(CE),[Item_equal(2,a,b,c)]). - For f=e it will be called with *cond_equal=(ptr(CE), []) and will transform *cond_equal into (ptr(CE),[Item_equal(f,e)]). @note Now only fields that have the same type definitions (verified by the Field::eq_def method) are placed to the same multiple equalities. Because of this some equality predicates are not eliminated and can be used in the constant propagation procedure. We could weaken the equality test as soon as at least one of the equal fields is to be equal to a constant. It would require a more complicated implementation: we would have to store, in general case, its own constant for each fields from the multiple equality. But at the same time it would allow us to get rid of constant propagation completely: it would be done by the call to build_equal_items_for_cond. The implementation does not follow exactly the above rules to build a new multiple equality for the equality predicate. If it processes the equality of the form field1=field2, it looks for multiple equalities me1 containing field1 and me2 containing field2. If only one of them is found the function expands it with the lacking field. If multiple equalities for both fields are found they are merged. If both searches fail a new multiple equality containing just field1 and field2 is added to the existing multiple equalities. If the function processes the predicate of the form field1=const, it looks for a multiple equality containing field1. If found, the function checks the constant of the multiple equality. If the value is unknown, it is setup to const. Otherwise the value is compared with const and the evaluation of the equality predicate is performed. When expanding/merging equality predicates from the upper levels the function first copies them for the current level. It looks acceptable, as this happens rarely. The implementation without copying would be much more complicated. @param thd Thread handler @param left_item left term of the equality to be checked @param right_item right term of the equality to be checked @param item equality item if the equality originates from a condition predicate, 0 if the equality is the result of row elimination @param cond_equal multiple equalities that must hold together with the equality @param[out] simple_equality true if the predicate is a simple equality predicate to be used for building multiple equalities false otherwise @returns false if success, true if error */ static bool check_simple_equality(THD *thd, Item *left_item, Item *right_item, Item *item, COND_EQUAL *cond_equal, bool *simple_equality) { *simple_equality = false; if (left_item->type() == Item::REF_ITEM && down_cast(left_item)->ref_type() == Item_ref::VIEW_REF) { if (down_cast(left_item)->depended_from) return false; left_item = left_item->real_item(); } if (right_item->type() == Item::REF_ITEM && down_cast(right_item)->ref_type() == Item_ref::VIEW_REF) { if (down_cast(right_item)->depended_from) return false; right_item = right_item->real_item(); } Item_field *left_item_field, *right_item_field; if (left_item->type() == Item::FIELD_ITEM && right_item->type() == Item::FIELD_ITEM && (left_item_field = down_cast(left_item)) && (right_item_field = down_cast(right_item)) && !left_item_field->depended_from && !right_item_field->depended_from) { /* The predicate the form field1=field2 is processed */ Field *const left_field = left_item_field->field; Field *const right_field = right_item_field->field; if (!left_field->eq_def(right_field)) return false; /* Search for multiple equalities containing field1 and/or field2 */ bool left_copyfl, right_copyfl; Item_equal *left_item_equal = find_item_equal(cond_equal, left_item_field, &left_copyfl); Item_equal *right_item_equal = find_item_equal(cond_equal, right_item_field, &right_copyfl); /* As (NULL=NULL) != TRUE we can't just remove the predicate f=f */ if (left_field->eq(right_field)) /* f = f */ { *simple_equality = !(left_field->maybe_null() && !left_item_equal); return false; } if (left_item_equal && left_item_equal == right_item_equal) { /* The equality predicate is inference of one of the existing multiple equalities, i.e the condition is already covered by upper level equalities */ *simple_equality = true; return false; } /* Copy the found multiple equalities at the current level if needed */ if (left_copyfl) { /* left_item_equal of an upper level contains left_item */ left_item_equal = new Item_equal(left_item_equal); if (left_item_equal == NULL) return true; cond_equal->current_level.push_back(left_item_equal); } if (right_copyfl) { /* right_item_equal of an upper level contains right_item */ right_item_equal = new Item_equal(right_item_equal); if (right_item_equal == NULL) return true; cond_equal->current_level.push_back(right_item_equal); } if (left_item_equal) { /* left item was found in the current or one of the upper levels */ if (!right_item_equal) left_item_equal->add(down_cast(right_item)); else { /* Merge two multiple equalities forming a new one */ if (left_item_equal->merge(thd, right_item_equal)) return true; /* Remove the merged multiple equality from the list */ List_iterator li(cond_equal->current_level); while ((li++) != right_item_equal) ; li.remove(); } } else { /* left item was not found neither the current nor in upper levels */ if (right_item_equal) { right_item_equal->add(down_cast(left_item)); } else { /* None of the fields was found in multiple equalities */ Item_equal *item_equal = new Item_equal(down_cast(left_item), down_cast(right_item)); if (item_equal == NULL) return true; cond_equal->current_level.push_back(item_equal); } } *simple_equality = true; return false; } { /* The predicate of the form field=const/const=field is processed */ Item *const_item = 0; Item_field *field_item = 0; if (left_item->type() == Item::FIELD_ITEM && (field_item = down_cast(left_item)) && field_item->depended_from == NULL && right_item->const_for_execution()) { const_item = right_item; } else if (right_item->type() == Item::FIELD_ITEM && (field_item = down_cast(right_item)) && field_item->depended_from == NULL && left_item->const_for_execution()) { const_item = left_item; } // Don't evaluate subqueries if they are disabled during optimization. if (const_item != nullptr && !evaluate_during_optimization(const_item, thd->lex->current_select())) return false; /* If the constant expression contains a reference to the field (for example, a = (a IS NULL)), we don't want to replace the field with the constant expression as it makes the predicates more complex and may introduce cycles in the Item tree. */ if (const_item != nullptr && const_item->walk(&Item::find_field_processor, enum_walk::POSTFIX, pointer_cast(field_item->field))) return false; if (const_item && field_item->result_type() == const_item->result_type()) { if (field_item->result_type() == STRING_RESULT) { const CHARSET_INFO *cs = field_item->field->charset(); if (!item) { Item_func_eq *const eq_item = new Item_func_eq(left_item, right_item); if (eq_item == NULL || eq_item->set_cmp_func()) return true; eq_item->quick_fix_field(); item = eq_item; } if ((cs != down_cast(item)->compare_collation()) || !cs->coll->propagate(cs, 0, 0)) return false; } bool copyfl; Item_equal *item_equal = find_item_equal(cond_equal, field_item, ©fl); if (copyfl) { item_equal = new Item_equal(item_equal); if (item_equal == NULL) return true; cond_equal->current_level.push_back(item_equal); } if (item_equal) { /* The flag cond_false will be set to 1 after this, if item_equal already contains a constant and its value is not equal to the value of const_item. */ if (item_equal->add(thd, const_item, field_item)) return true; } else { item_equal = new Item_equal(const_item, field_item); if (item_equal == NULL) return true; cond_equal->current_level.push_back(item_equal); } *simple_equality = true; return false; } } return false; } /** Convert row equalities into a conjunction of regular equalities. The function converts a row equality of the form (E1,...,En)=(E'1,...,E'n) into a list of equalities E1=E'1,...,En=E'n. For each of these equalities Ei=E'i the function checks whether it is a simple equality or a row equality. If it is a simple equality it is used to expand multiple equalities of cond_equal. If it is a row equality it converted to a sequence of equalities between row elements. If Ei=E'i is neither a simple equality nor a row equality the item for this predicate is added to eq_list. @param thd thread handle @param left_row left term of the row equality to be processed @param right_row right term of the row equality to be processed @param cond_equal multiple equalities that must hold together with the predicate @param eq_list results of conversions of row equalities that are not simple enough to form multiple equalities @param[out] simple_equality true if the row equality is composed of only simple equalities. @returns false if conversion succeeded, true if any error. */ static bool check_row_equality(THD *thd, Item *left_row, Item_row *right_row, COND_EQUAL *cond_equal, List *eq_list, bool *simple_equality) { *simple_equality = false; uint n = left_row->cols(); for (uint i = 0; i < n; i++) { bool is_converted; Item *left_item = left_row->element_index(i); Item *right_item = right_row->element_index(i); if (left_item->type() == Item::ROW_ITEM && right_item->type() == Item::ROW_ITEM) { if (check_row_equality(thd, down_cast(left_item), down_cast(right_item), cond_equal, eq_list, &is_converted)) return true; if (!is_converted) thd->lex->current_select()->cond_count++; } else { if (check_simple_equality(thd, left_item, right_item, nullptr, cond_equal, &is_converted)) return true; thd->lex->current_select()->cond_count++; } if (!is_converted) { Item_func_eq *const eq_item = new Item_func_eq(left_item, right_item); if (eq_item == NULL) return true; if (eq_item->set_cmp_func()) { // Failed to create cmp func -> not only simple equalitities return true; } eq_item->quick_fix_field(); eq_list->push_back(eq_item); } } *simple_equality = true; return false; } /** Eliminate row equalities and form multiple equalities predicates. This function checks whether the item is a simple equality i.e. the one that equates a field with another field or a constant (field=field_item or field=constant_item), or, a row equality. For a simple equality the function looks for a multiple equality in the lists referenced directly or indirectly by cond_equal inferring the given simple equality. If it doesn't find any, it builds/expands multiple equality that covers the predicate. Row equalities are eliminated substituted for conjunctive regular equalities which are treated in the same way as original equality predicates. @param thd thread handle @param item predicate to process @param cond_equal multiple equalities that must hold together with the predicate @param eq_list results of conversions of row equalities that are not simple enough to form multiple equalities @param[out] equality true if re-writing rules have been applied false otherwise, i.e. if the predicate is not an equality, or if the equality is neither a simple nor a row equality @returns false if success, true if error @note If the equality was created by IN->EXISTS, it may be removed later by subquery materialization. So we don't mix this possibly temporary equality with others; if we let it go into a multiple-equality (Item_equal), then we could not remove it later. There is however an exception: if the outer expression is a constant, it is safe to leave the equality even in materialization; all it can do is preventing NULL/FALSE distinction but if such distinction mattered the equality would be in a triggered condition so we would not come to this function. And injecting constants is good because it makes the materialized table smaller. */ static bool check_equality(THD *thd, Item *item, COND_EQUAL *cond_equal, List *eq_list, bool *equality) { *equality = false; DBUG_ASSERT(item->is_bool_func()); Item_func *item_func; if (item->type() == Item::FUNC_ITEM && (item_func = down_cast(item))->functype() == Item_func::EQ_FUNC) { Item *left_item = item_func->arguments()[0]; Item *right_item = item_func->arguments()[1]; if (item->created_by_in2exists() && !left_item->const_item()) return false; // See note above if (left_item->type() == Item::ROW_ITEM && right_item->type() == Item::ROW_ITEM) { thd->lex->current_select()->cond_count--; return check_row_equality(thd, down_cast(left_item), down_cast(right_item), cond_equal, eq_list, equality); } else return check_simple_equality(thd, left_item, right_item, item, cond_equal, equality); } return false; } /** Replace all equality predicates in a condition by multiple equality items. At each 'and' level the function detects items for equality predicates and replaces them by a set of multiple equality items of class Item_equal, taking into account inherited equalities from upper levels. If an equality predicate is used not in a conjunction it's just replaced by a multiple equality predicate. For each 'and' level the function set a pointer to the inherited multiple equalities in the cond_equal field of the associated object of the type Item_cond_and. The function also traverses the cond tree and for each field reference sets a pointer to the multiple equality item containing the field, if there is any. If this multiple equality equates fields to a constant the function replaces the field reference by the constant in the cases when the field is not of a string type or when the field reference is just an argument of a comparison predicate. The function also determines the maximum number of members in equality lists of each Item_cond_and object assigning it to thd->lex->current_select()->max_equal_elems. @note Multiple equality predicate =(f1,..fn) is equivalent to the conjuction of f1=f2, .., fn-1=fn. It substitutes any inference from these equality predicates that is equivalent to the conjunction. Thus, =(a1,a2,a3) can substitute for ((a1=a3) AND (a2=a3) AND (a2=a1)) as it is equivalent to ((a1=a2) AND (a2=a3)). The function always makes a substitution of all equality predicates occurred in a conjunction for a minimal set of multiple equality predicates. This set can be considered as a canonical representation of the sub-conjunction of the equality predicates. E.g. (t1.a=t2.b AND t2.b>5 AND t1.a=t3.c) is replaced by (=(t1.a,t2.b,t3.c) AND t2.b>5), not by (=(t1.a,t2.b) AND =(t1.a,t3.c) AND t2.b>5); while (t1.a=t2.b AND t2.b>5 AND t3.c=t4.d) is replaced by (=(t1.a,t2.b) AND =(t3.c=t4.d) AND t2.b>5), but if additionally =(t4.d,t2.b) is inherited, it will be replaced by (=(t1.a,t2.b,t3.c,t4.d) AND t2.b>5) The function performs the substitution in a recursive descent of the condition tree, passing to the next AND level a chain of multiple equality predicates which have been built at the upper levels. The Item_equal items built at the level are attached to other non-equality conjuncts as a sublist. The pointer to the inherited multiple equalities is saved in the and condition object (Item_cond_and). This chain allows us for any field reference occurence to easily find a multiple equality that must be held for this occurence. For each AND level we do the following: - scan it for all equality predicate (=) items - join them into disjoint Item_equal() groups - process the included OR conditions recursively to do the same for lower AND levels. We need to do things in this order as lower AND levels need to know about all possible Item_equal objects in upper levels. @param thd thread handle @param cond condition(expression) where to make replacement @param[out] retcond returned condition @param inherited path to all inherited multiple equality items @param do_inherit whether or not to inherit equalities from other parts of the condition @returns false if success, true if error */ static bool build_equal_items_for_cond(THD *thd, Item *cond, Item **retcond, COND_EQUAL *inherited, bool do_inherit) { Item_equal *item_equal; COND_EQUAL cond_equal; cond_equal.upper_levels = inherited; DBUG_ASSERT(cond->is_bool_func()); if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) return true; // Fatal error flag is set! const enum Item::Type cond_type = cond->type(); if (cond_type == Item::COND_ITEM) { List eq_list; Item_cond *const item_cond = down_cast(cond); const bool and_level = item_cond->functype() == Item_func::COND_AND_FUNC; List *args = item_cond->argument_list(); List_iterator li(*args); Item *item; if (and_level) { /* Retrieve all conjuncts of this level detecting the equality that are subject to substitution by multiple equality items and removing each such predicate from the conjunction after having found/created a multiple equality whose inference the predicate is. */ while ((item = li++)) { /* PS/SP note: we can safely remove a node from AND-OR structure here because it's restored before each re-execution of any prepared statement/stored procedure. */ bool equality; if (check_equality(thd, item, &cond_equal, &eq_list, &equality)) return true; if (equality) li.remove(); } /* Check if we eliminated all the predicates of the level, e.g. (a=a AND b=b AND a=a). */ if (!args->elements && !cond_equal.current_level.elements && !eq_list.elements) { *retcond = new Item_func_true(); return *retcond == NULL; } List_iterator_fast it(cond_equal.current_level); while ((item_equal = it++)) { if (item_equal->resolve_type(thd)) return true; item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select()->max_equal_elems, item_equal->members()); } Item_cond_and *const item_cond_and = down_cast(cond); item_cond_and->cond_equal = cond_equal; inherited = &item_cond_and->cond_equal; } /* Make replacement of equality predicates for lower levels of the condition expression. */ li.rewind(); while ((item = li++)) { Item *new_item; if (build_equal_items_for_cond(thd, item, &new_item, inherited, do_inherit)) return true; if (new_item != item) { /* This replacement happens only for standalone equalities */ /* This is ok with PS/SP as the replacement is done for arguments of an AND/OR item, which are restored for each execution of PS/SP. */ li.replace(new_item); } } if (and_level) { args->concat(&eq_list); args->concat((List *)&cond_equal.current_level); } } else if (cond->type() == Item::FUNC_ITEM) { List eq_list; /* If an equality predicate forms the whole and level, we call it standalone equality and it's processed here. E.g. in the following where condition WHERE a=5 AND (b=5 or a=c) (b=5) and (a=c) are standalone equalities. In general we can't leave alone standalone eqalities: for WHERE a=b AND c=d AND (b=c OR d=5) b=c is replaced by =(a,b,c,d). */ bool equality; if (check_equality(thd, cond, &cond_equal, &eq_list, &equality)) return true; if (equality) { int n = cond_equal.current_level.elements + eq_list.elements; if (n == 0) { *retcond = new Item_func_true(); return *retcond == NULL; } else if (n == 1) { if ((item_equal = cond_equal.current_level.pop())) { if (item_equal->resolve_type(thd)) return true; item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select()->max_equal_elems, item_equal->members()); *retcond = item_equal; return false; } *retcond = eq_list.pop(); return false; } else { /* Here a new AND level must be created. It can happen only when a row equality is processed as a standalone predicate. */ Item_cond_and *and_cond = new Item_cond_and(eq_list); if (and_cond == NULL) return true; and_cond->quick_fix_field(); List *args = and_cond->argument_list(); List_iterator_fast it(cond_equal.current_level); while ((item_equal = it++)) { if (item_equal->resolve_type(thd)) return true; item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select()->max_equal_elems, item_equal->members()); } and_cond->cond_equal = cond_equal; args->concat((List *)&cond_equal.current_level); *retcond = and_cond; return false; } } if (do_inherit) { /* For each field reference in cond, not from equal item predicates, set a pointer to the multiple equality it belongs to (if there is any) as soon the field is not of a string type or the field reference is an argument of a comparison predicate. */ uchar *is_subst_valid = (uchar *)1; cond = cond->compile(&Item::subst_argument_checker, &is_subst_valid, &Item::equal_fields_propagator, (uchar *)inherited); if (cond == NULL) return true; } cond->update_used_tables(); } *retcond = cond; return false; } /** Build multiple equalities for a WHERE condition and all join conditions that inherit these multiple equalities. The function first applies the build_equal_items_for_cond function to build all multiple equalities for condition cond utilizing equalities referred through the parameter inherited. The extended set of equalities is returned in the structure referred by the cond_equal_ref parameter. After this the function calls itself recursively for all join conditions whose direct references can be found in join_list and who inherit directly the multiple equalities just having built. @note The join condition used in an outer join operation inherits all equalities from the join condition of the embedding join, if there is any, or otherwise - from the where condition. This fact is not obvious, but presumably can be proved. Consider the following query: @code SELECT * FROM (t1,t2) LEFT JOIN (t3,t4) ON t1.a=t3.a AND t2.a=t4.a WHERE t1.a=t2.a; @endcode If the join condition in the query inherits =(t1.a,t2.a), then we can build the multiple equality =(t1.a,t2.a,t3.a,t4.a) that infers the equality t3.a=t4.a. Although the join condition t1.a=t3.a AND t2.a=t4.a AND t3.a=t4.a is not equivalent to the one in the query the latter can be replaced by the former: the new query will return the same result set as the original one. Interesting that multiple equality =(t1.a,t2.a,t3.a,t4.a) allows us to use t1.a=t3.a AND t3.a=t4.a under the join condition: @code SELECT * FROM (t1,t2) LEFT JOIN (t3,t4) ON t1.a=t3.a AND t3.a=t4.a WHERE t1.a=t2.a @endcode This query equivalent to: @code SELECT * FROM (t1 LEFT JOIN (t3,t4) ON t1.a=t3.a AND t3.a=t4.a),t2 WHERE t1.a=t2.a @endcode Similarly the original query can be rewritten to the query: @code SELECT * FROM (t1,t2) LEFT JOIN (t3,t4) ON t2.a=t4.a AND t3.a=t4.a WHERE t1.a=t2.a @endcode that is equivalent to: @code SELECT * FROM (t2 LEFT JOIN (t3,t4)ON t2.a=t4.a AND t3.a=t4.a), t1 WHERE t1.a=t2.a @endcode Thus, applying equalities from the where condition we basically can get more freedom in performing join operations. Although we don't use this property now, it probably makes sense to use it in the future. @param thd Thread handler @param cond condition to build the multiple equalities for @param[out] retcond Returned condition @param inherited path to all inherited multiple equality items @param do_inherit whether or not to inherit equalities from other parts of the condition @param join_list list of join tables that the condition refers to @param[out] cond_equal_ref pointer to the structure to place built equalities in @returns false if success, true if error */ bool build_equal_items(THD *thd, Item *cond, Item **retcond, COND_EQUAL *inherited, bool do_inherit, List *join_list, COND_EQUAL **cond_equal_ref) { COND_EQUAL *cond_equal = 0; if (cond) { if (build_equal_items_for_cond(thd, cond, &cond, inherited, do_inherit)) return true; cond->update_used_tables(); // update_used_tables() returns void but can stil fail. if (thd->is_error()) return true; const enum Item::Type cond_type = cond->type(); if (cond_type == Item::COND_ITEM && down_cast(cond)->functype() == Item_func::COND_AND_FUNC) cond_equal = &down_cast(cond)->cond_equal; else if (cond_type == Item::FUNC_ITEM && down_cast(cond)->functype() == Item_func::MULT_EQUAL_FUNC) { cond_equal = new (thd->mem_root) COND_EQUAL; if (cond_equal == NULL) return true; cond_equal->current_level.push_back(down_cast(cond)); } } if (cond_equal) { cond_equal->upper_levels = inherited; inherited = cond_equal; } *cond_equal_ref = cond_equal; if (join_list) { TABLE_LIST *table; List_iterator li(*join_list); while ((table = li++)) { if (table->join_cond_optim()) { List *nested_join_list = table->nested_join ? &table->nested_join->join_list : NULL; Item *join_cond; if (build_equal_items(thd, table->join_cond_optim(), &join_cond, inherited, do_inherit, nested_join_list, &table->cond_equal)) return true; table->set_join_cond_optim(join_cond); } } } *retcond = cond; return false; } /** Compare field items by table order in the execution plan. field1 considered as better than field2 if the table containing field1 is accessed earlier than the table containing field2. The function finds out what of two fields is better according this criteria. @param field1 first field item to compare @param field2 second field item to compare @param table_join_idx index to tables determining table order @retval -1 if field1 is better than field2 @retval 1 if field2 is better than field1 @retval 0 otherwise */ static int compare_fields_by_table_order(Item_field *field1, Item_field *field2, JOIN_TAB **table_join_idx) { int cmp = 0; bool outer_ref = 0; if (field1->used_tables() & OUTER_REF_TABLE_BIT) { outer_ref = 1; cmp = -1; } if (field2->used_tables() & OUTER_REF_TABLE_BIT) { outer_ref = 1; cmp++; } if (outer_ref) return cmp; /* table_join_idx is NULL if this function was not called from JOIN::optimize() but from e.g. mysql_delete() or mysql_update(). In these cases there is only one table and both fields belong to it. Example condition where this is the case: t1.fld1=t1.fld2 */ if (!table_join_idx) return 0; // Locate JOIN_TABs thanks to table_join_idx, then compare their index. cmp = table_join_idx[field1->table_ref->tableno()]->idx() - table_join_idx[field2->table_ref->tableno()]->idx(); return cmp < 0 ? -1 : (cmp ? 1 : 0); } /** Generate minimal set of simple equalities equivalent to a multiple equality. The function retrieves the fields of the multiple equality item item_equal and for each field f: - if item_equal contains const it generates the equality f=const_item; - otherwise, if f is not the first field, generates the equality f=item_equal->get_first(). All generated equality are added to the cond conjunction. @param thd the session context @param cond condition to add the generated equality to @param upper_levels structure to access multiple equality of upper levels @param item_equal multiple equality to generate simple equality from @note Before generating an equality function checks that it has not been generated for multiple equalities of the upper levels. E.g. for the following where condition WHERE a=5 AND ((a=b AND b=c) OR c>4) the upper level AND condition will contain =(5,a), while the lower level AND condition will contain =(5,a,b,c). When splitting =(5,a,b,c) into a separate equality predicates we should omit 5=a, as we have it already in the upper level. The following where condition gives us a more complicated case: WHERE t1.a=t2.b AND t3.c=t4.d AND (t2.b=t3.c OR t4.e>5 ...) AND ... Given the tables are accessed in the order t1->t2->t3->t4 for the selected query execution plan the lower level multiple equality =(t1.a,t2.b,t3.c,t4.d) formally should be converted to t1.a=t2.b AND t1.a=t3.c AND t1.a=t4.d. But t1.a=t2.a will be generated for the upper level. Also t3.c=t4.d will be generated there. So only t1.a=t3.c should be left in the lower level. If cond is equal to 0, then not more then one equality is generated and a pointer to it is returned as the result of the function. @return - The condition with generated simple equalities or a pointer to the simple generated equality, if success. - 0, otherwise. */ static Item *eliminate_item_equal(THD *thd, Item *cond, COND_EQUAL *upper_levels, Item_equal *item_equal) { List eq_list; Item *eq_item = NULL; if (((Item *)item_equal)->const_item() && !item_equal->val_int()) return new Item_func_false(); Item *const item_const = item_equal->get_const(); Item_equal_iterator it(*item_equal); if (!item_const) { /* If there is a const item, match all field items with the const item, otherwise match the second and subsequent field items with the first one: */ it++; } Item_field *item_field; // Field to generate equality for. while ((item_field = it++)) { /* Generate an equality of the form: item_field = some previous field in item_equal's list. First see if we really need to generate it: */ Item_equal *const upper = item_field->find_item_equal(upper_levels); if (upper) // item_field is in this upper equality { if (item_const && upper->get_const()) continue; // Const at both levels, no need to generate at current level /* If the upper-level multiple equality contains this item, there is no need to generate the equality, unless item_field belongs to a semi-join nest that is used for Materialization, and refers to tables that are outside of the materialized semi-join nest, As noted in Item_equal::get_subst_item(), subquery materialization does not have this problem. */ JOIN_TAB *const tab = item_field->field->table->reginfo.join_tab; if (!(tab && sj_is_materialize_strategy(tab->get_sj_strategy()))) { Item_field *item_match; Item_equal_iterator li(*item_equal); while ((item_match = li++) != item_field) { if (item_match->find_item_equal(upper_levels) == upper) break; // (item_match, item_field) is also in upper level equality } if (item_match != item_field) continue; } } // ... if (upper). /* item_field should be compared with the head of the multiple equality list. item_field may refer to a table that is within a semijoin materialization nest. In that case, the order of the join_tab entries may look like: ot1 ot2 ot5 SJM(it3 it4) If we have a multiple equality (ot1.c1, ot2.c2, .c it3.c3, it4.c4, ot5.c5), we should generate the following equalities: 1. ot1.c1 = ot2.c2 2. ot1.c1 = .c 3. it3.c3 = it4.c4 4. ot1.c1 = ot5.c5 Equalities 1) and 4) are regular equalities between two outer tables. Equality 2) is an equality that matches the outer query with a materialized temporary table. It is either performed as a lookup into the materialized table (SJM-lookup), or as a condition on the outer table (SJM-scan). Equality 3) is evaluated during semijoin materialization. If there is a const item, match against this one. Otherwise, match against the first field item in the multiple equality, unless the item is within a materialized semijoin nest, in case it will be matched against the first item within the SJM nest. @see JOIN::set_prefix_tables() @see Item_equal::get_subst_item() */ Item *const head = item_const ? item_const : item_equal->get_subst_item(item_field); if (head == item_field) continue; // we have a pair, can generate 'item_field=head' if (eq_item) eq_list.push_back(eq_item); eq_item = new Item_func_eq(item_field, head); if (!eq_item || down_cast(eq_item)->set_cmp_func()) return nullptr; eq_item->quick_fix_field(); if (item_const != nullptr) { eq_item->apply_is_true(); Item::cond_result res; if (fold_condition(thd, eq_item, &eq_item, &res)) return nullptr; if (res == Item::COND_FALSE) { eq_item = new (thd->mem_root) Item_func_false(); if (eq_item == nullptr) return nullptr; return eq_item; // entire AND is false } else if (res == Item::COND_TRUE) { eq_item = new (thd->mem_root) Item_func_true(); if (eq_item == nullptr) return nullptr; } } } // ... while ((item_field= it++)) if (!cond && !eq_list.head()) { if (!eq_item) return new Item_func_true(); return eq_item; } if (eq_item) eq_list.push_back(eq_item); if (!cond) cond = new Item_cond_and(eq_list); else { DBUG_ASSERT(cond->type() == Item::COND_ITEM); if (eq_list.elements) ((Item_cond *)cond)->add_at_head(&eq_list); } cond->quick_fix_field(); cond->update_used_tables(); return cond; } /** Substitute every field reference in a condition by the best equal field and eliminate all multiple equality predicates. The function retrieves the cond condition and for each encountered multiple equality predicate it sorts the field references in it according to the order of tables specified by the table_join_idx parameter. Then it eliminates the multiple equality predicate by replacing it with the conjunction of simple equality predicates equating every field from the multiple equality to the first field in it, or to the constant, if there is any. After this, the function retrieves all other conjuncted predicates and substitutes every field reference by the field reference to the first equal field or equal constant if there are any. @param thd the session context @param cond condition to process @param cond_equal multiple equalities to take into consideration @param table_join_idx index to tables determining field preference @note At the first glance, a full sort of fields in multiple equality seems to be an overkill. Yet it's not the case due to possible new fields in multiple equality item of lower levels. We want the order in them to comply with the order of upper levels. @return The transformed condition, or NULL in case of error */ Item *substitute_for_best_equal_field(THD *thd, Item *cond, COND_EQUAL *cond_equal, JOIN_TAB **table_join_idx) { DBUG_ASSERT(cond->is_bool_func()); if (cond->type() == Item::COND_ITEM) { List *cond_list = ((Item_cond *)cond)->argument_list(); bool and_level = ((Item_cond *)cond)->functype() == Item_func::COND_AND_FUNC; if (and_level) { cond_equal = &((Item_cond_and *)cond)->cond_equal; cond_list->disjoin((List *)&cond_equal->current_level); List_iterator_fast it(cond_equal->current_level); auto cmp = [table_join_idx](Item_field *f1, Item_field *f2) { return compare_fields_by_table_order(f1, f2, table_join_idx); }; Item_equal *item_equal; while ((item_equal = it++)) { item_equal->sort(cmp); } } List_iterator li(*cond_list); Item *item; while ((item = li++)) { Item *new_item = substitute_for_best_equal_field(thd, item, cond_equal, table_join_idx); if (new_item == NULL) return NULL; /* This works OK with PS/SP re-execution as changes are made to the arguments of AND/OR items only */ if (new_item != item) li.replace(new_item); } if (and_level) { List_iterator_fast it(cond_equal->current_level); Item_equal *item_equal; while ((item_equal = it++)) { cond = eliminate_item_equal(thd, cond, cond_equal->upper_levels, item_equal); if (cond == NULL) return NULL; // This occurs when eliminate_item_equal() founds that cond is // always false and substitutes it with a false value. // Due to this, value of item_equal will be 0, so just return it. if (cond->type() != Item::COND_ITEM) break; } } if (cond->type() == Item::COND_ITEM && !((Item_cond *)cond)->argument_list()->elements) cond = cond->val_bool() ? down_cast(new Item_func_true()) : down_cast(new Item_func_false()); } else if (cond->type() == Item::FUNC_ITEM && (down_cast(cond))->functype() == Item_func::MULT_EQUAL_FUNC) { Item_equal *item_equal = down_cast(cond); item_equal->sort([table_join_idx](Item_field *f1, Item_field *f2) { return compare_fields_by_table_order(f1, f2, table_join_idx); }); if (cond_equal && cond_equal->current_level.head() == item_equal) cond_equal = cond_equal->upper_levels; return eliminate_item_equal(thd, 0, cond_equal, item_equal); } else { cond->transform(&Item::replace_equal_field, 0); } return cond; } /** change field = field to field = const for each found field = const in the and_level @param thd Thread handler @param save_list @param and_father @param cond Condition where fields are replaced with constant values @param field The field that will be substituted @param value The substitution value @returns false if success, true if error */ static bool change_cond_ref_to_const(THD *thd, I_List *save_list, Item *and_father, Item *cond, Item *field, Item *value) { DBUG_ASSERT(cond->real_item()->is_bool_func()); if (cond->type() == Item::COND_ITEM) { Item_cond *const item_cond = down_cast(cond); bool and_level = item_cond->functype() == Item_func::COND_AND_FUNC; List_iterator li(*item_cond->argument_list()); Item *item; while ((item = li++)) { if (change_cond_ref_to_const(thd, save_list, and_level ? cond : item, item, field, value)) return true; } return false; } if (cond->eq_cmp_result() == Item::COND_OK) return false; // Not a boolean function Item_bool_func2 *func = down_cast(cond); Item **args = func->arguments(); Item *left_item = args[0]; Item *right_item = args[1]; Item_func::Functype functype = func->functype(); if (right_item->eq(field, 0) && left_item != value && right_item->cmp_context == field->cmp_context && (left_item->result_type() != STRING_RESULT || value->result_type() != STRING_RESULT || left_item->collation.collation == value->collation.collation)) { Item *const clone = value->clone_item(); if (thd->is_error()) return true; if (clone == NULL) return false; clone->collation.set(right_item->collation); thd->change_item_tree(args + 1, clone); func->update_used_tables(); if ((functype == Item_func::EQ_FUNC || functype == Item_func::EQUAL_FUNC) && and_father != cond && !left_item->const_item()) { cond->marker = Item::MARKER_CONST_PROPAG; COND_CMP *const cond_cmp = new COND_CMP(and_father, func); if (cond_cmp == NULL) return true; save_list->push_back(cond_cmp); } if (func->set_cmp_func()) return true; } else if (left_item->eq(field, 0) && right_item != value && left_item->cmp_context == field->cmp_context && (right_item->result_type() != STRING_RESULT || value->result_type() != STRING_RESULT || right_item->collation.collation == value->collation.collation)) { Item *const clone = value->clone_item(); if (thd->is_error()) return true; if (clone == NULL) return false; clone->collation.set(left_item->collation); thd->change_item_tree(args, clone); value = clone; func->update_used_tables(); if ((functype == Item_func::EQ_FUNC || functype == Item_func::EQUAL_FUNC) && and_father != cond && !right_item->const_item()) { args[0] = args[1]; // For easy check thd->change_item_tree(args + 1, value); cond->marker = Item::MARKER_CONST_PROPAG; COND_CMP *const cond_cmp = new COND_CMP(and_father, func); if (cond_cmp == NULL) return true; save_list->push_back(cond_cmp); } if (func->set_cmp_func()) return true; } return false; } /** Propagate constant values in a condition @param thd Thread handler @param save_list @param and_father @param cond Condition for which constant values are propagated @returns false if success, true if error */ static bool propagate_cond_constants(THD *thd, I_List *save_list, Item *and_father, Item *cond) { DBUG_ASSERT(cond->real_item()->is_bool_func()); if (cond->type() == Item::COND_ITEM) { Item_cond *const item_cond = down_cast(cond); bool and_level = item_cond->functype() == Item_func::COND_AND_FUNC; List_iterator_fast li(*item_cond->argument_list()); Item *item; I_List save; while ((item = li++)) { if (propagate_cond_constants(thd, &save, and_level ? cond : item, item)) return true; } if (and_level) { // Handle other found items I_List_iterator cond_itr(save); COND_CMP *cond_cmp; while ((cond_cmp = cond_itr++)) { Item **args = cond_cmp->cmp_func->arguments(); if (!args[0]->const_item() && change_cond_ref_to_const(thd, &save, cond_cmp->and_level, cond_cmp->and_level, args[0], args[1])) return true; } } } else if (and_father != cond && cond->marker != Item::MARKER_CONST_PROPAG) // In a AND group { Item_func *func; if (cond->type() == Item::FUNC_ITEM && (func = down_cast(cond)) && (func->functype() == Item_func::EQ_FUNC || func->functype() == Item_func::EQUAL_FUNC)) { Item **args = func->arguments(); bool left_const = args[0]->const_item(); bool right_const = args[1]->const_item(); if (!(left_const && right_const) && args[0]->result_type() == args[1]->result_type()) { if (right_const) { if (resolve_const_item(thd, &args[1], args[0])) return true; func->update_used_tables(); if (change_cond_ref_to_const(thd, save_list, and_father, and_father, args[0], args[1])) return true; } else if (left_const) { if (resolve_const_item(thd, &args[0], args[1])) return true; func->update_used_tables(); if (change_cond_ref_to_const(thd, save_list, and_father, and_father, args[1], args[0])) return true; } } } } return false; } /** Assign each nested join structure a bit in nested_join_map. @param join_list List of tables @param first_unused Number of first unused bit in nested_join_map before the call @note This function is called after simplify_joins(), when there are no redundant nested joins. We cannot have more nested joins in a query block than there are tables, so as long as the number of bits in nested_join_map is not less than the maximum number of tables in a query block, nested_join_map can never overflow. @return First unused bit in nested_join_map after the call. */ uint build_bitmap_for_nested_joins(List *join_list, uint first_unused) { DBUG_TRACE; List_iterator li(*join_list); TABLE_LIST *table; while ((table = li++)) { NESTED_JOIN *nested_join; if ((nested_join = table->nested_join)) { // We should have a join condition or a semi-join condition or both DBUG_ASSERT((table->join_cond() != NULL) || table->is_sj_nest()); nested_join->nj_map = 0; nested_join->nj_total = 0; /* We only record nested join information for outer join nests. Tables belonging in semi-join nests are recorded in the embedding outer join nest, if one exists. */ if (table->join_cond()) { DBUG_ASSERT(first_unused < sizeof(nested_join_map) * 8); nested_join->nj_map = (nested_join_map)1 << first_unused++; nested_join->nj_total = nested_join->join_list.elements; } else if (table->is_sj_nest()) { NESTED_JOIN *const outer_nest = table->embedding ? table->embedding->nested_join : NULL; /* The semi-join nest has already been counted into the table count for the outer join nest as one table, so subtract 1 from the table count. */ if (outer_nest) outer_nest->nj_total += (nested_join->join_list.elements - 1); } else DBUG_ASSERT(false); first_unused = build_bitmap_for_nested_joins(&nested_join->join_list, first_unused); } } return first_unused; } /** Update the dependency map for the tables. */ void JOIN::update_depend_map() { ASSERT_BEST_REF_IN_JOIN_ORDER(this); for (uint tableno = 0; tableno < tables; tableno++) { JOIN_TAB *const tab = best_ref[tableno]; TABLE_REF *const ref = &tab->ref(); table_map depend_map = 0; Item **item = ref->items; for (uint i = 0; i < ref->key_parts; i++, item++) depend_map |= (*item)->used_tables(); depend_map &= ~PSEUDO_TABLE_BITS; ref->depend_map = depend_map; for (JOIN_TAB **tab2 = map2table; depend_map; tab2++, depend_map >>= 1) { if (depend_map & 1) ref->depend_map |= (*tab2)->ref().depend_map; } } } /** Update the dependency map for the sort order. */ void JOIN::update_depend_map(ORDER *order) { DBUG_TRACE; for (; order; order = order->next) { table_map depend_map; order->item[0]->update_used_tables(); order->depend_map = depend_map = order->item[0]->used_tables() & ~INNER_TABLE_BIT; order->used = 0; // Not item_sum(), RAND() and no reference to table outside of sub select if (!(order->depend_map & (OUTER_REF_TABLE_BIT | RAND_TABLE_BIT)) && !order->item[0]->has_aggregation()) { for (JOIN_TAB **tab = map2table; depend_map; tab++, depend_map >>= 1) { if (depend_map & 1) order->depend_map |= (*tab)->ref().depend_map; } } } } /** Update equalities and keyuse references after semi-join materialization strategy is chosen. @details For each multiple equality that contains a field that is selected from a subquery, and that subquery is executed using a semi-join materialization strategy, add the corresponding column in the materialized temporary table to the equality. For each injected semi-join equality that is not converted to multiple equality, replace the reference to the expression selected from the subquery with the corresponding column in the temporary table. This is needed to properly reflect the equalities that involve injected semi-join equalities when materialization strategy is chosen. @see eliminate_item_equal() for how these equalities are used to generate correct equality predicates. The MaterializeScan semi-join strategy requires some additional processing: All primary tables after the materialized temporary table must be inspected for keyuse objects that point to expressions from the subquery tables. These references must be replaced with references to corresponding columns in the materialized temporary table instead. Those primary tables using ref access will thus be made to depend on the materialized temporary table instead of the subquery tables. Only the injected semi-join equalities need this treatment, other predicates will be handled correctly by the regular item substitution process. @return False if success, true if error */ bool JOIN::update_equalities_for_sjm() { ASSERT_BEST_REF_IN_JOIN_ORDER(this); List_iterator it(sjm_exec_list); Semijoin_mat_exec *sjm_exec; while ((sjm_exec = it++)) { TABLE_LIST *const sj_nest = sjm_exec->sj_nest; Item *cond; /* Conditions involving SJ-inner tables are only to be found in the closest nest's condition, which may be an AJ nest, a LEFT JOIN nest, or the WHERE clause. */ if (sj_nest->is_aj_nest()) cond = sj_nest->join_cond_optim(); else if (sj_nest->outer_join_nest()) cond = sj_nest->outer_join_nest()->join_cond_optim(); else cond = where_cond; if (!cond) continue; uchar *dummy = NULL; cond = cond->compile(&Item::equality_substitution_analyzer, &dummy, &Item::equality_substitution_transformer, (uchar *)sj_nest); if (cond == NULL) return true; cond->update_used_tables(); // Loop over all primary tables that follow the materialized table for (uint j = sjm_exec->mat_table_index + 1; j < primary_tables; j++) { JOIN_TAB *const tab = best_ref[j]; for (Key_use *keyuse = tab->position()->key; keyuse && keyuse->table_ref == tab->table_ref && keyuse->key == tab->position()->key->key; keyuse++) { List_iterator it(sj_nest->nested_join->sj_inner_exprs); Item *old; uint fieldno = 0; while ((old = it++)) { if (old->real_item()->eq(keyuse->val->real_item(), false)) { /* Replace the expression selected from the subquery with the corresponding column of the materialized temporary table. */ keyuse->val = sj_nest->nested_join->sjm.mat_fields[fieldno]; keyuse->used_tables = keyuse->val->used_tables(); break; } fieldno++; } } } } return false; } /** Assign set of available (prefix) tables to all tables in query block. Also set added tables, ie the tables added in each JOIN_TAB compared to the previous JOIN_TAB. This function must be called for every query block after the table order has been determined. */ void JOIN::set_prefix_tables() { ASSERT_BEST_REF_IN_JOIN_ORDER(this); DBUG_ASSERT(!plan_is_const()); /* The const tables are available together with the first non-const table in the join order. */ table_map const initial_tables_map = const_table_map | (allow_outer_refs ? OUTER_REF_TABLE_BIT : 0); table_map current_tables_map = initial_tables_map; table_map prev_tables_map = (table_map)0; table_map saved_tables_map = (table_map)0; JOIN_TAB *last_non_sjm_tab = NULL; // Track the last non-sjm table for (uint i = const_tables; i < tables; i++) { JOIN_TAB *const tab = best_ref[i]; if (!tab->table()) continue; /* Tables that are within SJ-Materialization nests cannot have their conditions referring to preceding non-const tables. - If we're looking at the first SJM table, reset current_tables_map to refer to only allowed tables @see Item_equal::get_subst_item() @see eliminate_item_equal() */ if (sj_is_materialize_strategy(tab->get_sj_strategy())) { const table_map sjm_inner_tables = tab->emb_sj_nest->sj_inner_tables; if (!(sjm_inner_tables & current_tables_map)) { saved_tables_map = current_tables_map; current_tables_map = initial_tables_map; prev_tables_map = (table_map)0; } current_tables_map |= tab->table_ref->map(); tab->set_prefix_tables(current_tables_map, prev_tables_map); prev_tables_map = current_tables_map; if (!(sjm_inner_tables & ~current_tables_map)) { // At the end of a semi-join materialization nest, restore previous map current_tables_map = saved_tables_map; prev_tables_map = last_non_sjm_tab ? last_non_sjm_tab->prefix_tables() : (table_map)0; } } else { last_non_sjm_tab = tab; current_tables_map |= tab->table_ref->map(); tab->set_prefix_tables(current_tables_map, prev_tables_map); prev_tables_map = current_tables_map; } } /* Random expressions must be added to the last table's condition. It solves problem with queries like SELECT * FROM t1 WHERE rand() > 0.5 */ if (last_non_sjm_tab != NULL) last_non_sjm_tab->add_prefix_tables(RAND_TABLE_BIT); } /** Calculate best possible join order and initialize the join structure. @return true if success, false if error. The JOIN object is populated with statistics about the query, and a plan with table order and access method selection is made. The list of tables to be optimized is taken from select_lex->leaf_tables. JOIN::where_cond is also used in the optimization. As a side-effect, JOIN::keyuse_array is populated with key_use information. Here is an overview of the logic of this function: - Initialize JOIN data structures and setup basic dependencies between tables. - Update dependencies based on join information. - Make key descriptions (update_ref_and_keys()). - Pull out semi-join tables based on table dependencies. - Extract tables with zero or one rows as const tables. - Read contents of const tables, substitute columns from these tables with actual data. Also keep track of empty tables vs. one-row tables. - After const table extraction based on row count, more tables may have become functionally dependent. Extract these as const tables. - Add new sargable predicates based on retrieved const values. - Calculate number of rows to be retrieved from each table. - Calculate cost of potential semi-join materializations. - Calculate best possible join order based on available statistics. - Fill in remaining information for the generated join order. */ bool JOIN::make_join_plan() { DBUG_TRACE; SARGABLE_PARAM *sargables = NULL; Opt_trace_context *const trace = &thd->opt_trace; if (init_planner_arrays()) // Create and initialize the arrays return true; // Outer join dependencies were initialized above, now complete the analysis. if (select_lex->outer_join || select_lex->is_recursive()) { if (propagate_dependencies()) { /* Catch illegal join order. SQL2011 forbids: WITH RECURSIVE rec AS ( ... UNION ALL SELECT ... FROM tbl LEFT JOIN rec ON...)c... MySQL also forbids the same query with STRAIGHT_JOIN instead of LEFT JOIN, because the algorithm of with-recursive imposes that "rec" be first in plan, i.e. "tbl" depends on "rec", but STRAIGHT_JOIN imposes the opposite dependency. */ DBUG_ASSERT(select_lex->is_recursive()); my_error(ER_CTE_RECURSIVE_FORBIDDEN_JOIN_ORDER, MYF(0), select_lex->recursive_reference->alias); return true; } init_key_dependencies(); } if (unlikely(trace->is_started())) trace_table_dependencies(trace, join_tab, primary_tables); // Build the key access information, which is the basis for ref access. if (where_cond || select_lex->outer_join) { if (update_ref_and_keys(thd, &keyuse_array, join_tab, tables, where_cond, ~select_lex->outer_join, select_lex, &sargables)) return true; } /* Pull out semi-join tables based on dependencies. Dependencies are valid throughout the lifetime of a query, so this operation can be performed on the first optimization only. */ if (!select_lex->sj_pullout_done && select_lex->sj_nests.elements && pull_out_semijoin_tables(this)) return true; select_lex->sj_pullout_done = true; const uint sj_nests = select_lex->sj_nests.elements; // Changed by pull-out if (!(select_lex->active_options() & OPTION_NO_CONST_TABLES)) { // Detect tables that are const (0 or 1 row) and read their contents. if (extract_const_tables()) return true; // Detect tables that are functionally dependent on const values. if (extract_func_dependent_tables()) return true; } // Possibly able to create more sargable predicates from const rows. if (const_tables && sargables) update_sargable_from_const(sargables); // Make a first estimate of the fanout for each table in the query block. if (estimate_rowcount()) return true; /* Apply join order hints, with the exception of JOIN_FIXED_ORDER and STRAIGHT_JOIN. */ if (select_lex->opt_hints_qb && !(select_lex->active_options() & SELECT_STRAIGHT_JOIN)) select_lex->opt_hints_qb->apply_join_order_hints(this); if (sj_nests) { set_semijoin_embedding(); select_lex->update_semijoin_strategies(thd); } if (!plan_is_const()) optimize_keyuse(); allow_outer_refs = true; if (sj_nests && optimize_semijoin_nests_for_materialization(this)) return true; // Choose the table order based on analysis done so far. if (Optimize_table_order(thd, this, NULL).choose_table_order()) return true; DBUG_EXECUTE_IF("bug13820776_1", thd->killed = THD::KILL_QUERY;); if (thd->killed || thd->is_error()) return true; // If this is a subquery, decide between In-to-exists and materialization if (unit->item && decide_subquery_strategy()) return true; refine_best_rowcount(); if (!(thd->variables.option_bits & OPTION_BIG_SELECTS) && best_read > (double)thd->variables.max_join_size && !thd->lex->is_explain()) { /* purecov: inspected */ my_error(ER_TOO_BIG_SELECT, MYF(0)); error = -1; return 1; } positions = NULL; // But keep best_positions for get_best_combination // Generate an execution plan from the found optimal join order. if (get_best_combination()) return true; // Cleanup after update_ref_and_keys has added keys for derived tables. if (select_lex->materialized_derived_table_count || select_lex->table_func_count) finalize_derived_keys(); // No need for this struct after new JOIN_TAB array is set up. best_positions = NULL; // Some called function may still set error status unnoticed if (thd->is_error()) return true; // There is at least one empty const table if (const_table_map != found_const_table_map) zero_result_cause = "no matching row in const table"; return false; } /** Initialize scratch arrays for the join order optimization @returns false if success, true if error @note If something fails during initialization, JOIN::cleanup() will free anything that has been partially allocated and set up. Arrays are created in the execution mem_root, so they will be deleted automatically when the mem_root is re-initialized. */ bool JOIN::init_planner_arrays() { // Up to one extra slot per semi-join nest is needed (if materialized) const uint sj_nests = select_lex->sj_nests.elements; const uint table_count = select_lex->leaf_table_count; DBUG_ASSERT(primary_tables == 0 && tables == 0); if (!(join_tab = alloc_jtab_array(thd, table_count))) return true; /* We add 2 cells: - because planning stage uses 0-termination so needs +1 - because after get_best_combination, we don't use 0-termination but need +2, to host at most 2 tmp sort/group/distinct tables. */ if (!(best_ref = (JOIN_TAB **)thd->alloc( sizeof(JOIN_TAB *) * (table_count + sj_nests + 2 + m_windows.elements)))) return true; // sort/group tmp tables have no map if (!(map2table = (JOIN_TAB **)thd->alloc(sizeof(JOIN_TAB *) * (table_count + sj_nests)))) return true; if (!(positions = new (thd->mem_root) POSITION[table_count])) return true; if (!(best_positions = new (thd->mem_root) POSITION[table_count + sj_nests])) return true; /* Initialize data structures for tables to be joined. Initialize dependencies between tables. */ JOIN_TAB **best_ref_p = best_ref; TABLE_LIST *tl = select_lex->leaf_tables; for (JOIN_TAB *tab = join_tab; tl; tab++, tl = tl->next_leaf, best_ref_p++) { *best_ref_p = tab; TABLE *const table = tl->table; tab->table_ref = tl; tab->set_table(table); const int err = tl->fetch_number_of_rows(); // Initialize the cost model for the table table->init_cost_model(cost_model()); DBUG_EXECUTE_IF("bug11747970_raise_error", { if (!err) { my_error(ER_UNKNOWN_ERROR, MYF(0)); return true; } }); if (err) { table->file->print_error(err, MYF(0)); return true; } all_table_map |= tl->map(); tab->set_join(this); tab->dependent = tl->dep_tables; // Initialize table dependencies if (select_lex->is_recursive()) { if (select_lex->recursive_reference != tl) // Recursive reference must go first tab->dependent |= select_lex->recursive_reference->map(); else { // Recursive reference mustn't use any index table->covering_keys.clear_all(); table->keys_in_use_for_group_by.clear_all(); table->keys_in_use_for_order_by.clear_all(); } } if (tl->schema_table) table->file->stats.records = 2; table->quick_condition_rows = table->file->stats.records; tab->init_join_cond_ref(tl); if (tl->outer_join_nest()) { // tab belongs to a nested join, maybe to several embedding joins tab->embedding_map = 0; for (TABLE_LIST *embedding = tl->embedding; embedding; embedding = embedding->embedding) { NESTED_JOIN *const nested_join = embedding->nested_join; tab->embedding_map |= nested_join->nj_map; tab->dependent |= embedding->dep_tables; } } else if (tab->join_cond()) { // tab is the only inner table of an outer join tab->embedding_map = 0; for (TABLE_LIST *embedding = tl->embedding; embedding; embedding = embedding->embedding) tab->embedding_map |= embedding->nested_join->nj_map; } if (tl->is_derived() && tl->derived_unit()->m_lateral_deps) has_lateral = true; tables++; // Count number of initialized tables } primary_tables = tables; *best_ref_p = NULL; // Last element of array must be NULL return false; } /** Propagate dependencies between tables due to outer join relations. @returns false if success, true if error Build transitive closure for relation 'to be dependent on'. This will speed up the plan search for many cases with outer joins, as well as allow us to catch illegal cross references. Warshall's algorithm is used to build the transitive closure. As we may restart the outer loop upto 'table_count' times, the complexity of the algorithm is O((number of tables)^3). However, most of the iterations will be shortcircuited when there are no dependencies to propagate. */ bool JOIN::propagate_dependencies() { for (uint i = 0; i < tables; i++) { if (!join_tab[i].dependent) continue; // Add my dependencies to other tables depending on me uint j; JOIN_TAB *tab; for (j = 0, tab = join_tab; j < tables; j++, tab++) { if (tab->dependent & join_tab[i].table_ref->map()) { const table_map was_dependent = tab->dependent; tab->dependent |= join_tab[i].dependent; /* If we change dependencies for a table we already have processed: Redo dependency propagation from this table. */ if (i > j && tab->dependent != was_dependent) { i = j - 1; break; } } } } JOIN_TAB *const tab_end = join_tab + tables; for (JOIN_TAB *tab = join_tab; tab < tab_end; tab++) { if ((tab->dependent & tab->table_ref->map())) return true; } return false; } /** Extract const tables based on row counts. @returns false if success, true if error This extraction must be done for each execution. Tables containing exactly zero or one rows are marked as const, but notice the additional constraints checked below. Tables that are extracted have their rows read before actual execution starts and are placed in the beginning of the join_tab array. Thus, they do not take part in join order optimization process, which can significantly reduce the optimization time. The data read from these tables can also be regarded as "constant" throughout query execution, hence the column values can be used for additional constant propagation and extraction of const tables based on eq-ref properties. The tables are given the type JT_SYSTEM. */ bool JOIN::extract_const_tables() { enum enum_const_table_extraction { extract_no_table = 0, extract_empty_table = 1, extract_const_table = 2 }; JOIN_TAB *const tab_end = join_tab + tables; for (JOIN_TAB *tab = join_tab; tab < tab_end; tab++) { TABLE *const table = tab->table(); TABLE_LIST *const tl = tab->table_ref; enum enum_const_table_extraction extract_method = extract_const_table; const bool all_partitions_pruned_away = table->all_partitions_pruned_away; if (tl->outer_join_nest()) { /* Table belongs to a nested join, no candidate for const table extraction. */ extract_method = extract_no_table; } else if (tl->embedding && tl->embedding->is_sj_or_aj_nest()) { /* Table belongs to a semi-join. We do not currently pull out const tables from semi-join nests. */ extract_method = extract_no_table; } else if (tab->join_cond()) { // tab is the only inner table of an outer join, extract empty tables extract_method = extract_empty_table; } switch (extract_method) { case extract_no_table: break; case extract_empty_table: // Extract tables with zero rows, but only if statistics are exact if ((table->file->stats.records == 0 || all_partitions_pruned_away) && (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT)) mark_const_table(tab, NULL); break; case extract_const_table: /* Extract tables with zero or one rows, but do not extract tables that 1. are dependent upon other tables, or 2. have no exact statistics, or 3. are full-text searched */ if ((table->s->system || table->file->stats.records <= 1 || all_partitions_pruned_away) && !tab->dependent && // 1 (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT) && // 2 !table->fulltext_searched) // 3 mark_const_table(tab, NULL); break; } } // Read const tables (tables matching no more than 1 rows) if (!const_tables) return false; for (POSITION *p_pos = positions, *p_end = p_pos + const_tables; p_pos < p_end; p_pos++) { JOIN_TAB *const tab = p_pos->table; const int status = join_read_const_table(tab, p_pos); if (status > 0) return true; else if (status == 0) { found_const_table_map |= tab->table_ref->map(); tab->table_ref->optimized_away = true; } } return false; } /** Extract const tables based on functional dependencies. @returns false if success, true if error This extraction must be done for each execution. Mark as const the tables that - are functionally dependent on constant values, or - are inner tables of an outer join and contain exactly zero or one rows Tables that are extracted have their rows read before actual execution starts and are placed in the beginning of the join_tab array, just as described for JOIN::extract_const_tables(). The tables are given the type JT_CONST. */ bool JOIN::extract_func_dependent_tables() { // loop until no more const tables are found bool ref_changed; // Tables referenced by others; if they're const the others may be too. table_map found_ref; do { more_const_tables_found: ref_changed = false; found_ref = 0; // Loop over all tables that are not already determined to be const for (JOIN_TAB **pos = best_ref + const_tables; *pos; pos++) { JOIN_TAB *const tab = *pos; TABLE *const table = tab->table(); TABLE_LIST *const tl = tab->table_ref; /* If equi-join condition by a key is null rejecting and after a substitution of a const table the key value happens to be null then we can state that there are no matches for this equi-join. */ Key_use *keyuse = tab->keyuse(); if (keyuse && tab->join_cond() && !tab->embedding_map) { /* When performing an outer join operation if there are no matching rows for the single row of the outer table all the inner tables are to be null complemented and thus considered as constant tables. Here we apply this consideration to the case of outer join operations with a single inner table only because the case with nested tables would require a more thorough analysis. TODO. Apply single row substitution to null complemented inner tables for nested outer join operations. */ while (keyuse->table_ref == tl) { if (!(keyuse->val->used_tables() & ~const_table_map) && keyuse->val->is_null() && keyuse->null_rejecting) { table->set_null_row(); table->const_table = true; found_const_table_map |= tl->map(); mark_const_table(tab, keyuse); goto more_const_tables_found; } keyuse++; } } if (tab->dependent) // If dependent on some table { // All dependent tables must be const if (tab->dependent & ~const_table_map) { found_ref |= tab->dependent; continue; } /* Mark a dependent table as constant if 1. it has exactly zero or one rows (it is a system table), and 2. it is not within a nested outer join, and 3. it does not have an expensive outer join condition. This is because we have to determine whether an outer-joined table has a real row or a null-extended row in the optimizer phase. We have no possibility to evaluate its join condition at execution time, when it is marked as a system table. */ if (table->file->stats.records <= 1L && // 1 (table->file->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT) && // 1 !tl->outer_join_nest() && // 2 !(tab->join_cond() && tab->join_cond()->is_expensive())) // 3 { // system table mark_const_table(tab, NULL); const int status = join_read_const_table(tab, positions + const_tables - 1); if (status > 0) return true; else if (status == 0) found_const_table_map |= tl->map(); continue; } } // Check if table can be read by key or table only uses const refs if ((keyuse = tab->keyuse())) { while (keyuse->table_ref == tl) { Key_use *const start_keyuse = keyuse; const uint key = keyuse->key; tab->keys().set_bit(key); // QQ: remove this ? table_map refs = 0; Key_map const_ref, eq_part; do { if (keyuse->val->type() != Item::NULL_ITEM && !keyuse->optimize) { if (!((~found_const_table_map) & keyuse->used_tables)) const_ref.set_bit(keyuse->keypart); else refs |= keyuse->used_tables; eq_part.set_bit(keyuse->keypart); } keyuse++; } while (keyuse->table_ref == tl && keyuse->key == key); /* Extract const tables with proper key dependencies. Exclude tables that 1. are full-text searched, or 2. are part of nested outer join, or 3. are part of semi-join, or 4. have an expensive outer join condition. 5. are blocked by handler for const table optimize. 6. are not going to be used, typically because they are streamed instead of materialized (see SELECT_LEX_UNIT::can_materialize_directly_into_result()). */ if (eq_part.is_prefix(table->key_info[key].user_defined_key_parts) && !table->fulltext_searched && // 1 !tl->outer_join_nest() && // 2 !(tl->embedding && tl->embedding->is_sj_or_aj_nest()) && // 3 !(tab->join_cond() && tab->join_cond()->is_expensive()) && // 4 !(table->file->ha_table_flags() & HA_BLOCK_CONST_TABLE) && // 5 table->is_created()) { // 6 if (table->key_info[key].flags & HA_NOSAME) { if (const_ref == eq_part) { // Found everything for ref. ref_changed = true; mark_const_table(tab, start_keyuse); if (create_ref_for_key(this, tab, start_keyuse, found_const_table_map)) return true; const int status = join_read_const_table(tab, positions + const_tables - 1); if (status > 0) return true; else if (status == 0) found_const_table_map |= tl->map(); break; } else found_ref |= refs; // Table is const if all refs are const } else if (const_ref == eq_part) tab->const_keys.set_bit(key); } } } } } while /* A new const table appeared, that is referenced by others, so re-check others: */ ((const_table_map & found_ref) && ref_changed); return false; } /** Update info on indexes that can be used for search lookups as reading const tables may has added new sargable predicates. */ void JOIN::update_sargable_from_const(SARGABLE_PARAM *sargables) { for (; sargables->field; sargables++) { Field *const field = sargables->field; JOIN_TAB *const tab = field->table->reginfo.join_tab; Key_map possible_keys = field->key_start; possible_keys.intersect(field->table->keys_in_use_for_query); bool is_const = true; for (uint j = 0; j < sargables->num_values; j++) is_const &= sargables->arg_value[j]->const_item(); if (is_const) { tab->const_keys.merge(possible_keys); tab->keys().merge(possible_keys); } } } /** Estimate the number of matched rows for each joined table. Set up range scan for tables that have proper predicates. @returns false if success, true if error */ bool JOIN::estimate_rowcount() { Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_array trace_records(trace, "rows_estimation"); JOIN_TAB *const tab_end = join_tab + tables; for (JOIN_TAB *tab = join_tab; tab < tab_end; tab++) { const Cost_model_table *const cost_model = tab->table()->cost_model(); Opt_trace_object trace_table(trace); trace_table.add_utf8_table(tab->table_ref); if (tab->type() == JT_SYSTEM || tab->type() == JT_CONST) { trace_table.add("rows", 1) .add("cost", 1) .add_alnum("table_type", (tab->type() == JT_SYSTEM) ? "system" : "const") .add("empty", tab->table()->has_null_row()); // Only one matching row and one block to read tab->set_records(tab->found_records = 1); tab->worst_seeks = cost_model->page_read_cost(1.0); tab->read_time = tab->worst_seeks; continue; } // Approximate number of found rows and cost to read them tab->set_records(tab->found_records = tab->table()->file->stats.records); const Cost_estimate table_scan_time = tab->table()->file->table_scan_cost(); tab->read_time = table_scan_time.total_cost(); /* Set a max value for the cost of seek operations we can expect when using key lookup. This can't be too high as otherwise we are likely to use table scan. */ tab->worst_seeks = min(cost_model->page_read_cost((double)tab->found_records / 10), tab->read_time * 3); const double min_worst_seek = cost_model->page_read_cost(2.0); if (tab->worst_seeks < min_worst_seek) // Fix for small tables tab->worst_seeks = min_worst_seek; /* Add to tab->const_keys the indexes for which all group fields or all select distinct fields participate in one index. Add to tab->skip_scan_keys indexes which can be used for skip scan access if no aggregates are present. */ add_loose_index_scan_and_skip_scan_keys(this, tab); /* Perform range analysis if there are keys it could use (1). Don't do range analysis if on the inner side of an outer join (2). Do range analysis if on the inner side of a semi-join (3). */ TABLE_LIST *const tl = tab->table_ref; if ((!tab->const_keys.is_clear_all() || !tab->skip_scan_keys.is_clear_all()) && // (1) (!tl->embedding || // (2) (tl->embedding && tl->embedding->is_sj_or_aj_nest()))) // (3) { /* This call fills tab->quick() with the best QUICK access method possible for this table, and only if it's better than table scan. It also fills tab->needed_reg. */ ha_rows records = get_quick_record_count(thd, tab, row_limit); if (records == 0 && thd->is_error()) return true; /* Check for "impossible range", but make sure that we do not attempt to mark semi-joined tables as "const" (only semi-joined tables that are functionally dependent can be marked "const", and subsequently pulled out of their semi-join nests). */ if (records == 0 && tab->table()->reginfo.impossible_range && (!(tl->embedding && tl->embedding->is_sj_or_aj_nest()))) { /* Impossible WHERE condition or join condition In case of join cond, mark that one empty NULL row is matched. In case of WHERE, don't set found_const_table_map to get the caller to abort with a zero row result. */ mark_const_table(tab, NULL); tab->set_type(JT_CONST); // Override setting made in mark_const_table() if (tab->join_cond()) { // Generate an empty row trace_table.add("returning_empty_null_row", true) .add_alnum("cause", "impossible_on_condition"); found_const_table_map |= tl->map(); tab->table()->set_null_row(); // All fields are NULL } else { trace_table.add("rows", 0).add_alnum("cause", "impossible_where_condition"); } } if (records != HA_POS_ERROR) { tab->found_records = records; tab->read_time = tab->quick() ? tab->quick()->cost_est.total_cost() : 0.0; } } else { Opt_trace_object(trace, "table_scan") .add("rows", tab->found_records) .add("cost", tab->read_time); } } return false; } /** Set semi-join embedding join nest pointers. Set pointer to embedding semi-join nest for all semi-joined tables. This is the closest semi-join or anti-join nest. Note that this must be done for every table inside all semi-join nests, even for tables within outer join nests embedded in semi-join nests. A table can never be part of multiple semi-join nests, hence no ambiguities can ever occur. Note also that the pointer is not set for TABLE_LIST objects that are outer join nests within semi-join nests. */ void JOIN::set_semijoin_embedding() { DBUG_ASSERT(!select_lex->sj_nests.is_empty()); JOIN_TAB *const tab_end = join_tab + primary_tables; for (JOIN_TAB *tab = join_tab; tab < tab_end; tab++) { tab->emb_sj_nest = nullptr; for (TABLE_LIST *tl = tab->table_ref; tl->embedding; tl = tl->embedding) { if (tl->embedding->is_sj_or_aj_nest()) { DBUG_ASSERT(!tab->emb_sj_nest); tab->emb_sj_nest = tl->embedding; // Let the up-walk continue, to assert there's no AJ/SJ nest above. } } } } /** @brief Check if semijoin's compared types allow materialization. @param[inout] sj_nest Semi-join nest containing information about correlated expressions. Set nested_join->sjm.scan_allowed to true if MaterializeScan strategy allowed. Set nested_join->sjm.lookup_allowed to true if MaterializeLookup strategy allowed @details This is a temporary fix for BUG#36752. There are two subquery materialization strategies for semijoin: 1. Materialize and do index lookups in the materialized table. See BUG#36752 for description of restrictions we need to put on the compared expressions. In addition, since indexes are not supported for BLOB columns, this strategy can not be used if any of the columns in the materialized table will be BLOB/GEOMETRY columns. (Note that also columns for non-BLOB values that may be greater in size than CONVERT_IF_BIGGER_TO_BLOB, will be represented as BLOB columns.) 2. Materialize and then do a full scan of the materialized table. The same criteria as for MaterializeLookup are applied, except that BLOB/GEOMETRY columns are allowed. */ static void semijoin_types_allow_materialization(TABLE_LIST *sj_nest) { DBUG_TRACE; DBUG_ASSERT(sj_nest->nested_join->sj_outer_exprs.elements == sj_nest->nested_join->sj_inner_exprs.elements); if (sj_nest->nested_join->sj_outer_exprs.elements > MAX_REF_PARTS || sj_nest->nested_join->sj_outer_exprs.elements == 0) { // building an index is impossible sj_nest->nested_join->sjm.scan_allowed = false; sj_nest->nested_join->sjm.lookup_allowed = false; return; } List_iterator it1(sj_nest->nested_join->sj_outer_exprs); List_iterator it2(sj_nest->nested_join->sj_inner_exprs); sj_nest->nested_join->sjm.scan_allowed = true; sj_nest->nested_join->sjm.lookup_allowed = true; bool blobs_involved = false; Item *outer, *inner; uint total_lookup_index_length = 0; uint max_key_length, max_key_part_length, max_key_parts; /* Maximum lengths for keys and key parts that are supported by the temporary table storage engine(s). */ get_max_key_and_part_length(&max_key_length, &max_key_part_length, &max_key_parts); while (outer = it1++, inner = it2++) { DBUG_ASSERT(outer->real_item() && inner->real_item()); if (!types_allow_materialization(outer, inner)) { sj_nest->nested_join->sjm.scan_allowed = false; sj_nest->nested_join->sjm.lookup_allowed = false; return; } blobs_involved |= inner->is_blob_field(); // Calculate the index length of materialized table const uint lookup_index_length = get_key_length_tmp_table(inner); if (lookup_index_length > max_key_part_length) sj_nest->nested_join->sjm.lookup_allowed = false; total_lookup_index_length += lookup_index_length; } if (total_lookup_index_length > max_key_length) sj_nest->nested_join->sjm.lookup_allowed = false; if (blobs_involved) sj_nest->nested_join->sjm.lookup_allowed = false; DBUG_PRINT("info", ("semijoin_types_allow_materialization: ok, allowed")); } /** Index dive can be skipped if the following conditions are satisfied: F1) For a single table query: a) FORCE INDEX applies to a single index. b) No subquery is present. c) Fulltext Index is not involved. d) No GROUP-BY or DISTINCT clause. e) No ORDER-BY clause. F2) Not applicable to multi-table query. F3) This optimization is not applicable to EXPLAIN queries. @param tab JOIN_TAB object. @param thd THD object. */ static bool check_skip_records_in_range_qualification(JOIN_TAB *tab, THD *thd) { SELECT_LEX *select = thd->lex->current_select(); TABLE *table = tab->table(); return ((table->force_index && table->pos_in_table_list->index_hints->elements == 1) && // F1.a select->parent_lex->is_single_level_stmt() && // F1.b !select->has_ft_funcs() && // F1.c (!select->is_grouped() && !select->is_distinct()) && // F1.d !select->is_ordered() && // F1.e select->join_list->elements == 1 && // F2 !thd->lex->is_explain()); // F3 } /***************************************************************************** Create JOIN_TABS, make a guess about the table types, Approximate how many records will be used in each table *****************************************************************************/ /** Returns estimated number of rows that could be fetched by given access method. The function calls the range optimizer to estimate the cost of the cheapest QUICK_* index access method to scan one or several of the 'keys' using the conditions 'select->cond'. The range optimizer compares several different types of 'quick select' methods (range scan, index merge, loose index scan) and selects the cheapest one. If the best index access method is cheaper than a table- and an index scan, then the range optimizer also constructs the corresponding QUICK_* object and assigns it to select->quick. In most cases this is the QUICK_* object used at later (optimization and execution) phases. @param thd Session that runs the query. @param tab JOIN_TAB of source table. @param limit maximum number of rows to select. @note In case of valid range, a QUICK_SELECT_I object will be constructed and saved in select->quick. @return Estimated number of result rows selected from 'tab'. @retval HA_POS_ERROR For derived tables/views or if an error occur. @retval 0 If impossible query (i.e. certainly no rows will be selected.) */ static ha_rows get_quick_record_count(THD *thd, JOIN_TAB *tab, ha_rows limit) { DBUG_TRACE; uchar buff[STACK_BUFF_ALLOC]; if (check_stack_overrun(thd, STACK_MIN_SIZE, buff)) return 0; // Fatal error flag is set TABLE_LIST *const tl = tab->table_ref; tab->set_skip_records_in_range( check_skip_records_in_range_qualification(tab, thd)); // Derived tables aren't filled yet, so no stats are available. if (!tl->uses_materialization()) { QUICK_SELECT_I *qck; Key_map keys_to_use = tab->const_keys; keys_to_use.merge(tab->skip_scan_keys); int error = test_quick_select( thd, keys_to_use, 0, // empty table_map limit, false, // don't force quick range ORDER_NOT_RELEVANT, tab, tab->join_cond() ? tab->join_cond() : tab->join()->where_cond, &tab->needed_reg, &qck, tab->table()->force_index); tab->set_quick(qck); if (error == 1) return qck->records; if (error == -1) { tl->table->reginfo.impossible_range = 1; return 0; } DBUG_PRINT("warning", ("Couldn't use record count on const keypart")); } else if (tl->is_table_function() || tl->materializable_is_const()) { tl->fetch_number_of_rows(); return tl->table->file->stats.records; } return HA_POS_ERROR; } /* Get estimated record length for semi-join materialization temptable SYNOPSIS get_tmp_table_rec_length() items IN subquery's select list. DESCRIPTION Calculate estimated record length for semi-join materialization temptable. It's an estimate because we don't follow every bit of create_tmp_table()'s logic. This isn't necessary as the return value of this function is used only for cost calculations. RETURN Length of the temptable record, in bytes */ static uint get_tmp_table_rec_length(List &items) { uint len = 0; Item *item; List_iterator it(items); while ((item = it++)) { switch (item->result_type()) { case REAL_RESULT: len += sizeof(double); break; case INT_RESULT: if (item->max_length >= (MY_INT32_NUM_DECIMAL_DIGITS - 1)) len += 8; else len += 4; break; case STRING_RESULT: /* DATE/TIME and GEOMETRY fields have STRING_RESULT result type. */ if (item->is_temporal() || item->data_type() == MYSQL_TYPE_GEOMETRY) len += 8; else len += item->max_length; break; case DECIMAL_RESULT: len += 10; break; case ROW_RESULT: default: DBUG_ASSERT(0); /* purecov: deadcode */ break; } } return len; } /** Writes to the optimizer trace information about dependencies between tables. @param trace optimizer trace @param join_tabs all JOIN_TABs of the join @param table_count how many JOIN_TABs in the 'join_tabs' array */ static void trace_table_dependencies(Opt_trace_context *trace, JOIN_TAB *join_tabs, uint table_count) { Opt_trace_object trace_wrapper(trace); Opt_trace_array trace_dep(trace, "table_dependencies"); for (uint i = 0; i < table_count; i++) { TABLE_LIST *table_ref = join_tabs[i].table_ref; Opt_trace_object trace_one_table(trace); trace_one_table.add_utf8_table(table_ref).add( "row_may_be_null", table_ref->table->is_nullable()); const table_map map = table_ref->map(); DBUG_ASSERT(map < (1ULL << table_count)); for (uint j = 0; j < table_count; j++) { if (map & (1ULL << j)) { trace_one_table.add("map_bit", j); break; } } Opt_trace_array depends_on(trace, "depends_on_map_bits"); static_assert(sizeof(table_ref->map()) <= 64, "RAND_TABLE_BIT may be in join_tabs[i].dependent, so we test " "all 64 bits."); for (uint j = 0; j < 64; j++) { if (join_tabs[i].dependent & (1ULL << j)) depends_on.add(j); } } } /** Add to join_tab[i]->condition() "table.field IS NOT NULL" conditions we've inferred from ref/eq_ref access performed. This function is a part of "Early NULL-values filtering for ref access" optimization. Example of this optimization: For query SELECT * FROM t1,t2 WHERE t2.key=t1.field @n and plan " any-access(t1), ref(t2.key=t1.field) " @n add "t1.field IS NOT NULL" to t1's table condition. @n Description of the optimization: We look through equalities choosen to perform ref/eq_ref access, pick equalities that have form "tbl.part_of_key = othertbl.field" (where othertbl is a non-const table and othertbl.field may be NULL) and add them to conditions on correspoding tables (othertbl in this example). Exception from that is the case when referred_tab->join != join. I.e. don't add NOT NULL constraints from any embedded subquery. Consider this query: @code SELECT A.f2 FROM t1 LEFT JOIN t2 A ON A.f2 = f1 WHERE A.f3=(SELECT MIN(f3) FROM t2 C WHERE A.f4 = C.f4) OR A.f3 IS NULL; @endcode Here condition A.f3 IS NOT NULL is going to be added to the WHERE condition of the embedding query. Another example: SELECT * FROM t10, t11 WHERE (t10.a < 10 OR t10.a IS NULL) AND t11.b <=> t10.b AND (t11.a = (SELECT MAX(a) FROM t12 WHERE t12.b = t10.a )); Here condition t10.a IS NOT NULL is going to be added. In both cases addition of NOT NULL condition will erroneously reject some rows of the result set. referred_tab->join != join constraint would disallow such additions. This optimization doesn't affect the choices that ref, range, or join optimizer make. This was intentional because this was added after 4.1 was GA. Implementation overview 1. update_ref_and_keys() accumulates info about null-rejecting predicates in in Key_field::null_rejecting 1.1 add_key_part saves these to Key_use. 2. create_ref_for_key copies them to TABLE_REF. 3. add_not_null_conds adds "x IS NOT NULL" to join_tab->m_condition of appropiate JOIN_TAB members. */ static void add_not_null_conds(JOIN *join) { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(join); for (uint i = join->const_tables; i < join->tables; i++) { JOIN_TAB *const tab = join->best_ref[i]; if ((tab->type() == JT_REF || tab->type() == JT_EQ_REF || tab->type() == JT_REF_OR_NULL) && !tab->table()->is_nullable()) { for (uint keypart = 0; keypart < tab->ref().key_parts; keypart++) { if (tab->ref().null_rejecting & ((key_part_map)1 << keypart)) { Item *item = tab->ref().items[keypart]; Item *notnull; Item *real = item->real_item(); DBUG_ASSERT(real->type() == Item::FIELD_ITEM); Item_field *not_null_item = (Item_field *)real; JOIN_TAB *referred_tab = not_null_item->field->table->reginfo.join_tab; /* For UPDATE queries such as: UPDATE t1 SET t1.f2=(SELECT MAX(t2.f4) FROM t2 WHERE t2.f3=t1.f1); not_null_item is the t1.f1, but it's referred_tab is 0. */ if (!referred_tab || referred_tab->join() != join) continue; /* Skip if we already have a 'not null' predicate for 'item' */ if (has_not_null_predicate(referred_tab->condition(), not_null_item)) continue; if (!(notnull = new Item_func_isnotnull(not_null_item))) return; /* We need to do full fix_fields() call here in order to have correct notnull->const_item(). This is needed e.g. by test_quick_select when it is called from make_join_select after this function is called. */ if (notnull->fix_fields(join->thd, ¬null)) return; DBUG_EXECUTE("where", print_where(join->thd, notnull, referred_tab->table()->alias, QT_ORDINARY);); referred_tab->and_with_condition(notnull); } } } } } /** Check all existing AND'ed predicates in 'cond' for an existing 'is not null 'not_null_item''-predicate. A condition consisting of multiple AND'ed terms is recursively decomposed in the search for the specified not null predicate. @param cond Condition to be checked. @param not_null_item The item in: 'is not null 'item'' to search for @return true if 'is not null 'not_null_item'' is a predicate in the specified 'cond'. */ static bool has_not_null_predicate(Item *cond, Item_field *not_null_item) { if (cond == nullptr) return false; if (cond->type() == Item::FUNC_ITEM) { Item_func *item_func = down_cast(cond); const Item_func::Functype func_type = item_func->functype(); return (func_type == Item_func::ISNOTNULL_FUNC && item_func->key_item() == not_null_item); } else if (cond->type() == Item::COND_ITEM) { Item_cond *item_cond = down_cast(cond); if (item_cond->functype() == Item_func::COND_AND_FUNC) { List_iterator li(*item_cond->argument_list()); Item *item; while ((item = li++)) { if (has_not_null_predicate(item, not_null_item)) return true; } } } return false; } /** Check if given expression only uses fields covered by index @a keyno in the table tbl. The expression can use any fields in any other tables. The expression is guaranteed not to be AND or OR - those constructs are handled outside of this function. Restrict some function types from being pushed down to storage engine: a) Don't push down the triggered conditions. Nested outer joins execution code may need to evaluate a condition several times (both triggered and untriggered). TODO: Consider cloning the triggered condition and using the copies for: 1. push the first copy down, to have most restrictive index condition possible. 2. Put the second copy into tab->m_condition. b) Stored functions contain a statement that might start new operations (like DML statements) from within the storage engine. This does not work against all SEs. c) Subqueries might contain nested subqueries and involve more tables. TODO: ROY: CHECK THIS d) Do not push down internal functions of type DD_INTERNAL_FUNC. When ICP is enabled, pushing internal functions to storage engine for evaluation will open data-dictionary tables. In InnoDB storage engine this will result in situation like recursive latching of same page by the same thread. To avoid such situation, internal functions of type DD_INTERNAL_FUNC are not pushed to storage engine for evaluation. @param item Expression to check @param tbl The table having the index @param keyno The index number @param other_tbls_ok true <=> Fields of other non-const tables are allowed @return false if No, true if Yes */ bool uses_index_fields_only(Item *item, TABLE *tbl, uint keyno, bool other_tbls_ok) { // Restrictions b and c. if (item->has_stored_program() || item->has_subquery()) return false; // No table fields in const items if (item->const_item()) return true; const Item::Type item_type = item->type(); switch (item_type) { case Item::FUNC_ITEM: { Item_func *item_func = (Item_func *)item; const Item_func::Functype func_type = item_func->functype(); if (func_type == Item_func::TRIG_COND_FUNC || // Restriction a. func_type == Item_func::DD_INTERNAL_FUNC) // Restriction d. return false; /* This is a function, apply condition recursively to arguments */ if (item_func->argument_count() > 0) { Item **item_end = (item_func->arguments()) + item_func->argument_count(); for (Item **child = item_func->arguments(); child != item_end; child++) { if (!uses_index_fields_only(*child, tbl, keyno, other_tbls_ok)) return false; } } return true; } case Item::COND_ITEM: { /* This is a AND/OR condition. Regular AND/OR clauses are handled by make_cond_for_index() which will chop off the part that can be checked with index. This code is for handling non-top-level AND/ORs, e.g. func(x AND y). */ List_iterator li(*((Item_cond *)item)->argument_list()); Item *item; while ((item = li++)) { if (!uses_index_fields_only(item, tbl, keyno, other_tbls_ok)) return false; } return true; } case Item::FIELD_ITEM: { Item_field *item_field = (Item_field *)item; if (item_field->field->table != tbl) return other_tbls_ok; /* The below is probably a repetition - the first part checks the other two, but let's play it safe: */ return item_field->field->part_of_key.is_set(keyno) && item_field->field->type() != MYSQL_TYPE_GEOMETRY && item_field->field->type() != MYSQL_TYPE_BLOB; } case Item::REF_ITEM: return uses_index_fields_only(item->real_item(), tbl, keyno, other_tbls_ok); default: return false; /* Play it safe, don't push unknown non-const items */ } } /** Optimize semi-join nests that could be run with sj-materialization @param join The join to optimize semi-join nests for @details Optimize each of the semi-join nests that can be run with materialization. For each of the nests, we - Generate the best join order for this "sub-join" and remember it; - Remember the sub-join execution cost (it's part of materialization cost); - Calculate other costs that will be incurred if we decide to use materialization strategy for this semi-join nest. All obtained information is saved and will be used by the main join optimization pass. @return false if successful, true if error */ static bool optimize_semijoin_nests_for_materialization(JOIN *join) { DBUG_TRACE; List_iterator sj_list_it(join->select_lex->sj_nests); TABLE_LIST *sj_nest; Opt_trace_context *const trace = &join->thd->opt_trace; while ((sj_nest = sj_list_it++)) { /* As a precaution, reset pointers that were used in prior execution */ sj_nest->nested_join->sjm.positions = NULL; /* Calculate the cost of materialization if materialization is allowed. */ if (sj_nest->nested_join->sj_enabled_strategies & OPTIMIZER_SWITCH_MATERIALIZATION) { /* A semi-join nest should not contain tables marked as const */ DBUG_ASSERT(!(sj_nest->sj_inner_tables & join->const_table_map)); Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_sjmat( trace, "execution_plan_for_potential_materialization"); Opt_trace_array trace_sjmat_steps(trace, "steps"); /* Try semijoin materialization if the semijoin is classified as non-trivially-correlated. */ if (sj_nest->nested_join->sj_corr_tables) continue; /* Check whether data types allow execution with materialization. */ semijoin_types_allow_materialization(sj_nest); if (!sj_nest->nested_join->sjm.scan_allowed && !sj_nest->nested_join->sjm.lookup_allowed) continue; if (Optimize_table_order(join->thd, join, sj_nest).choose_table_order()) return true; const uint n_tables = my_count_bits(sj_nest->sj_inner_tables); calculate_materialization_costs(join, sj_nest, n_tables, &sj_nest->nested_join->sjm); /* Cost data is in sj_nest->nested_join->sjm. We also need to save the plan: */ if (!(sj_nest->nested_join->sjm.positions = (POSITION *)join->thd->alloc(sizeof(POSITION) * n_tables))) return true; memcpy(sj_nest->nested_join->sjm.positions, join->best_positions + join->const_tables, sizeof(POSITION) * n_tables); } } return false; } /* Check if table's Key_use elements have an eq_ref(outer_tables) candidate SYNOPSIS find_eq_ref_candidate() tl Table to be checked sj_inner_tables Bitmap of inner tables. eq_ref(inner_table) doesn't count. DESCRIPTION Check if table's Key_use elements have an eq_ref(outer_tables) candidate TODO Check again if it is feasible to factor common parts with constant table search RETURN true - There exists an eq_ref(outer-tables) candidate false - Otherwise */ static bool find_eq_ref_candidate(TABLE_LIST *tl, table_map sj_inner_tables) { Key_use *keyuse = tl->table->reginfo.join_tab->keyuse(); if (keyuse) { while (1) /* For each key */ { const uint key = keyuse->key; KEY *const keyinfo = tl->table->key_info + key; key_part_map bound_parts = 0; if ((keyinfo->flags & (HA_NOSAME)) == HA_NOSAME) { do /* For all equalities on all key parts */ { /* Check if this is "t.keypart = expr(outer_tables) */ if (!(keyuse->used_tables & sj_inner_tables) && !(keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL)) { /* Consider only if the resulting condition does not pass a NULL value through. Especially needed for a UNIQUE index on NULLable columns where a duplicate row is possible with NULL values. */ if (keyuse->null_rejecting || !keyuse->val->maybe_null || !keyinfo->key_part[keyuse->keypart].field->maybe_null()) bound_parts |= (key_part_map)1 << keyuse->keypart; } keyuse++; } while (keyuse->key == key && keyuse->table_ref == tl); if (bound_parts == LOWER_BITS(uint, keyinfo->user_defined_key_parts)) return true; if (keyuse->table_ref != tl) return false; } else { do { keyuse++; if (keyuse->table_ref != tl) return false; } while (keyuse->key == key); } } } return false; } /** Pull tables out of semi-join nests based on functional dependencies @param join The join where to do the semi-join table pullout @return False if successful, true if error (Out of memory) @details Pull tables out of semi-join nests based on functional dependencies, ie. if a table is accessed via eq_ref(outer_tables). The function may be called several times, the caller is responsible for setting up proper key information that this function acts upon. PRECONDITIONS When this function is called, the join may have several semi-join nests but it is guaranteed that one semi-join nest does not contain another. For functionally dependent tables to be pulled out, key information must have been calculated (see update_ref_and_keys()). POSTCONDITIONS * Tables that were pulled out are removed from the semi-join nest they belonged to and added to the parent join nest. * For these tables, the used_tables and not_null_tables fields of the semi-join nest they belonged to will be adjusted. The semi-join nest is also marked as correlated, and sj_corr_tables and sj_depends_on are adjusted if necessary. * Semi-join nests' sj_inner_tables is set equal to used_tables NOTE Table pullout may make uncorrelated subquery correlated. Consider this example: ... WHERE oe IN (SELECT it1.primary_key WHERE p(it1, it2) ... ) here table it1 can be pulled out (we have it1.primary_key=oe which gives us functional dependency). Once it1 is pulled out, all references to it1 from p(it1, it2) become references to outside of the subquery and thus make the subquery (i.e. its semi-join nest) correlated. Making the subquery (i.e. its semi-join nest) correlated prevents us from using Materialization or LooseScan to execute it. */ static bool pull_out_semijoin_tables(JOIN *join) { TABLE_LIST *sj_nest; DBUG_TRACE; DBUG_ASSERT(!join->select_lex->sj_nests.is_empty()); List_iterator sj_list_it(join->select_lex->sj_nests); Opt_trace_context *const trace = &join->thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_array trace_pullout(trace, "pulled_out_semijoin_tables"); /* Try pulling out tables from each semi-join nest */ while ((sj_nest = sj_list_it++)) { if (sj_nest->is_aj_nest()) continue; table_map pulled_tables = 0; List_iterator child_li(sj_nest->nested_join->join_list); TABLE_LIST *tbl; /* Calculate set of tables within this semi-join nest that have other dependent tables */ table_map dep_tables = 0; while ((tbl = child_li++)) { TABLE *const table = tbl->table; if (table && (table->reginfo.join_tab->dependent & sj_nest->nested_join->used_tables)) dep_tables |= table->reginfo.join_tab->dependent; } /* Find which tables we can pull out based on key dependency data. Note that pulling one table out can allow us to pull out some other tables too. */ bool pulled_a_table; do { pulled_a_table = false; child_li.rewind(); while ((tbl = child_li++)) { if (tbl->table && !(pulled_tables & tbl->map()) && !(dep_tables & tbl->map())) { if (find_eq_ref_candidate( tbl, sj_nest->nested_join->used_tables & ~pulled_tables)) { pulled_a_table = true; pulled_tables |= tbl->map(); Opt_trace_object(trace).add_utf8_table(tbl).add( "functionally_dependent", true); /* Pulling a table out of uncorrelated subquery in general makes it correlated. See the NOTE to this function. */ sj_nest->nested_join->sj_corr_tables |= tbl->map(); sj_nest->nested_join->sj_depends_on |= tbl->map(); } } } } while (pulled_a_table); child_li.rewind(); /* Move the pulled out TABLE_LIST elements to the parents. */ sj_nest->nested_join->used_tables &= ~pulled_tables; sj_nest->nested_join->not_null_tables &= ~pulled_tables; /* sj_inner_tables is a copy of nested_join->used_tables */ sj_nest->sj_inner_tables = sj_nest->nested_join->used_tables; if (pulled_tables) { List *upper_join_list = (sj_nest->embedding != NULL) ? &sj_nest->embedding->nested_join->join_list : &join->select_lex->top_join_list; Prepared_stmt_arena_holder ps_arena_holder(join->thd); while ((tbl = child_li++)) { if (tbl->table && !(sj_nest->nested_join->used_tables & tbl->map())) { /* Pull the table up in the same way as simplify_joins() does: update join_list and embedding pointers but keep next[_local] pointers. */ child_li.remove(); if (upper_join_list->push_back(tbl)) return true; tbl->join_list = upper_join_list; tbl->embedding = sj_nest->embedding; } } /* Remove the sj-nest itself if we've removed everything from it */ if (!sj_nest->nested_join->used_tables) { List_iterator li(*upper_join_list); /* Find the sj_nest in the list. */ while (sj_nest != li++) { } li.remove(); /* Also remove it from the list of SJ-nests: */ sj_list_it.remove(); } } } return false; } /** @defgroup RefOptimizerModule Ref Optimizer @{ This module analyzes all equality predicates to determine the best independent ref/eq_ref/ref_or_null index access methods. The 'ref' optimizer determines the columns (and expressions over them) that reference columns in other tables via an equality, and analyzes which keys and key parts can be used for index lookup based on these references. The main outcomes of the 'ref' optimizer are: - A bi-directional graph of all equi-join conditions represented as an array of Key_use elements. This array is stored in JOIN::keyuse_array in table, key, keypart order. Each JOIN_TAB::keyuse points to the first Key_use element with the same table as JOIN_TAB::table. - The table dependencies needed by the optimizer to determine what tables must be before certain table so that they provide the necessary column bindings for the equality predicates. - Computed properties of the equality predicates such as null_rejecting and the result size of each separate condition. Updates in JOIN_TAB: - JOIN_TAB::keys Bitmap of all used keys. - JOIN_TAB::const_keys Bitmap of all keys that may be used with quick_select. - JOIN_TAB::keyuse Pointer to possible keys. */ /** A Key_field is a descriptor of a predicate of the form (column @ val). Currently 'op' is one of {'=', '<=>', 'IS [NOT] NULL', 'arg1 IN arg2'}, and 'val' can be either another column or an expression (including constants). Key_field's are used to analyze columns that may potentially serve as parts of keys for index lookup. If 'field' is part of an index, then add_key_part() creates a corresponding Key_use object and inserts it into the JOIN::keyuse_array which is passed by update_ref_and_keys(). The structure is used only during analysis of the candidate columns for index 'ref' access. */ struct Key_field { Key_field(Item_field *item_field, Item *val, uint level, uint optimize, bool eq_func, bool null_rejecting, bool *cond_guard, uint sj_pred_no) : item_field(item_field), val(val), level(level), optimize(optimize), eq_func(eq_func), null_rejecting(null_rejecting), cond_guard(cond_guard), sj_pred_no(sj_pred_no) {} Item_field *item_field; ///< Item representing the column Item *val; ///< May be empty if diff constant uint level; uint optimize; ///< KEY_OPTIMIZE_* bool eq_func; /** If true, the condition this struct represents will not be satisfied when val IS NULL. @sa Key_use::null_rejecting . */ bool null_rejecting; bool *cond_guard; ///< @sa Key_use::cond_guard uint sj_pred_no; ///< @sa Key_use::sj_pred_no }; /* Values in optimize */ #define KEY_OPTIMIZE_EXISTS 1 #define KEY_OPTIMIZE_REF_OR_NULL 2 /** Merge new key definitions to old ones, remove those not used in both. This is called for OR between different levels. To be able to do 'ref_or_null' we merge a comparison of a column and 'column IS NULL' to one test. This is useful for sub select queries that are internally transformed to something like:. @code SELECT * FROM t1 WHERE t1.key=outer_ref_field or t1.key IS NULL @endcode Key_field::null_rejecting is processed as follows: @n result has null_rejecting=true if it is set for both ORed references. for example: - (t2.key = t1.field OR t2.key = t1.field) -> null_rejecting=true - (t2.key = t1.field OR t2.key <=> t1.field) -> null_rejecting=false @todo The result of this is that we're missing some 'ref' accesses. OptimizerTeam: Fix this */ static Key_field *merge_key_fields(Key_field *start, Key_field *new_fields, Key_field *end, uint and_level) { if (start == new_fields) return start; // Impossible or if (new_fields == end) return start; // No new fields, skip all Key_field *first_free = new_fields; /* Mark all found fields in old array */ for (; new_fields != end; new_fields++) { Field *const new_field = new_fields->item_field->field; for (Key_field *old = start; old != first_free; old++) { Field *const old_field = old->item_field->field; /* Check that the Field objects are the same, as we may have several Item_field objects pointing to the same Field: */ if (old_field == new_field) { /* NOTE: below const_item() call really works as "!used_tables()", i.e. it can return false where it is feasible to make it return true. The cause is as follows: Some of the tables are already known to be const tables (the detection code is in JOIN::make_join_plan(), above the update_ref_and_keys() call), but we didn't propagate information about this: TABLE::const_table is not set to true, and Item::update_used_tables() hasn't been called for each item. The result of this is that we're missing some 'ref' accesses. TODO: OptimizerTeam: Fix this */ if (!new_fields->val->const_item()) { /* If the value matches, we can use the key reference. If not, we keep it until we have examined all new values */ if (old->val->eq(new_fields->val, old_field->binary())) { old->level = and_level; old->optimize = ((old->optimize & new_fields->optimize & KEY_OPTIMIZE_EXISTS) | ((old->optimize | new_fields->optimize) & KEY_OPTIMIZE_REF_OR_NULL)); old->null_rejecting = (old->null_rejecting && new_fields->null_rejecting); } } else if (old->eq_func && new_fields->eq_func && old->val->eq_by_collation(new_fields->val, old_field->binary(), old_field->charset())) { old->level = and_level; old->optimize = ((old->optimize & new_fields->optimize & KEY_OPTIMIZE_EXISTS) | ((old->optimize | new_fields->optimize) & KEY_OPTIMIZE_REF_OR_NULL)); old->null_rejecting = (old->null_rejecting && new_fields->null_rejecting); } else if (old->eq_func && new_fields->eq_func && ((old->val->const_item() && old->val->is_null()) || new_fields->val->is_null())) { /* field = expression OR field IS NULL */ old->level = and_level; old->optimize = KEY_OPTIMIZE_REF_OR_NULL; /* Remember the NOT NULL value unless the value does not depend on other tables. */ if (!old->val->used_tables() && old->val->is_null()) old->val = new_fields->val; /* The referred expression can be NULL: */ old->null_rejecting = 0; } else { /* We are comparing two different const. In this case we can't use a key-lookup on this so it's better to remove the value and let the range optimizer handle it */ if (old == --first_free) // If last item break; *old = *first_free; // Remove old value old--; // Retry this value } } } } /* Remove all not used items */ for (Key_field *old = start; old != first_free;) { if (old->level != and_level) { // Not used in all levels if (old == --first_free) break; *old = *first_free; // Remove old value continue; } old++; } return first_free; } /** Given a field, return its index in semi-join's select list, or UINT_MAX @param item_field Field to be looked up in select list @retval =UINT_MAX Field is not from a semijoin-transformed subquery @retval table_ref->embedding; if (emb_sj_nest && emb_sj_nest->is_sj_or_aj_nest()) { List &items = emb_sj_nest->nested_join->sj_inner_exprs; List_iterator it(items); for (uint i = 0; i < items.elements; i++) { Item *sel_item = it++; if (sel_item->type() == Item::FIELD_ITEM && ((Item_field *)sel_item)->field->eq(item_field->field)) return i; } } return UINT_MAX; } /** @brief If EXPLAIN or if the --safe-updates option is enabled, add a warning that an index cannot be used for ref access. @details If EXPLAIN or if the --safe-updates option is enabled, add a warning for each index that cannot be used for ref access due to either type conversion or different collations on the field used for comparison Example type conversion (char compared to int): CREATE TABLE t1 (url char(1) PRIMARY KEY); SELECT * FROM t1 WHERE url=1; Example different collations (danish vs german2): CREATE TABLE t1 (url char(1) PRIMARY KEY) collate latin1_danish_ci; SELECT * FROM t1 WHERE url='1' collate latin1_german2_ci; @param thd Thread for the connection that submitted the query @param field Field used in comparison @param cant_use_index Indexes that cannot be used for lookup */ static void warn_index_not_applicable(THD *thd, const Field *field, const Key_map cant_use_index) { Functional_index_error_handler functional_index_error_handler(field, thd); if (thd->lex->is_explain() || thd->variables.option_bits & OPTION_SAFE_UPDATES) for (uint j = 0; j < field->table->s->keys; j++) if (cant_use_index.is_set(j)) push_warning_printf(thd, Sql_condition::SL_WARNING, ER_WARN_INDEX_NOT_APPLICABLE, ER_THD(thd, ER_WARN_INDEX_NOT_APPLICABLE), "ref", field->table->key_info[j].name, field->field_name); } /** Add a possible key to array of possible keys if it's usable as a key @param [in,out] key_fields Used as an input parameter in the sense that it is a pointer to a pointer to a memory area where an array of Key_field objects will stored. It is used as an out parameter in the sense that the pointer will be updated to point beyond the last Key_field written. @param thd session context @param[in,out] key_fields is incremented if a key was stored in the array @param and_level And level, to be stored in Key_field @param cond Condition predicate @param item_field Field used in comparison @param eq_func True if we used =, <=> or IS NULL @param value Array of values used for comparison with field @param num_values Number of elements in the array of values @param usable_tables Tables which can be used for key optimization @param sargables IN/OUT Array of found sargable candidates. Will be ignored in case eq_func is true. @note If we are doing a NOT NULL comparison on a NOT NULL field in a outer join table, we store this to be able to do not exists optimization later. @returns false if success, true if error */ static bool add_key_field(THD *thd, Key_field **key_fields, uint and_level, Item_func *cond, Item_field *item_field, bool eq_func, Item **value, uint num_values, table_map usable_tables, SARGABLE_PARAM **sargables) { DBUG_ASSERT(cond->is_bool_func()); DBUG_ASSERT(eq_func || sargables); Field *const field = item_field->field; TABLE_LIST *const tl = item_field->table_ref; if (tl->table->reginfo.join_tab == NULL) { /* Due to a bug in IN-to-EXISTS (grep for real_item() in item_subselect.cc for more info), an index over a field from an outer query might be considered here, which is incorrect. Their query has been fully optimized already so their reginfo.join_tab is NULL and we reject them. */ return false; } DBUG_PRINT("info", ("add_key_field for field %s", field->field_name)); uint exists_optimize = 0; if (!tl->derived_keys_ready && tl->uses_materialization() && !tl->table->is_created()) { bool allocated; if (tl->update_derived_keys(thd, field, value, num_values, &allocated)) return true; if (!allocated) return false; } if (!(field->flags & PART_KEY_FLAG)) { // Don't remove column IS NULL on a LEFT JOIN table if (!eq_func || (*value)->type() != Item::NULL_ITEM || !tl->table->is_nullable() || field->real_maybe_null()) return false; // Not a key. Skip it exists_optimize = KEY_OPTIMIZE_EXISTS; DBUG_ASSERT(num_values == 1); } else { table_map used_tables = 0; bool optimizable = false; for (uint i = 0; i < num_values; i++) { used_tables |= (value[i])->used_tables(); if (!((value[i])->used_tables() & (tl->map() | RAND_TABLE_BIT))) optimizable = true; } if (!optimizable) return false; if (!(usable_tables & tl->map())) { if (!eq_func || (*value)->type() != Item::NULL_ITEM || !tl->table->is_nullable() || field->real_maybe_null()) return false; // Can't use left join optimize exists_optimize = KEY_OPTIMIZE_EXISTS; } else { JOIN_TAB *stat = tl->table->reginfo.join_tab; Key_map possible_keys = field->key_start; possible_keys.intersect(tl->table->keys_in_use_for_query); stat[0].keys().merge(possible_keys); // Add possible keys /* Save the following cases: Field op constant Field LIKE constant where constant doesn't start with a wildcard Field = field2 where field2 is in a different table Field op formula Field IS NULL Field IS NOT NULL Field BETWEEN ... Field IN ... */ stat[0].key_dependent |= used_tables; bool is_const = true; for (uint i = 0; i < num_values; i++) { if (!(is_const &= value[i]->const_for_execution())) break; } if (is_const) stat[0].const_keys.merge(possible_keys); else if (!eq_func) { /* Save info to be able check whether this predicate can be considered as sargable for range analysis after reading const tables. We do not save info about equalities as update_const_equal_items will take care of updating info on keys from sargable equalities. */ DBUG_ASSERT(sargables); (*sargables)--; /* The sargables and key_fields arrays share the same memory buffer, and grow from opposite directions, so make sure they don't cross. */ DBUG_ASSERT(*sargables > reinterpret_cast(*key_fields)); (*sargables)->field = field; (*sargables)->arg_value = value; (*sargables)->num_values = num_values; } /* We can't always use indexes when comparing a string index to a number. cmp_type() is checked to allow compare of dates to numbers. eq_func is NEVER true when num_values > 1 */ if (!eq_func) return false; /* Check if the field and value are comparable in the index. @todo: This code is almost identical to comparable_in_index() in opt_range.cc. Consider replacing the checks below with a function call to comparable_in_index() */ if (field->result_type() == STRING_RESULT) { if ((*value)->result_type() != STRING_RESULT) { if (field->cmp_type() != (*value)->result_type()) { warn_index_not_applicable(stat->join()->thd, field, possible_keys); return false; } } else { /* Can't optimize datetime_column=indexed_varchar_column, also can't use indexes if the effective collation of the operation differ from the field collation. IndexedTimeComparedToDate: can't optimize 'indexed_time = temporal_expr_with_date_part' because: - without index, a TIME column with value '48:00:00' is equal to a DATETIME column with value 'CURDATE() + 2 days' - with ref access into the TIME column, CURDATE() + 2 days becomes "00:00:00" (Field_timef::store_internal() simply extracts the time part from the datetime) which is a lookup key which does not match "48:00:00"; so ref access is not be able to give the same result as without index, so is disabled. On the other hand, we can optimize indexed_datetime = time because Field_temporal_with_date::store_time() will convert 48:00:00 to CURDATE() + 2 days which is the correct lookup key. */ if ((!field->is_temporal() && value[0]->is_temporal()) || (field->cmp_type() == STRING_RESULT && field->match_collation_to_optimize_range() && field->charset() != cond->compare_collation()) || field_time_cmp_date(field, value[0])) { warn_index_not_applicable(stat->join()->thd, field, possible_keys); return false; } } } /* We can't use indexes when comparing to a JSON value. For example, the string '{}' should compare equal to the JSON string "{}". If we use a string index to compare the two strings, we will be comparing '{}' and '"{}"', which don't compare equal. The only exception is Item_json, which is a basic const item and is used to contain value coerced to index's type. */ if (value[0]->result_type() == STRING_RESULT && value[0]->data_type() == MYSQL_TYPE_JSON && !value[0]->basic_const_item()) { warn_index_not_applicable(stat->join()->thd, field, possible_keys); return false; } } } /* For the moment eq_func is always true. This slot is reserved for future extensions where we want to remembers other things than just eq comparisons */ DBUG_ASSERT(eq_func); /* If the condition has form "tbl.keypart = othertbl.field" and othertbl.field can be NULL, there will be no matches if othertbl.field has NULL value. We use null_rejecting in add_not_null_conds() to add 'othertbl.field IS NOT NULL' to tab->m_condition, if this is not an outer join. We also use it to shortcut reading "tbl" when othertbl.field is found to be a NULL value (in RefIterator and BKA). */ Item *const real = (*value)->real_item(); const bool null_rejecting = ((cond->functype() == Item_func::EQ_FUNC) || (cond->functype() == Item_func::MULT_EQUAL_FUNC)) && (real->type() == Item::FIELD_ITEM) && ((Item_field *)real)->field->maybe_null(); /* Store possible eq field */ new (*key_fields) Key_field(item_field, *value, and_level, exists_optimize, eq_func, null_rejecting, NULL, get_semi_join_select_list_index(item_field)); (*key_fields)++; /* The sargables and key_fields arrays share the same memory buffer, and grow from opposite directions, so make sure they don't cross. But if sargables was NULL, eq_func had to be true and we don't write any sargables. */ DBUG_ASSERT(sargables == NULL || *key_fields < reinterpret_cast(*sargables)); return false; } /** Add possible keys to array of possible keys originated from a simple predicate. @param thd session context @param[in,out] key_fields Pointer to add key, if usable is incremented if key was stored in the array @param and_level And level, to be stored in Key_field @param cond Condition predicate @param field_item Field used in comparision @param eq_func True if we used =, <=> or IS NULL @param val Value used for comparison with field Is NULL for BETWEEN and IN @param num_values @param usable_tables Tables which can be used for key optimization @param sargables IN/OUT Array of found sargable candidates @note If field items f1 and f2 belong to the same multiple equality and a key is added for f1, the the same key is added for f2. @returns false if success, true if error */ static bool add_key_equal_fields(THD *thd, Key_field **key_fields, uint and_level, Item_func *cond, Item_field *field_item, bool eq_func, Item **val, uint num_values, table_map usable_tables, SARGABLE_PARAM **sargables) { DBUG_ASSERT(cond->is_bool_func()); if (add_key_field(thd, key_fields, and_level, cond, field_item, eq_func, val, num_values, usable_tables, sargables)) return true; Item_equal *item_equal = field_item->item_equal; if (item_equal == nullptr) return false; /* Add to the set of possible key values every substitution of the field for an equal field included into item_equal */ Item_equal_iterator it(*item_equal); Item_field *item; while ((item = it++)) { if (!field_item->field->eq(item->field)) { if (add_key_field(thd, key_fields, and_level, cond, item, eq_func, val, num_values, usable_tables, sargables)) return true; } } return false; } /** Check if an expression is a non-outer field. Checks if an expression is a field and belongs to the current select. @param field Item expression to check @return boolean @retval true the expression is a local field @retval false it's something else */ static bool is_local_field(Item *field) { return field->real_item()->type() == Item::FIELD_ITEM && !(field->used_tables() & OUTER_REF_TABLE_BIT) && !down_cast(field)->depended_from && !down_cast(field->real_item())->depended_from; } /** Check if a row constructor expression is over columns in the same query block. @param item_row Row expression to check. @return boolean @retval true The expression is a local column reference. @retval false It's something else. */ static bool is_row_of_local_columns(Item_row *item_row) { for (uint i = 0; i < item_row->cols(); ++i) if (!is_local_field(item_row->element_index(i))) return false; return true; } /** The guts of the ref optimizer. This function, along with the other add_key_* functions, make up a recursive procedure that analyzes a condition expression (a tree of AND and OR predicates) and does many things. @param thd session context @param join The query block involving the condition. @param [in,out] key_fields Start of memory buffer, see below. @param [in,out] and_level Current 'and level', see below. @param cond The conditional expression to analyze. @param usable_tables Tables not in this bitmap will not be examined. @param [in,out] sargables End of memory buffer, see below. @returns false if success, true if error This documentation is the result of reverse engineering and may therefore not capture the full gist of the procedure, but it is known to do the following: - Populate a raw memory buffer from two directions at the same time. An 'array' of Key_field objects fill the buffer from low to high addresses whilst an 'array' of SARGABLE_PARAM's fills the buffer from high to low addresses. At the first call to this function, it is assumed that key_fields points to the beginning of the buffer and sargables point to the end (except for a poor-mans 'null element' at the very end). - Update a number of properties in the JOIN_TAB's that can be used to find search keys (sargables). - JOIN_TAB::keys - JOIN_TAB::key_dependent - JOIN_TAB::const_keys (dictates if the range optimizer will be run later.) The Key_field objects are marked with something called an 'and_level', which does @b not correspond to their nesting depth within the expression tree. It is rather a tag to group conjunctions together. For instance, in the conditional expression @code a = 0 AND b = 0 @endcode two Key_field's are produced, both having an and_level of 0. In an expression such as @code a = 0 AND b = 0 OR a = 1 @endcode three Key_field's are produced, the first two corresponding to 'a = 0' and 'b = 0', respectively, both with and_level 0. The third one corresponds to 'a = 1' and has an and_level of 1. A separate function, merge_key_fields() performs ref access validation on the Key_field array on the recursice ascent. If some Key_field's cannot be used for ref access, the key_fields pointer is rolled back. All other modifications to the query plan remain. */ static bool add_key_fields(THD *thd, JOIN *join, Key_field **key_fields, uint *and_level, Item *cond, table_map usable_tables, SARGABLE_PARAM **sargables) { DBUG_ASSERT(cond->is_bool_func()); if (cond->type() == Item_func::COND_ITEM) { List_iterator_fast li(*((Item_cond *)cond)->argument_list()); Key_field *org_key_fields = *key_fields; if (down_cast(cond)->functype() == Item_func::COND_AND_FUNC) { Item *item; while ((item = li++)) { if (add_key_fields(thd, join, key_fields, and_level, item, usable_tables, sargables)) return true; } for (; org_key_fields != *key_fields; org_key_fields++) org_key_fields->level = *and_level; } else { (*and_level)++; if (add_key_fields(thd, join, key_fields, and_level, li++, usable_tables, sargables)) return true; Item *item; while ((item = li++)) { Key_field *start_key_fields = *key_fields; (*and_level)++; if (add_key_fields(thd, join, key_fields, and_level, item, usable_tables, sargables)) return true; *key_fields = merge_key_fields(org_key_fields, start_key_fields, *key_fields, ++(*and_level)); } } return false; } /* Subquery optimization: Conditions that are pushed down into subqueries are wrapped into Item_func_trig_cond. We process the wrapped condition but need to set cond_guard for Key_use elements generated from it. */ if (cond->type() == Item::FUNC_ITEM && down_cast(cond)->functype() == Item_func::TRIG_COND_FUNC) { Item *const cond_arg = down_cast(cond)->arguments()[0]; if (!join->group_list && !join->order && join->unit->item && join->unit->item->substype() == Item_subselect::IN_SUBS && !join->unit->is_union()) { Key_field *save = *key_fields; if (add_key_fields(thd, join, key_fields, and_level, cond_arg, usable_tables, sargables)) return true; // Indicate that this ref access candidate is for subquery lookup: for (; save != *key_fields; save++) save->cond_guard = ((Item_func_trig_cond *)cond)->get_trig_var(); } return false; } /* If item is of type 'field op field/constant' add it to key_fields */ if (cond->type() != Item::FUNC_ITEM) return false; Item_func *const cond_func = down_cast(cond); auto optimize = cond_func->select_optimize(thd); // Catch errors that might be thrown during select_optimize() if (thd->is_error()) return true; switch (optimize) { case Item_func::OPTIMIZE_NONE: break; case Item_func::OPTIMIZE_KEY: { Item **values; /* Build list of possible keys for 'a BETWEEN low AND high'. It is handled similar to the equivalent condition 'a >= low AND a <= high': */ if (cond_func->functype() == Item_func::BETWEEN) { Item_field *field_item; bool equal_func = false; uint num_values = 2; values = cond_func->arguments(); bool binary_cmp = (values[0]->real_item()->type() == Item::FIELD_ITEM) ? ((Item_field *)values[0]->real_item())->field->binary() : true; /* Additional optimization: If 'low = high': Handle as if the condition was "t.key = low". */ if (!((Item_func_between *)cond_func)->negated && values[1]->eq(values[2], binary_cmp)) { equal_func = true; num_values = 1; } /* Append keys for 'field value[]' if the condition is of the form:: ' BETWEEN value[1] AND value[2]' */ if (is_local_field(values[0])) { field_item = (Item_field *)(values[0]->real_item()); if (add_key_equal_fields(thd, key_fields, *and_level, cond_func, field_item, equal_func, &values[1], num_values, usable_tables, sargables)) return true; } /* Append keys for 'value[0] field' if the condition is of the form: 'value[0] BETWEEN field1 AND field2' */ for (uint i = 1; i <= num_values; i++) { if (is_local_field(values[i])) { field_item = (Item_field *)(values[i]->real_item()); if (add_key_equal_fields(thd, key_fields, *and_level, cond_func, field_item, equal_func, values, 1, usable_tables, sargables)) return true; } } } // if ( ... Item_func::BETWEEN) else if (cond_func->functype() == Item_func::MEMBER_OF_FUNC && is_local_field(cond_func->key_item())) { // The predicate is IN () add_key_equal_fields(thd, key_fields, *and_level, cond_func, (Item_field *)(cond_func->key_item()->real_item()), true, cond_func->arguments(), 1, usable_tables, sargables); } else if (cond_func->functype() == Item_func::JSON_CONTAINS || cond_func->functype() == Item_func::JSON_OVERLAPS) { Json_wrapper cont_wr; String str; Field *field; /* Applicability analysis was done during substitute_gc(). Check here that a typed array field is used and there's a key over it. 1) func has a key item 2) key item is a local field 3) key item is a typed array field If so, mark appropriate index as available for range optimizer */ if (!cond_func->key_item() || // 1 !is_local_field(cond_func->key_item()) || // 2 !cond_func->key_item()->returns_array()) // 3 break; field = (down_cast(cond_func->key_item()))->field; JOIN_TAB *tab = field->table->reginfo.join_tab; Key_map possible_keys = field->key_start; possible_keys.intersect(field->table->keys_in_use_for_query); tab->keys().merge(possible_keys); // Add possible keys tab->const_keys.merge(possible_keys); // Add possible keys } // if (... Item_func::CONTAINS) // The predicate is IN or <> else if (is_local_field(cond_func->key_item()) && !(cond_func->used_tables() & OUTER_REF_TABLE_BIT)) { values = cond_func->arguments() + 1; if (cond_func->functype() == Item_func::NE_FUNC && is_local_field(cond_func->arguments()[1])) values--; DBUG_ASSERT(cond_func->functype() != Item_func::IN_FUNC || cond_func->argument_count() != 2); if (add_key_equal_fields( thd, key_fields, *and_level, cond_func, (Item_field *)(cond_func->key_item()->real_item()), 0, values, cond_func->argument_count() - 1, usable_tables, sargables)) return true; } else if (cond_func->functype() == Item_func::IN_FUNC && cond_func->key_item()->type() == Item::ROW_ITEM) { /* The condition is (column1, column2, ... ) IN ((const1_1, const1_2), ...) and there is an index on (column1, column2, ...) The code below makes sure that the row constructor on the lhs indeed contains only column references before calling add_key_field on them. We can't do a ref access on IN, yet here we are. Why? We need to run add_key_field() only because it verifies that there are only constant expressions in the rows on the IN's rhs, see comment above the call to add_key_field() below. Actually, We could in theory do a ref access if the IN rhs contained just a single row, but there is a hack in the parser causing such IN predicates be parsed as row equalities. */ Item_row *lhs_row = static_cast(cond_func->key_item()); if (is_row_of_local_columns(lhs_row)) { for (uint i = 0; i < lhs_row->cols(); ++i) { Item *const lhs_item = lhs_row->element_index(i)->real_item(); DBUG_ASSERT(lhs_item->type() == Item::FIELD_ITEM); Item_field *const lhs_column = static_cast(lhs_item); // j goes from 1 since arguments()[0] is the lhs of IN. for (uint j = 1; j < cond_func->argument_count(); ++j) { // Here we pick out the i:th column in the j:th row. Item *rhs_item = cond_func->arguments()[j]; DBUG_ASSERT(rhs_item->type() == Item::ROW_ITEM); Item_row *rhs_row = static_cast(rhs_item); DBUG_ASSERT(rhs_row->cols() == lhs_row->cols()); Item **rhs_expr_ptr = rhs_row->addr(i); /* add_key_field() will write a Key_field on each call here, but we don't care, it will never be used. We only call it for the side effect: update JOIN_TAB::const_keys so the range optimizer can be invoked. We pass a scrap buffer and pointer here. */ Key_field scrap_key_field = **key_fields; Key_field *scrap_key_field_ptr = &scrap_key_field; if (add_key_field(thd, &scrap_key_field_ptr, *and_level, cond_func, lhs_column, true, // eq_func rhs_expr_ptr, 1, // Number of expressions: one usable_tables, NULL)) // sargables return true; // The pointer is not supposed to increase by more than one. DBUG_ASSERT(scrap_key_field_ptr <= &scrap_key_field + 1); } } } } break; } case Item_func::OPTIMIZE_OP: { bool equal_func = (cond_func->functype() == Item_func::EQ_FUNC || cond_func->functype() == Item_func::EQUAL_FUNC); if (is_local_field(cond_func->arguments()[0])) { if (add_key_equal_fields( thd, key_fields, *and_level, cond_func, (Item_field *)(cond_func->arguments()[0])->real_item(), equal_func, cond_func->arguments() + 1, 1, usable_tables, sargables)) return true; } else { Item *real_item = cond_func->arguments()[0]->real_item(); if (real_item->type() == Item::FUNC_ITEM) { Item_func *func_item = down_cast(real_item); if (func_item->functype() == Item_func::COLLATE_FUNC) { Item *key_item = func_item->key_item(); if (key_item->type() == Item::FIELD_ITEM) { if (add_key_equal_fields(thd, key_fields, *and_level, cond_func, down_cast(key_item), equal_func, cond_func->arguments() + 1, 1, usable_tables, sargables)) return true; } } } } if (is_local_field(cond_func->arguments()[1]) && cond_func->functype() != Item_func::LIKE_FUNC) { if (add_key_equal_fields( thd, key_fields, *and_level, cond_func, (Item_field *)(cond_func->arguments()[1])->real_item(), equal_func, cond_func->arguments(), 1, usable_tables, sargables)) return true; } else { Item *real_item = cond_func->arguments()[1]->real_item(); if (real_item->type() == Item::FUNC_ITEM) { Item_func *func_item = down_cast(real_item); if (func_item->functype() == Item_func::COLLATE_FUNC) { Item *key_item = func_item->key_item(); if (key_item->type() == Item::FIELD_ITEM) { if (add_key_equal_fields(thd, key_fields, *and_level, cond_func, down_cast(key_item), equal_func, cond_func->arguments(), 1, usable_tables, sargables)) return true; } } } } break; } case Item_func::OPTIMIZE_NULL: /* column_name IS [NOT] NULL */ if (is_local_field(cond_func->arguments()[0]) && !(cond_func->used_tables() & OUTER_REF_TABLE_BIT)) { Item *tmp = new Item_null; if (tmp == nullptr) return true; if (add_key_equal_fields( thd, key_fields, *and_level, cond_func, (Item_field *)(cond_func->arguments()[0])->real_item(), cond_func->functype() == Item_func::ISNULL_FUNC, &tmp, 1, usable_tables, sargables)) return true; } break; case Item_func::OPTIMIZE_EQUAL: Item_equal *item_equal = (Item_equal *)cond; Item *const_item = item_equal->get_const(); if (const_item) { /* For each field field1 from item_equal consider the equality field1=const_item as a condition allowing an index access of the table with field1 by the keys value of field1. */ Item_equal_iterator it(*item_equal); Item_field *item; while ((item = it++)) { if (add_key_field(thd, key_fields, *and_level, cond_func, item, true, &const_item, 1, usable_tables, sargables)) return true; } } else { /* Consider all pairs of different fields included into item_equal. For each of them (field1, field1) consider the equality field1=field2 as a condition allowing an index access of the table with field1 by the keys value of field2. */ Item_equal_iterator outer_it(*item_equal); Item_equal_iterator inner_it(*item_equal); Item_field *outer; while ((outer = outer_it++)) { Item_field *inner; while ((inner = inner_it++)) { if (!outer->field->eq(inner->field)) { if (add_key_field(thd, key_fields, *and_level, cond_func, outer, true, (Item **)&inner, 1, usable_tables, sargables)) return true; } } inner_it.rewind(); } } break; } return false; } /* Add all keys with uses 'field' for some keypart If field->and_level != and_level then only mark key_part as const_part RETURN 0 - OK 1 - Out of memory. */ static bool add_key_part(Key_use_array *keyuse_array, Key_field *key_field) { if (key_field->eq_func && !(key_field->optimize & KEY_OPTIMIZE_EXISTS)) { Field *const field = key_field->item_field->field; TABLE_LIST *const tl = key_field->item_field->table_ref; TABLE *const table = tl->table; for (uint key = 0; key < table->s->keys; key++) { if (!(table->keys_in_use_for_query.is_set(key))) continue; if (table->key_info[key].flags & (HA_FULLTEXT | HA_SPATIAL)) continue; // ToDo: ft-keys in non-ft queries. SerG uint key_parts = actual_key_parts(&table->key_info[key]); for (uint part = 0; part < key_parts; part++) { if (field->eq(table->key_info[key].key_part[part].field)) { const Key_use keyuse(tl, key_field->val, key_field->val->used_tables(), key, part, key_field->optimize & KEY_OPTIMIZE_REF_OR_NULL, (key_part_map)1 << part, ~(ha_rows)0, // will be set in optimize_keyuse key_field->null_rejecting, key_field->cond_guard, key_field->sj_pred_no); if (keyuse_array->push_back(keyuse)) return true; /* purecov: inspected */ } } } } return false; } /** Function parses WHERE condition and add key_use for FT index into key_use array if suitable MATCH function is found. Condition should be a set of AND expression, OR is not supported. MATCH function should be a part of simple expression. Simple expression is MATCH only function or MATCH is a part of comparison expression ('>=' or '>' operations are supported). It also sets FT_HINTS values(op_type, op_value). @param keyuse_array Key_use array @param stat JOIN_TAB structure @param cond WHERE condition @param usable_tables usable tables @param simple_match_expr true if this is the first call false otherwise. if MATCH function is found at first call it means that MATCH is simple expression, otherwise, in case of AND/OR condition this parameter will be false. @retval true if FT key was added to Key_use array @retval false if no key was added to Key_use array */ static bool add_ft_keys(Key_use_array *keyuse_array, JOIN_TAB *stat, Item *cond, table_map usable_tables, bool simple_match_expr) { Item_func_match *cond_func = NULL; if (!cond) return false; DBUG_ASSERT(cond->is_bool_func()); if (cond->type() == Item::FUNC_ITEM) { Item_func *func = down_cast(cond); Item_func::Functype functype = func->functype(); if (functype == Item_func::MATCH_FUNC) { func = down_cast(func->arguments()[0]); functype = func->functype(); } enum ft_operation op_type = FT_OP_NO; double op_value = 0.0; if (functype == Item_func::FT_FUNC) { cond_func = down_cast(func)->get_master(); cond_func->set_hints_op(op_type, op_value); } else if (func->arg_count == 2) { Item *arg0 = func->arguments()[0]; Item *arg1 = func->arguments()[1]; if (arg1->const_item() && arg0->type() == Item::FUNC_ITEM && down_cast(arg0)->functype() == Item_func::FT_FUNC && ((functype == Item_func::GE_FUNC && (op_value = arg1->val_real()) > 0) || (functype == Item_func::GT_FUNC && (op_value = arg1->val_real()) >= 0))) { cond_func = down_cast(arg0)->get_master(); if (functype == Item_func::GE_FUNC) op_type = FT_OP_GE; else if (functype == Item_func::GT_FUNC) op_type = FT_OP_GT; cond_func->set_hints_op(op_type, op_value); } else if (arg0->const_item() && arg1->type() == Item::FUNC_ITEM && down_cast(arg1)->functype() == Item_func::FT_FUNC && ((functype == Item_func::LE_FUNC && (op_value = arg0->val_real()) > 0) || (functype == Item_func::LT_FUNC && (op_value = arg0->val_real()) >= 0))) { cond_func = down_cast(arg1)->get_master(); if (functype == Item_func::LE_FUNC) op_type = FT_OP_GE; else if (functype == Item_func::LT_FUNC) op_type = FT_OP_GT; cond_func->set_hints_op(op_type, op_value); } } } else if (cond->type() == Item::COND_ITEM) { List_iterator_fast li(*down_cast(cond)->argument_list()); if (down_cast(cond)->functype() == Item_func::COND_AND_FUNC) { Item *item; while ((item = li++)) { if (add_ft_keys(keyuse_array, stat, item, usable_tables, false)) return true; } } } if (!cond_func || cond_func->key == NO_SUCH_KEY || !(usable_tables & cond_func->table_ref->map())) return false; cond_func->set_simple_expression(simple_match_expr); const Key_use keyuse(cond_func->table_ref, cond_func, cond_func->key_item()->used_tables(), cond_func->key, FT_KEYPART, 0, // optimize 0, // keypart_map ~(ha_rows)0, // ref_table_rows false, // null_rejecting NULL, // cond_guard UINT_MAX); // sj_pred_no return keyuse_array->push_back(keyuse); } /** Compares two keyuse elements. @param a first Key_use element @param b second Key_use element Compare Key_use elements so that they are sorted as follows: -# By table. -# By key for each table. -# By keypart for each key. -# Const values. -# Ref_or_null. @retval true If a < b. @retval false If a >= b. */ static bool sort_keyuse(const Key_use &a, const Key_use &b) { if (a.table_ref->tableno() != b.table_ref->tableno()) return a.table_ref->tableno() < b.table_ref->tableno(); if (a.key != b.key) return a.key < b.key; if (a.keypart != b.keypart) return a.keypart < b.keypart; // Place const values before other ones int res; if ((res = MY_TEST((a.used_tables & ~OUTER_REF_TABLE_BIT)) - MY_TEST((b.used_tables & ~OUTER_REF_TABLE_BIT)))) return res < 0; /* Place rows that are not 'OPTIMIZE_REF_OR_NULL' first */ return (a.optimize & KEY_OPTIMIZE_REF_OR_NULL) < (b.optimize & KEY_OPTIMIZE_REF_OR_NULL); } /* Add to Key_field array all 'ref' access candidates within nested join. This function populates Key_field array with entries generated from the ON condition of the given nested join, and does the same for nested joins contained within this nested join. @param thd session context @param[in] nested_join_table Nested join pseudo-table to process @param[in,out] end End of the key field array @param[in,out] and_level And-level @param[in,out] sargables Array of found sargable candidates @returns false if success, true if error @note We can add accesses to the tables that are direct children of this nested join (1), and are not inner tables w.r.t their neighbours (2). Example for #1 (outer brackets pair denotes nested join this function is invoked for): @code ... LEFT JOIN (t1 LEFT JOIN (t2 ... ) ) ON cond @endcode Example for #2: @code ... LEFT JOIN (t1 LEFT JOIN t2 ) ON cond @endcode In examples 1-2 for condition cond, we can add 'ref' access candidates to t1 only. Example #3: @code ... LEFT JOIN (t1, t2 LEFT JOIN t3 ON inner_cond) ON cond @endcode Here we can add 'ref' access candidates for t1 and t2, but not for t3. */ static bool add_key_fields_for_nj(THD *thd, JOIN *join, TABLE_LIST *nested_join_table, Key_field **end, uint *and_level, SARGABLE_PARAM **sargables) { List_iterator li(nested_join_table->nested_join->join_list); List_iterator li2(nested_join_table->nested_join->join_list); bool have_another = false; table_map tables = 0; TABLE_LIST *table; DBUG_ASSERT(nested_join_table->nested_join); while ((table = li++) || (have_another && (li = li2, have_another = false, (table = li++)))) { if (table->nested_join) { if (!table->join_cond_optim()) { /* It's a semi-join nest. Walk into it as if it wasn't a nest */ have_another = true; li2 = li; li = List_iterator(table->nested_join->join_list); } else { if (add_key_fields_for_nj(thd, join, table, end, and_level, sargables)) return true; } } else if (!table->join_cond_optim()) tables |= table->map(); } if (nested_join_table->join_cond_optim()) { if (add_key_fields(thd, join, end, and_level, nested_join_table->join_cond_optim(), tables, sargables)) return true; } return false; } /// @} (end of group RefOptimizerModule) /** Check for the presence of AGGFN(DISTINCT a) queries that may be subject to loose index scan. Check if the query is a subject to AGGFN(DISTINCT) using loose index scan (QUICK_GROUP_MIN_MAX_SELECT). Optionally (if out_args is supplied) will push the arguments of AGGFN(DISTINCT) to the list Check for every COUNT(DISTINCT), AVG(DISTINCT) or SUM(DISTINCT). These can be resolved by Loose Index Scan as long as all the aggregate distinct functions refer to the same fields. Thus: SELECT AGGFN(DISTINCT a, b), AGGFN(DISTINCT b, a)... => can use LIS SELECT AGGFN(DISTINCT a), AGGFN(DISTINCT a) ... => can use LIS SELECT AGGFN(DISTINCT a, b), AGGFN(DISTINCT a) ... => cannot use LIS SELECT AGGFN(DISTINCT a), AGGFN(DISTINCT b) ... => cannot use LIS etc. @param join the join to check @param[out] out_args Collect the arguments of the aggregate functions to a list. We don't worry about duplicates as these will be sorted out later in get_best_group_min_max. @return does the query qualify for indexed AGGFN(DISTINCT) @retval true it does @retval false AGGFN(DISTINCT) must apply distinct in it. */ bool is_indexed_agg_distinct(JOIN *join, List *out_args) { Item_sum **sum_item_ptr; bool result = false; Field_map first_aggdistinct_fields; if (join->primary_tables > 1 || /* reference more than 1 table */ join->select_distinct || /* or a DISTINCT */ join->select_lex->olap == ROLLUP_TYPE) /* Check (B3) for ROLLUP */ return false; if (join->make_sum_func_list(join->all_fields, join->fields_list, true)) return false; for (sum_item_ptr = join->sum_funcs; *sum_item_ptr; sum_item_ptr++) { Item_sum *sum_item = *sum_item_ptr; Field_map cur_aggdistinct_fields; Item *expr; /* aggregate is not AGGFN(DISTINCT) or more than 1 argument to it */ switch (sum_item->sum_func()) { case Item_sum::MIN_FUNC: case Item_sum::MAX_FUNC: continue; case Item_sum::COUNT_DISTINCT_FUNC: break; case Item_sum::AVG_DISTINCT_FUNC: case Item_sum::SUM_DISTINCT_FUNC: if (sum_item->get_arg_count() == 1) break; /* fall through */ default: return false; } for (uint i = 0; i < sum_item->get_arg_count(); i++) { expr = sum_item->get_arg(i); /* The AGGFN(DISTINCT) arg is not an attribute? */ if (expr->real_item()->type() != Item::FIELD_ITEM) return false; Item_field *item = static_cast(expr->real_item()); if (out_args) out_args->push_back(item); cur_aggdistinct_fields.set_bit(item->field->field_index); result = true; } /* If there are multiple aggregate functions, make sure that they all refer to exactly the same set of columns. */ if (first_aggdistinct_fields.is_clear_all()) first_aggdistinct_fields.merge(cur_aggdistinct_fields); else if (first_aggdistinct_fields != cur_aggdistinct_fields) return false; } return result; } /** Print keys that were appended to join_tab->const_keys because they can be used for GROUP BY or DISTINCT to the optimizer trace. @param trace The optimizer trace context we're adding info to @param join_tab The table the indexes cover @param new_keys The keys that are considered useful because they can be used for GROUP BY or DISTINCT @param cause Zero-terminated string with reason for adding indexes to const_keys @see add_group_and_distinct_keys() */ static void trace_indexes_added_group_distinct(Opt_trace_context *trace, const JOIN_TAB *join_tab, const Key_map new_keys, const char *cause) { if (likely(!trace->is_started())) return; KEY *key_info = join_tab->table()->key_info; Key_map existing_keys = join_tab->const_keys; uint nbrkeys = join_tab->table()->s->keys; Opt_trace_object trace_summary(trace, "const_keys_added"); { Opt_trace_array trace_key(trace, "keys"); for (uint j = 0; j < nbrkeys; j++) if (new_keys.is_set(j) && !existing_keys.is_set(j)) trace_key.add_utf8(key_info[j].name); } trace_summary.add_alnum("cause", cause); } /** Discover the indexes that might be used for GROUP BY or DISTINCT queries or indexes that might be used for SKIP SCAN. If the query has a GROUP BY clause, find all indexes that contain all GROUP BY fields, and add those indexes to join_tab->const_keys and join_tab->keys. If the query has a DISTINCT clause, find all indexes that contain all SELECT fields, and add those indexes to join_tab->const_keys and join_tab->keys. This allows later on such queries to be processed by a QUICK_GROUP_MIN_MAX_SELECT. If the query does not have GROUP BY clause or any aggregate function the function collects possible keys to use for skip scan access. Note that indexes that are not usable for resolving GROUP BY/DISTINCT may also be added in some corner cases. For example, an index covering 'a' and 'b' is not usable for the following query but is still added: "SELECT DISTINCT a+b FROM t1". This is not a big issue because a) although the optimizer will consider using the index, it will not chose it (so minor calculation cost added but not wrong result) and b) it applies only to corner cases. @param join @param join_tab @return None */ static void add_loose_index_scan_and_skip_scan_keys(JOIN *join, JOIN_TAB *join_tab) { DBUG_ASSERT(join_tab->const_keys.is_subset(join_tab->keys())); List indexed_fields; List_iterator indexed_fields_it(indexed_fields); ORDER *cur_group; Item_field *cur_item; const char *cause; /* Find the indexes that might be used for skip scan queries. */ if (hint_table_state(join->thd, join_tab->table_ref, SKIP_SCAN_HINT_ENUM, OPTIMIZER_SKIP_SCAN) && join->where_cond && join->primary_tables == 1 && !join->group_list && !is_indexed_agg_distinct(join, &indexed_fields) && !join->select_distinct) { join->where_cond->walk(&Item::collect_item_field_processor, enum_walk::POSTFIX, (uchar *)&indexed_fields); Key_map possible_keys; possible_keys.set_all(); join_tab->skip_scan_keys.clear_all(); while ((cur_item = indexed_fields_it++)) { if (cur_item->used_tables() != join_tab->table_ref->map()) return; possible_keys.intersect(cur_item->field->part_of_key); } join_tab->skip_scan_keys.merge(possible_keys); cause = "skip_scan"; return; } if (join->group_list) { /* Collect all query fields referenced in the GROUP clause. */ for (cur_group = join->group_list; cur_group; cur_group = cur_group->next) (*cur_group->item) ->walk(&Item::collect_item_field_processor, enum_walk::POSTFIX, (uchar *)&indexed_fields); cause = "group_by"; } else if (join->select_distinct) { /* Collect all query fields referenced in the SELECT clause. */ List &select_items = join->fields_list; List_iterator select_items_it(select_items); Item *item; while ((item = select_items_it++)) item->walk(&Item::collect_item_field_processor, enum_walk::POSTFIX, (uchar *)&indexed_fields); cause = "distinct"; } else if (join->tmp_table_param.sum_func_count && is_indexed_agg_distinct(join, &indexed_fields)) { /* SELECT list with AGGFN(distinct col). The query qualifies for loose index scan, and is_indexed_agg_distinct() has already collected all referenced fields into indexed_fields. */ join->streaming_aggregation = true; cause = "indexed_distinct_aggregate"; } else return; if (indexed_fields.elements == 0) return; Key_map possible_keys; possible_keys.set_all(); /* Intersect the keys of all group fields. */ while ((cur_item = indexed_fields_it++)) { if (cur_item->used_tables() != join_tab->table_ref->map()) { /* Doing GROUP BY or DISTINCT on a field in another table so no index in this table is usable */ return; } else possible_keys.intersect(cur_item->field->part_of_key); } /* At this point, possible_keys has key bits set only for usable indexes because indexed_fields is non-empty and if any of the fields belong to a different table the function would exit in the loop above. */ if (!possible_keys.is_clear_all() && !possible_keys.is_subset(join_tab->const_keys)) { trace_indexes_added_group_distinct(&join->thd->opt_trace, join_tab, possible_keys, cause); join_tab->const_keys.merge(possible_keys); join_tab->keys().merge(possible_keys); } DBUG_ASSERT(join_tab->const_keys.is_subset(join_tab->keys())); } /** Update keyuse array with all possible keys we can use to fetch rows. @param thd session context @param[out] keyuse Put here ordered array of Key_use structures @param join_tab Array in table number order @param tables Number of tables in join @param cond WHERE condition (note that the function analyzes join_tab[i]->join_cond() too) @param normal_tables Tables not inner w.r.t some outer join (ones for which we can make ref access based the WHERE clause) @param select_lex current SELECT @param[out] sargables Array of found sargable candidates @returns false if success, true if error */ static bool update_ref_and_keys(THD *thd, Key_use_array *keyuse, JOIN_TAB *join_tab, uint tables, Item *cond, table_map normal_tables, SELECT_LEX *select_lex, SARGABLE_PARAM **sargables) { DBUG_ASSERT(cond == nullptr || cond->is_bool_func()); uint and_level, i, found_eq_constant; Key_field *key_fields, *end, *field; size_t sz; uint m = max(select_lex->max_equal_elems, 1U); JOIN *const join = select_lex->join; /* We use the same piece of memory to store both Key_field and SARGABLE_PARAM structure. Key_field values are placed at the beginning this memory while SARGABLE_PARAM values are put at the end. All predicates that are used to fill arrays of Key_field and SARGABLE_PARAM structures have at most 2 arguments except BETWEEN predicates that have 3 arguments and IN predicates. This any predicate if it's not BETWEEN/IN can be used directly to fill at most 2 array elements, either of Key_field or SARGABLE_PARAM type. For a BETWEEN predicate 3 elements can be filled as this predicate is considered as saragable with respect to each of its argument. An IN predicate can require at most 1 element as currently it is considered as sargable only for its first argument. Multiple equality can add elements that are filled after substitution of field arguments by equal fields. There can be not more than select_lex->max_equal_elems such substitutions. */ sz = max(sizeof(Key_field), sizeof(SARGABLE_PARAM)) * (((select_lex->cond_count + 1) * 2 + select_lex->between_count) * m + 1); if (!(key_fields = (Key_field *)thd->alloc(sz))) return true; /* purecov: inspected */ and_level = 0; field = end = key_fields; *sargables = (SARGABLE_PARAM *)key_fields + (sz - sizeof((*sargables)[0].field)) / sizeof(SARGABLE_PARAM); /* set a barrier for the array of SARGABLE_PARAM */ (*sargables)[0].field = 0; if (cond) { if (add_key_fields(thd, join, &end, &and_level, cond, normal_tables, sargables)) return true; for (Key_field *fld = field; fld != end; fld++) { /* Mark that we can optimize LEFT JOIN */ if (fld->val->type() == Item::NULL_ITEM && !fld->item_field->field->real_maybe_null()) { /* Example: SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.a WHERE t2.a IS NULL; this just wants rows of t1 where t1.a does not exist in t2. */ fld->item_field->field->table->reginfo.not_exists_optimize = true; } } } for (i = 0; i < tables; i++) { /* Block the creation of keys for inner tables of outer joins. Here only the outer joins that can not be converted to inner joins are left and all nests that can be eliminated are flattened. In the future when we introduce conditional accesses for inner tables in outer joins these keys will be taken into account as well. */ if (join_tab[i].join_cond()) { if (add_key_fields(thd, join, &end, &and_level, join_tab[i].join_cond(), join_tab[i].table_ref->map(), sargables)) return true; } } /* Process ON conditions for the nested joins */ { List_iterator li(select_lex->top_join_list); TABLE_LIST *tl; while ((tl = li++)) { if (tl->nested_join && add_key_fields_for_nj(thd, join, tl, &end, &and_level, sargables)) return true; } } /* Generate keys descriptions for derived tables */ if (select_lex->materialized_derived_table_count || select_lex->table_func_count) { if (join->generate_derived_keys()) return true; } /* fill keyuse with found key parts */ for (; field != end; field++) { if (add_key_part(keyuse, field)) return true; } if (select_lex->ftfunc_list->elements) { if (add_ft_keys(keyuse, join_tab, cond, normal_tables, true)) return true; } /* Sort the array of possible keys and remove the following key parts: - ref if there is a keypart which is a ref and a const. (e.g. if there is a key(a,b) and the clause is a=3 and b=7 and b=t2.d, then we skip the key part corresponding to b=t2.d) - keyparts without previous keyparts (e.g. if there is a key(a,b,c) but only b < 5 (or a=2 and c < 3) is used in the query, we drop the partial key parts from consideration). Special treatment for ft-keys. */ if (!keyuse->empty()) { Key_use *save_pos, *use; std::sort(keyuse->begin(), keyuse->begin() + keyuse->size(), sort_keyuse); const Key_use key_end(NULL, NULL, 0, 0, 0, 0, 0, 0, false, NULL, 0); if (keyuse->push_back(key_end)) // added for easy testing return true; use = save_pos = keyuse->begin(); const Key_use *prev = &key_end; found_eq_constant = 0; for (i = 0; i < keyuse->size() - 1; i++, use++) { TABLE *const table = use->table_ref->table; if (!use->used_tables && use->optimize != KEY_OPTIMIZE_REF_OR_NULL) table->const_key_parts[use->key] |= use->keypart_map; if (use->keypart != FT_KEYPART) { if (use->key == prev->key && use->table_ref == prev->table_ref) { if (prev->keypart + 1 < use->keypart || (prev->keypart == use->keypart && found_eq_constant)) continue; /* remove */ } else if (use->keypart != 0) // First found must be 0 continue; } /* Protect against self assignment. The compiler *may* generate a call to memcpy() to do the assignment, and that is undefined behaviour (memory overlap). */ if (save_pos != use) *save_pos = *use; prev = use; found_eq_constant = !use->used_tables; /* Save ptr to first use */ if (!table->reginfo.join_tab->keyuse()) table->reginfo.join_tab->set_keyuse(save_pos); table->reginfo.join_tab->checked_keys.set_bit(use->key); save_pos++; } i = (uint)(save_pos - keyuse->begin()); keyuse->at(i) = key_end; keyuse->chop(i); } print_keyuse_array(&thd->opt_trace, keyuse); /* Number of functions here call val_x() methods, which might throw an error. Catch those errors here. */ return thd->is_error(); } /** Create a keyuse array for a table with a primary key. To be used when creating a materialized temporary table. @param thd THD pointer, for memory allocation @param keyparts Number of key parts in the primary key @param fields @param outer_exprs List of items used for key lookup @return Pointer to created keyuse array, or NULL if error */ Key_use_array *create_keyuse_for_table(THD *thd, uint keyparts, Item_field **fields, List outer_exprs) { void *mem = thd->alloc(sizeof(Key_use_array)); if (!mem) return NULL; Key_use_array *keyuses = new (mem) Key_use_array(thd->mem_root); List_iterator outer_expr(outer_exprs); for (uint keypartno = 0; keypartno < keyparts; keypartno++) { Item *const item = outer_expr++; Key_field key_field(fields[keypartno], item, 0, 0, true, // null_rejecting must be true for field items only, // add_not_null_conds() is incapable of handling // other item types. (item->type() == Item::FIELD_ITEM), NULL, UINT_MAX); if (add_key_part(keyuses, &key_field)) return NULL; } const Key_use key_end(NULL, NULL, 0, 0, 0, 0, 0, 0, false, NULL, 0); if (keyuses->push_back(key_end)) // added for easy testing return NULL; return keyuses; } /** Move const tables first in the position array. Increment the number of const tables and set same basic properties for the const table. A const table looked up by a key has type JT_CONST. A const table with a single row has type JT_SYSTEM. @param tab Table that is designated as a const table @param key The key definition to use for this table (NULL if table scan) */ void JOIN::mark_const_table(JOIN_TAB *tab, Key_use *key) { POSITION *const position = positions + const_tables; position->table = tab; position->key = key; position->rows_fetched = 1.0; // This is a const table position->filter_effect = 1.0; position->prefix_rowcount = 1.0; position->read_cost = 0.0; position->ref_depend_map = 0; position->loosescan_key = MAX_KEY; // Not a LooseScan position->sj_strategy = SJ_OPT_NONE; positions->use_join_buffer = false; // Move the const table as far down as possible in best_ref JOIN_TAB **pos = best_ref + const_tables + 1; for (JOIN_TAB *next = best_ref[const_tables]; next != tab; pos++) { JOIN_TAB *const tmp = pos[0]; pos[0] = next; next = tmp; } best_ref[const_tables] = tab; tab->set_type(key ? JT_CONST : JT_SYSTEM); const_table_map |= tab->table_ref->map(); const_tables++; } void JOIN::make_outerjoin_info() { DBUG_TRACE; DBUG_ASSERT(select_lex->outer_join); ASSERT_BEST_REF_IN_JOIN_ORDER(this); select_lex->reset_nj_counters(); for (uint i = const_tables; i < tables; ++i) { JOIN_TAB *const tab = best_ref[i]; TABLE *const table = tab->table(); if (!table) continue; TABLE_LIST *const tbl = tab->table_ref; /* If 'tbl' is inside a SJ/AJ nest served by materialization, we must limit setting first_inner, last_inner and first_upper for join nests inside the materialized table. Indeed it is the SJ-tmp table, and not 'tbl', which interacts with the nests outer to the SJ/AJ nest. */ const bool sj_mat_inner = sj_is_materialize_strategy(tab->get_sj_strategy()); if (tbl->outer_join) { /* Table tab is the only one inner table for outer join. (Like table t4 for the table reference t3 LEFT JOIN t4 ON t3.a=t4.a is in the query above.) */ tab->set_last_inner(i); tab->set_first_inner(i); tab->init_join_cond_ref(tbl); tab->cond_equal = tbl->cond_equal; /* If this outer join nest is embedded in another join nest, link the join-tabs: */ TABLE_LIST *const outer_join_nest = tbl->outer_join_nest(); if (outer_join_nest) { DBUG_ASSERT(outer_join_nest->nested_join->first_nested != NO_PLAN_IDX); if (!sj_mat_inner || (tab->emb_sj_nest->sj_inner_tables & best_ref[outer_join_nest->nested_join->first_nested] ->table_ref->map())) tab->set_first_upper(outer_join_nest->nested_join->first_nested); } } for (TABLE_LIST *embedding = tbl->embedding; embedding; embedding = embedding->embedding) { // When reaching the outer tables of the materialized temporary table, // the decoration for this table is complete. if (sj_mat_inner && embedding == tab->emb_sj_nest) break; // Ignore join nests that are not outer join nests: if (!embedding->join_cond_optim()) continue; NESTED_JOIN *const nested_join = embedding->nested_join; if (!nested_join->nj_counter) { /* Table tab is the first inner table for nested_join. Save reference to it in the nested join structure. */ nested_join->first_nested = i; // The table's condition is set to point to the join nest's condition tab->init_join_cond_ref(embedding); tab->cond_equal = tbl->cond_equal; TABLE_LIST *const outer_join_nest = embedding->outer_join_nest(); if (outer_join_nest) { DBUG_ASSERT(outer_join_nest->nested_join->first_nested != NO_PLAN_IDX); if (!sj_mat_inner || (tab->emb_sj_nest->sj_inner_tables & best_ref[outer_join_nest->nested_join->first_nested] ->table_ref->map())) tab->set_first_upper(outer_join_nest->nested_join->first_nested); } } if (tab->first_inner() == NO_PLAN_IDX) tab->set_first_inner(nested_join->first_nested); /* If including the sj-mat tmp table, this also implicitely includes the inner tables of the sj-nest. */ nested_join->nj_counter += tab->sj_mat_exec() ? tab->sj_mat_exec()->table_count : 1; if (nested_join->nj_counter < nested_join->nj_total) break; // Table tab is the last inner table for nested join. best_ref[nested_join->first_nested]->set_last_inner(i); } } } /** Build a condition guarded by match variables for embedded outer joins. When generating a condition for a table as part of an outer join condition or the WHERE condition, the table in question may also be part of an embedded outer join. In such cases, the condition must be guarded by the match variable for this embedded outer join. Such embedded outer joins may also be recursively embedded in other joins. The function recursively adds guards for a condition ascending from tab to root_tab, which is the first inner table of an outer join, or NULL if the condition being handled is the WHERE clause. @param join @param idx index of the first inner table for the inner-most outer join @param cond the predicate to be guarded (must be set) @param root_idx index of the inner table to stop at (is NO_PLAN_IDX if this is the WHERE clause) @return - pointer to the guarded predicate, if success - NULL if error */ static Item *add_found_match_trig_cond(JOIN *join, plan_idx idx, Item *cond, plan_idx root_idx) { ASSERT_BEST_REF_IN_JOIN_ORDER(join); DBUG_ASSERT(cond->is_bool_func()); for (; idx != root_idx; idx = join->best_ref[idx]->first_upper()) { if (!(cond = new Item_func_trig_cond(cond, NULL, join, idx, Item_func_trig_cond::FOUND_MATCH))) return NULL; cond->quick_fix_field(); cond->update_used_tables(); } return cond; } /** Helper for JOIN::attach_join_conditions(). Attaches bits of 'join_cond' to each table in the range [first_inner, last_tab], with proper guards. If 'sj_mat_cond' is true, we do not see first_inner (and tables on the same level of it) as inner to anything, as they're at the top from the POV of the materialization of the tmp table. So, if the SJ-mat nest is A LJ B, A will get a part of condition without any guard; B will get another part with a guard on A->found_match. It's like pushing a WHERE. */ bool JOIN::attach_join_condition_to_nest(plan_idx first_inner, plan_idx last_tab, Item *join_cond, bool is_sj_mat_cond) { /* Add the constant part of the join condition to the first inner table of the outer join. */ Item *cond = make_cond_for_table(thd, join_cond, const_table_map, table_map(0), false); if (cond) { if (!is_sj_mat_cond) { cond = new Item_func_trig_cond(cond, NULL, this, first_inner, Item_func_trig_cond::IS_NOT_NULL_COMPL); if (!cond) return true; if (cond->fix_fields(thd, NULL)) return true; } if (best_ref[first_inner]->and_with_condition(cond)) return true; } /* Split the non-constant part of the join condition into parts that can be attached to the inner tables of the outer join. */ for (plan_idx i = first_inner; i <= last_tab; ++i) { table_map prefix_tables = best_ref[i]->prefix_tables(); table_map added_tables = best_ref[i]->added_tables(); /* When handling the first inner table of an outer join, we may also reference all tables ahead of this table: */ if (i == first_inner) added_tables = prefix_tables; /* We need RAND_TABLE_BIT on the last inner table, in case there is a non-deterministic function in the join condition. (RAND_TABLE_BIT is set for the last table of the join plan, but this is not sufficient for join conditions, which may have a last inner table that is ahead of the last table of the join plan). */ if (i == last_tab) { prefix_tables |= RAND_TABLE_BIT; added_tables |= RAND_TABLE_BIT; } cond = make_cond_for_table(thd, join_cond, prefix_tables, added_tables, false); if (cond == NULL) continue; /* If the table is part of an outer join that is embedded in the outer join currently being processed, wrap the condition in triggered conditions for match variables of such embedded outer joins. */ if (!(cond = add_found_match_trig_cond( this, best_ref[i]->first_inner(), cond, is_sj_mat_cond ? NO_PLAN_IDX : first_inner))) return true; if (!is_sj_mat_cond) { // Add the guard turning the predicate off for the null-complemented row. cond = new Item_func_trig_cond(cond, NULL, this, first_inner, Item_func_trig_cond::IS_NOT_NULL_COMPL); if (!cond) return true; if (cond->fix_fields(thd, NULL)) return true; } // Add the generated condition to the existing table condition if (best_ref[i]->and_with_condition(cond)) return true; } return false; } /** Attach outer join conditions to generated table conditions in an optimal way. @param last_tab - Last table that has been added to the current plan. Pre-condition: If this is the last inner table of an outer join operation, a join condition is attached to the first inner table of that outer join operation. @return false if success, true if error. Outer join conditions are attached to individual tables, but we can analyze those conditions only when reaching the last inner table of an outer join operation. Notice also that a table can be last within several outer join nests, hence the outer for() loop of this function. Example: SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON t2.a=t3.a) ON t1.a=t2.a Table t3 is last both in the join nest (t2 - t3) and in (t1 - (t2 - t3)) Thus, join conditions for both join nests will be evaluated when reaching this table. For each outer join operation processed, the join condition is split optimally over the inner tables of the outer join. The split-out conditions are later referred to as table conditions (but note that several table conditions stemming from different join operations may be combined into a composite table condition). Example: Consider the above query once more. The predicate t1.a=t2.a can be evaluated when rows from t1 and t2 are ready, ie at table t2. The predicate t2.a=t3.a can be evaluated at table t3. Each non-constant split-out table condition is guarded by a match variable that enables it only when a matching row is found for all the embedded outer join operations. Each split-out table condition is guarded by a variable that turns the condition off just before a null-complemented row for the outer join operation is formed. Thus, the join condition will not be checked for the null-complemented row. */ bool JOIN::attach_join_conditions(plan_idx last_tab) { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(this); JOIN_TAB *lt = best_ref[last_tab]; for (plan_idx first_inner = lt->first_inner(); first_inner != NO_PLAN_IDX && best_ref[first_inner]->last_inner() == last_tab; first_inner = best_ref[first_inner]->first_upper()) { /* Table last_tab is the last inner table of an outer join, locate the corresponding join condition from the first inner table of the same outer join: */ Item *const join_cond = best_ref[first_inner]->join_cond(); DBUG_ASSERT(join_cond); if (attach_join_condition_to_nest(first_inner, last_tab, join_cond, false)) return true; } if (sj_is_materialize_strategy(lt->get_sj_strategy())) { plan_idx mat_tbl = NO_PLAN_IDX; /* The SJ nest's condition contains both the SJ equality condition and the WHERE of the replaced subquery. This WHERE must be pushed to SJ-inner tables for evaluation during materialization! */ Semijoin_mat_exec *sjm = nullptr; for (plan_idx j = last_tab;; j--) { sjm = best_ref[j]->sj_mat_exec(); if (sjm && sjm->sj_nest == lt->emb_sj_nest) { // 'j' is the sj-mat tmp table mat_tbl = j; break; } } DBUG_ASSERT(sjm); if (sjm->inner_table_index + sjm->table_count - 1 == (uint)last_tab) { // we're at last table of sjmat nest auto join_cond = best_ref[mat_tbl]->join_cond(); if (join_cond && attach_join_condition_to_nest(sjm->inner_table_index, last_tab, join_cond, true)) return true; } } /* See if 'last_tab' is the first inner of an antijoin nest, then add a IS NULL condition on it. By attaching the condition to the first inner table, we know that if it is not satisfied we can just jump back to the table right before it. */ if (lt->table_ref->embedding && lt->table_ref->embedding->is_aj_nest() && last_tab == lt->first_inner() && /* Exception: in A AJ (B LJ C) where C is a single table: there is no join nest for C as it's single; C->embedding is thus the AJ nest; but C->first_inner() is C (as it's the first inner of the LJ operation). In that case it's not the first inner table of the AJ. Catch this case: */ !lt->table_ref->join_cond()) { Item *cond = new Item_func_false(); if (!cond) return true; // This is a signal for JOIN::create_iterators cond->item_name.set(antijoin_null_cond); /* For A AJ B ON COND, we need an IS NULL condition which is tested on the result rows of A LEFT JOIN B ON COND. It must be tested only after the "match status" of a row of B has been decided, so is wrapped in a condition triggered by B->found_match. To have it test IS NULL, it's wrapped in a triggered condition which is false if B is not NULL-complemented. We needn't wrap this condition with triggers from upper nests, hence the last argument of the call below. */ cond = add_found_match_trig_cond(this, last_tab, cond, lt->first_upper()); if (!cond) return true; cond = new Item_func_trig_cond(cond, NULL, this, last_tab, Item_func_trig_cond::IS_NOT_NULL_COMPL); if (!cond) return true; if (cond->fix_fields(thd, NULL)) return true; if (lt->and_with_condition(cond)) return true; lt->table()->reginfo.not_exists_optimize = true; } return false; } /***************************************************************************** Remove calculation with tables that aren't yet read. Remove also tests against fields that are read through key where the table is not a outer join table. We can't remove tests that are made against columns which are stored in sorted order. *****************************************************************************/ static Item *part_of_refkey(TABLE *table, TABLE_REF *ref, Field *field) { uint ref_parts = ref->key_parts; if (ref_parts) { if (ref->has_guarded_conds()) return NULL; const KEY_PART_INFO *key_part = table->key_info[ref->key].key_part; for (uint part = 0; part < ref_parts; part++, key_part++) if (field->eq(key_part->field) && !(key_part->key_part_flag & HA_PART_KEY_SEG)) return ref->items[part]; } return NULL; } /** @brief Identify redundant predicates. @details Test if the equality predicate 'left_item = right_item' is redundant due to a REF-access already being set up on the table, where 'left_item' is part of the REF-key being used, and 'right_item' is equal to the key value specified for that field in the key. In such cases the predicate is known to be 'true' for any rows retrieved from that table. Thus it is redundant. @param left_item The Item_field possibly being part of A ref-KEY. @param right_item The equality value specified for 'left_item'. @return 'true' if the predicate is redundant. @note See comments in reduce_cond_for_table() about careful usage/modifications of test_if_ref(). */ static bool test_if_ref(Item_field *left_item, Item *right_item) { if (left_item->depended_from) return false; // don't even read join_tab of inner subquery! Field *field = left_item->field; JOIN_TAB *join_tab = field->table->reginfo.join_tab; if (join_tab == nullptr) return false; ASSERT_BEST_REF_IN_JOIN_ORDER(join_tab->join()); // No need to change const test if (!field->table->const_table && /* "ref_or_null" implements "x=y or x is null", not "x=y" */ (join_tab->type() != JT_REF_OR_NULL)) { Item *ref_item = part_of_refkey(field->table, &join_tab->ref(), field); if (ref_item && ref_item->eq(right_item, 1)) { right_item = right_item->real_item(); if (right_item->type() == Item::FIELD_ITEM) return (field->eq_def(down_cast(right_item)->field)); /* remove equalities injected by IN->EXISTS transformation */ else if (right_item->type() == Item::CACHE_ITEM) return down_cast(right_item)->eq_def(field); if (right_item->const_for_execution() && !(right_item->is_null())) { /* We can remove all fields except: 1. String data types: - For BINARY/VARBINARY fields with equality against a string: Ref access can return more rows than match the string. The reason seems to be that the string constant is not "padded" to the full length of the field when setting up ref access. @todo Change how ref access for BINARY/VARBINARY fields are done so that only qualifying rows are returned from the storage engine. 2. Float data type: Comparison of float can differ - When we search "WHERE field=value" using an index, the "value" side is converted from double to float by Field_float::store(), then two floats are compared. - When we search "WHERE field=value" without indexes, the "field" side is converted from float to double by Field_float::val_real(), then two doubles are compared. */ if (field->type() == MYSQL_TYPE_STRING && field->charset()->pad_attribute == NO_PAD) { /* For "NO PAD" collations on CHAR columns, this function must return false, because removal of trailing space in CHAR columns makes the table value and the index value compare differently. As the column strips trailing spaces, it can return false candidates. Further comparison of the actual table values is required. */ return false; } if (!((field->type() == MYSQL_TYPE_STRING || // 1 field->type() == MYSQL_TYPE_VARCHAR) && field->binary()) && !(field->type() == MYSQL_TYPE_FLOAT && field->decimals() > 0)) // 2 { return !right_item->save_in_field_no_warnings(field, true); } } } } return false; // keep predicate } /** @brief Remove redundant predicates from condition, return the reduced condition. @details A predicate of the form 'field = value' may be redundant if the (ref-) access choosen for the table use an index containing 'field', where 'value' is specified as (part of) its ref-key. This method remove such redundant predicates, thus reducing the condition, possibly eliminating it entirely. If comparing 'values' against outer-joined tables, these are possibly 'null-extended'. Thus the usage of these values in the ref-key, is not sufficient anymore to guarantee that 'field = value' is 'TRUE'. The 'null_extended' argument hold the table_map of any such possibly null-extended tables which are excluded from the above 'reduce' logic. Any tables referred in Item_func_trig_cond(FOUND_MATCH) conditions are aggregated into this null_extended table_map. @param cond The condition to be 'reduced'. @param null_extended table_map of possibly null-extended outer-tables. @return The condition with redundant predicates removed, possibly nullptr. */ static Item *reduce_cond_for_table(Item *cond, table_map null_extended) { DBUG_TRACE; DBUG_EXECUTE("where", print_where(current_thd, cond, "cond term", QT_ORDINARY);); if (cond->type() == Item::COND_ITEM) { List *arguments = down_cast(cond)->argument_list(); List_iterator li(*arguments); if (down_cast(cond)->functype() == Item_func::COND_AND_FUNC) { Item *item; while ((item = li++)) { Item *upd_item = reduce_cond_for_table(item, null_extended); if (upd_item == nullptr) { li.remove(); } else if (upd_item != item) { li.replace(upd_item); } } switch (arguments->elements) { case 0: return nullptr; // All 'true' -> And-cond true case 1: return arguments->head(); } } else { // Or list Item *item; while ((item = li++)) { Item *upd_item = reduce_cond_for_table(item, null_extended); if (upd_item == nullptr) { return nullptr; // Term 'true' -> entire Or-cond true } else if (upd_item != item) { li.replace(upd_item); } } } } else if (cond->type() == Item::FUNC_ITEM) { Item_func *func = down_cast(cond); if (func->functype() == Item_func::TRIG_COND_FUNC) { Item_func_trig_cond *func_trig = down_cast(func); if (func_trig->get_trig_type() == Item_func_trig_cond::FOUND_MATCH) { /* All inner-tables are possible null-extended when evaluating the 'FOUND_MATCH'. Thus, predicates embedded in this trig_cond, refering these tables, should not be eliminated. -> Add to null_extended map. */ null_extended |= func_trig->get_inner_tables(); } Item *cond_arg = func->arguments()[0]; Item *upd_arg = reduce_cond_for_table(cond_arg, null_extended); if (upd_arg == nullptr) { return nullptr; } func->arguments()[0] = upd_arg; } else if (func->functype() == Item_func::EQ_FUNC) { /* Remove equalities that are guaranteed to be true by use of 'ref' access method. Note that ref access implements "table1.field1 <=> table2.indexed_field2", i.e. if it passed a NULL field1, it will return NULL indexed_field2 if there are. Thus the equality "table1.field1 = table2.indexed_field2", is equivalent to "ref access AND table1.field1 IS NOT NULL" i.e. "ref access and proper setting/testing of ref->null_rejecting". Thus, we must be careful, that when we remove equalities below we also set ref->null_rejecting, and test it at execution; otherwise wrong NULL matches appear. So: - for the optimization phase, the code which is below, and the code in test_if_ref(), and in add_key_field(), must be kept in sync: if the applicability conditions in one place are relaxed, they should also be relaxed elsewhere. - for the execution phase, all possible execution methods must test ref->null_rejecting. */ Item *left_item = func->arguments()[0]->real_item(); Item *right_item = func->arguments()[1]->real_item(); if ((left_item->type() == Item::FIELD_ITEM && !(left_item->used_tables() & null_extended) && test_if_ref(down_cast(left_item), right_item)) || (right_item->type() == Item::FIELD_ITEM && !(right_item->used_tables() & null_extended) && test_if_ref(down_cast(right_item), left_item))) { return nullptr; } } } return cond; } /** @brief Remove redundant predicates and cache constant expressions. @details Do a final round on pushed down table conditions and HAVING clause. Optimize them for faster execution by removing predicates being obsolete due to the access path selected for the table. Constant expressions are also cached to avoid evaluating them for each row being compared. @return False if success, True if error @note This function is run after conditions have been pushed down to individual tables, so transformation is applied to JOIN_TAB::condition and not to the WHERE condition. */ bool JOIN::finalize_table_conditions() { /* Unnecessary to reduce conditions for const tables as they are only evaluated once. */ DBUG_ASSERT(!plan_is_const()); ASSERT_BEST_REF_IN_JOIN_ORDER(this); Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_array trace_tables(trace, "finalizing_table_conditions"); for (uint i = const_tables; i < tables; i++) { Item *condition = best_ref[i]->condition(); if (condition == nullptr) continue; /* Table predicates known to be true by the selected (ref-)access method may be removed from the condition */ Opt_trace_object trace_cond(trace); trace_cond.add_utf8_table(best_ref[i]->table_ref); trace_cond.add("original_table_condition", condition); /* Calculate the set of possibly NULL extended tables when 'condition' is evaluated. As it is evaluated on a found row from table, that table is subtracted from the nullable tables. Note that a FOUND_MATCH trigger is a special case, handled in reduce_cond_for_table(). */ const table_map null_extended = select_lex->outer_join & ~best_ref[i]->table_ref->map(); condition = reduce_cond_for_table(condition, null_extended); if (condition != nullptr) condition->update_used_tables(); /* Cache constant expressions in table conditions. (Moved down from WHERE- and ON-clauses) */ if (condition != nullptr) { cache_const_expr_arg cache_arg; cache_const_expr_arg *analyzer_arg = &cache_arg; condition = condition->compile( &Item::cache_const_expr_analyzer, (uchar **)&analyzer_arg, &Item::cache_const_expr_transformer, (uchar *)&cache_arg); if (condition == nullptr) return true; } trace_cond.add("final_table_condition ", condition); best_ref[i]->set_condition(condition); } /* Cache constant expressions in HAVING-clauses. */ if (having_cond != nullptr) { cache_const_expr_arg cache_arg; cache_const_expr_arg *analyzer_arg = &cache_arg; having_cond = having_cond->compile( &Item::cache_const_expr_analyzer, (uchar **)&analyzer_arg, &Item::cache_const_expr_transformer, (uchar *)&cache_arg); if (having_cond == nullptr) return true; } return false; } /** @brief Add keys to derived tables'/views' result tables in a list @details This function generates keys for all derived tables/views of the select_lex to which this join corresponds to with help of the TABLE_LIST:generate_keys function. @return false all keys were successfully added. @return true OOM error */ bool JOIN::generate_derived_keys() { DBUG_ASSERT(select_lex->materialized_derived_table_count || select_lex->table_func_count); for (TABLE_LIST *table = select_lex->leaf_tables; table; table = table->next_leaf) { table->derived_keys_ready = true; /* Process tables that aren't materialized yet. */ if (table->uses_materialization() && !table->table->is_created() && table->generate_keys()) return true; } return false; } /** For each materialized derived table/view, informs every TABLE of the key it will (not) use, segregates used keys from unused keys in TABLE::key_info, and eliminates unused keys. */ void JOIN::finalize_derived_keys() { DBUG_ASSERT(select_lex->materialized_derived_table_count || select_lex->table_func_count); ASSERT_BEST_REF_IN_JOIN_ORDER(this); bool adjust_key_count = false; table_map processed_tables = 0; for (uint i = 0; i < tables; i++) { JOIN_TAB *tab = best_ref[i]; TABLE *table = tab->table(); TABLE_LIST *table_ref = tab->table_ref; /* Save chosen key description if: 1) it's a materialized derived table 2) it's not yet instantiated 3) some keys are defined for it */ if (table && table_ref->uses_materialization() && // (1) !table->is_created() && // (2) table->s->keys > 0) // (3) { /* If there are two local references to the same CTE, and they use the same key, the iteration for the second reference is unnecessary. */ if (processed_tables & table_ref->map()) continue; adjust_key_count = true; Key_map used_keys; // Mark all unique indexes as in use, since they have an effect // (deduplication) whether any expression refers to them or not. // In particular, they are used if we want to materialize a UNION DISTINCT // directly into the derived table. for (uint i = 0; i < table->s->keys; ++i) { if (table->key_info[i].flags & HA_NOSAME) { used_keys.set_bit(i); } } // Same for the hash key used for manual deduplication, if any. (It always // has index 0 if it exists.) if (table->hash_field) { used_keys.set_bit(0); } Key_use *const keyuse = tab->position()->key; if (keyuse == nullptr && used_keys.is_clear_all()) { // Nothing uses any keys. tab->keys().clear_all(); tab->const_keys.clear_all(); continue; } Derived_refs_iterator it(table_ref); while (TABLE *t = it.get_next()) { /* Eliminate possible keys created by this JOIN and which it doesn't use. Collect all keys of this table which are used by any reference in this query block. Any other query block doesn't matter as: - either it was optimized before, so it's not using a key we may want to drop. - or it was optimized in this same window, so: * either we own the window, then any key we may want to drop is not visible to it. * or it owns the window, then we are using only existing keys. - or it will be optimized after, so it's not using any key yet. used_keys is a mix of possible used keys and existing used keys. */ if (t->pos_in_table_list->select_lex == select_lex) { JOIN_TAB *tab = t->reginfo.join_tab; Key_use *keyuse = tab->position()->key; if (keyuse) used_keys.set_bit(keyuse->key); } } uint new_idx = table->s->find_first_unused_tmp_key( used_keys); // Also updates table->s->first_unused_tmp_key. if (keyuse == nullptr) { continue; } const uint old_idx = keyuse->key; DBUG_ASSERT(old_idx != new_idx); if (old_idx > new_idx) { DBUG_ASSERT(table->s->owner_of_possible_tmp_keys == select_lex); it.rewind(); while (TABLE *t = it.get_next()) { /* Unlike the collection of used_keys, references from other query blocks must be considered here, as they need a key_info array consistent with the to-be-changed table->s->keys. */ t->copy_tmp_key(old_idx, it.is_first()); } } else new_idx = old_idx; // Index stays at same slot /* If the key was created by earlier-optimized query blocks, and is already used by nonlocal references, those don't need any further update: they are already setup to use it and we're not moving the key. If the key was created by this query block, nonlocal references cannot possibly be referencing it. In both cases, only local references need to update their Key_use. */ it.rewind(); while (TABLE *t = it.get_next()) { if (t->pos_in_table_list->select_lex != select_lex) continue; JOIN_TAB *tab = t->reginfo.join_tab; Key_use *keyuse = tab->position()->key; if (keyuse && keyuse->key == old_idx) { processed_tables |= t->pos_in_table_list->map(); const bool key_is_const = tab->const_keys.is_set(old_idx); // tab->keys() was never set, so must be set tab->keys().clear_all(); tab->keys().set_bit(new_idx); tab->const_keys.clear_all(); if (key_is_const) tab->const_keys.set_bit(new_idx); for (Key_use *it = keyuse; it->table_ref == tab->table_ref && it->key == old_idx; it++) it->key = new_idx; } } } } if (!adjust_key_count) return; // Finally we know how many keys remain in the table. for (uint i = 0; i < tables; i++) { JOIN_TAB *tab = best_ref[i]; TABLE *table = tab->table(); TABLE_LIST *table_ref = tab->table_ref; if (table && table_ref->uses_materialization() && !table->is_created() && table->s->keys > 0) { if (table->s->owner_of_possible_tmp_keys != select_lex) continue; /* Release lock. As a bonus, avoid double work when this loop later processes another local reference to the same table (similar to the processed_tables map in the first loop). */ table->s->owner_of_possible_tmp_keys = nullptr; Derived_refs_iterator it(table_ref); while (TABLE *t = it.get_next()) t->drop_unused_tmp_keys(it.is_first()); } } } /** @brief Extract a condition that can be checked after reading given table @param thd Current session. @param cond Condition to analyze @param tables Tables for which "current field values" are available @param used_table Table(s) that we are extracting the condition for (may also include PSEUDO_TABLE_BITS, and may be zero) @param exclude_expensive_cond Do not push expensive conditions @retval <>NULL Generated condition @retval = NULL Already checked, OR error @details Extract the condition that can be checked after reading the table(s) specified in @c used_table, given that current-field values for tables specified in @c tables bitmap are available. If @c used_table is 0, extract conditions for all tables in @c tables. This function can be used to extract conditions relevant for a table in a join order. Together with its caller, it will ensure that all conditions are attached to the first table in the join order where all necessary fields are available, and it will also ensure that a given condition is attached to only one table. To accomplish this, first initialize @c tables to the empty set. Then, loop over all tables in the join order, set @c used_table to the bit representing the current table, accumulate @c used_table into the @c tables set, and call this function. To ensure correct handling of const expressions and outer references, add the const table map and OUTER_REF_TABLE_BIT to @c used_table for the first table. To ensure that random expressions are evaluated for the final table, add RAND_TABLE_BIT to @c used_table for the final table. The function assumes that constant, inexpensive parts of the condition have already been checked. Constant, expensive parts will be attached to the first table in the join order, provided that the above call sequence is followed. The call order will ensure that conditions covering tables in @c tables minus those in @c used_table, have already been checked. The function takes into account that some parts of the condition are guaranteed to be true by employed 'ref' access methods (the code that does this is located at the end, search down for "EQ_FUNC"). @note make_cond_for_info_schema() uses an algorithm similar to make_cond_for_table(). */ Item *make_cond_for_table(THD *thd, Item *cond, table_map tables, table_map used_table, bool exclude_expensive_cond) { /* May encounter an Item_cache_int as "condition" here, so cannot assert that it satisfies is_bool_func(). */ /* Ignore this condition if 1. We are extracting conditions for a specific table, and 2. that table is not referenced by the condition, but not if 3. this is a constant condition not checked at optimization time and this is the first table we are extracting conditions for. (Assuming that used_table == tables for the first table.) */ if (used_table && // 1 !(cond->used_tables() & used_table) && // 2 !(cond->is_expensive() && used_table == tables)) // 3 return NULL; if (cond->type() == Item::COND_ITEM) { if (((Item_cond *)cond)->functype() == Item_func::COND_AND_FUNC) { /* Create new top level AND item */ Item_cond_and *new_cond = new Item_cond_and; if (!new_cond) return NULL; List_iterator li(*((Item_cond *)cond)->argument_list()); Item *item; while ((item = li++)) { Item *fix = make_cond_for_table(thd, item, tables, used_table, exclude_expensive_cond); if (fix) new_cond->argument_list()->push_back(fix); } switch (new_cond->argument_list()->elements) { case 0: return NULL; // Always true case 1: return new_cond->argument_list()->head(); default: if (new_cond->fix_fields(thd, NULL)) return NULL; return new_cond; } } else { // Or list Item_cond_or *new_cond = new Item_cond_or; if (!new_cond) return NULL; List_iterator li(*((Item_cond *)cond)->argument_list()); Item *item; while ((item = li++)) { Item *fix = make_cond_for_table(thd, item, tables, table_map(0), exclude_expensive_cond); if (!fix) return NULL; // Always true new_cond->argument_list()->push_back(fix); } if (new_cond->fix_fields(thd, NULL)) return NULL; return new_cond; } } /* Omit this condition if 1. Some tables referred by the condition are not available, or 2. We are extracting conditions for all tables, the condition is considered 'expensive', and we want to delay evaluation of such conditions to the execution phase. */ if ((cond->used_tables() & ~tables) || // 1 (!used_table && exclude_expensive_cond && cond->is_expensive())) // 2 return NULL; return cond; } /** Separates the predicates in a join condition and pushes them to the join step where all involved tables are available in the join prefix. ON clauses from JOIN expressions are also pushed to the most appropriate step. @param join Join object where predicates are pushed. @param cond Pointer to condition which may contain an arbitrary number of predicates, combined using AND, OR and XOR items. If NULL, equivalent to a predicate that returns true for all row combinations. @retval true Found impossible WHERE clause, or out-of-memory @retval false Other */ static bool make_join_select(JOIN *join, Item *cond) { DBUG_ASSERT(cond == nullptr || cond->is_bool_func()); THD *thd = join->thd; Opt_trace_context *const trace = &thd->opt_trace; DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(join); // Add IS NOT NULL conditions to table conditions: add_not_null_conds(join); /* Extract constant conditions that are part of the WHERE clause. Constant parts of join conditions from outer joins are attached to the appropriate table condition in JOIN::attach_join_conditions(). */ if (cond) /* Because of QUICK_GROUP_MIN_MAX_SELECT */ { /* there may be a select without a cond. */ if (join->primary_tables > 1) cond->update_used_tables(); // Table number may have changed if (join->plan_is_const() && join->select_lex->master_unit() == thd->lex->unit) // The outer-most query block join->const_table_map |= RAND_TABLE_BIT; } /* Extract conditions that depend on constant tables. The const part of the query's WHERE clause can be checked immediately and if it is not satisfied then the join has empty result */ Item *const_cond = NULL; if (cond) const_cond = make_cond_for_table(thd, cond, join->const_table_map, table_map(0), true); // Add conditions added by add_not_null_conds() for (uint i = 0; i < join->const_tables; i++) { if (and_conditions(&const_cond, join->best_ref[i]->condition())) return true; } DBUG_EXECUTE("where", print_where(thd, const_cond, "constants", QT_ORDINARY);); if (const_cond != nullptr && evaluate_during_optimization(const_cond, join->select_lex)) { const bool const_cond_result = const_cond->val_int() != 0; if (thd->is_error()) return true; Opt_trace_object trace_const_cond(trace); trace_const_cond.add("condition_on_constant_tables", const_cond) .add("condition_value", const_cond_result); if (const_cond_result) { if (join->plan_is_const() && !(cond->used_tables() & ~join->const_table_map)) { DBUG_PRINT("info", ("Found always true WHERE condition")); join->where_cond = NULL; } } else { DBUG_PRINT("info", ("Found impossible WHERE condition")); return true; } } /* Extract remaining conditions from WHERE clause and join conditions, and attach them to the most appropriate table condition. This means that a condition will be evaluated as soon as all fields it depends on are available. For outer join conditions, the additional criterion is that we must have determined whether outer-joined rows are available, or have been NULL-extended, see JOIN::attach_join_conditions() for details. */ { Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_conditions(trace, "attaching_conditions_to_tables"); trace_conditions.add("original_condition", cond); Opt_trace_array trace_attached_comp(trace, "attached_conditions_computation"); for (uint i = join->const_tables; i < join->tables; i++) { JOIN_TAB *const tab = join->best_ref[i]; if (!tab->position()) continue; /* first_inner is the X in queries like: SELECT * FROM t1 LEFT OUTER JOIN (t2 JOIN t3) ON X */ const plan_idx first_inner = tab->first_inner(); const table_map used_tables = tab->prefix_tables(); const table_map current_map = tab->added_tables(); Item *tmp = NULL; if (cond) tmp = make_cond_for_table(thd, cond, used_tables, current_map, false); /* Add conditions added by add_not_null_conds(). */ if (tab->condition() && and_conditions(&tmp, tab->condition())) return true; if (cond && !tmp && tab->quick()) { // Outer join DBUG_ASSERT(tab->type() == JT_RANGE || tab->type() == JT_INDEX_MERGE); /* Hack to handle the case where we only refer to a table in the ON part of an OUTER JOIN. In this case we want the code below to check if we should use 'quick' instead. */ DBUG_PRINT("info", ("Item_func_true")); tmp = new Item_func_true(); // Always true } if (tmp || !cond || tab->type() == JT_REF || tab->type() == JT_REF_OR_NULL || tab->type() == JT_EQ_REF || first_inner != NO_PLAN_IDX) { DBUG_EXECUTE("where", print_where(thd, tmp, tab->table()->alias, QT_ORDINARY);); /* If tab is an inner table of an outer join operation, add a match guard to the pushed down predicate. The guard will turn the predicate on only after the first match for outer tables is encountered. */ if (cond && tmp) { /* Because of QUICK_GROUP_MIN_MAX_SELECT there may be a select without a cond, so neutralize the hack above. */ if (!(tmp = add_found_match_trig_cond(join, first_inner, tmp, NO_PLAN_IDX))) return true; tab->set_condition(tmp); } else { tab->set_condition(NULL); } DBUG_EXECUTE("where", print_where(thd, tmp, tab->table()->alias, QT_ORDINARY);); if (tab->quick()) { if (tab->needed_reg.is_clear_all() && tab->type() != JT_CONST) { /* We keep (for now) the QUICK AM calculated in get_quick_record_count(). */ DBUG_ASSERT(tab->quick()->is_valid()); } else { delete tab->quick(); tab->set_quick(NULL); } } if ((tab->type() == JT_ALL || tab->type() == JT_RANGE || tab->type() == JT_INDEX_MERGE || tab->type() == JT_INDEX_SCAN) && tab->use_quick != QS_RANGE) { /* We plan to scan (table/index/range scan). Check again if we should use an index. We can use an index if: 1a) There is a condition that range optimizer can work on, and 1b) There are non-constant conditions on one or more keys, and 1c) Some of the non-constant fields may have been read already. This may be the case if this is not the first table in the join OR this is a subselect with non-constant conditions referring to an outer table (dependent subquery) or, 2a) There are conditions only relying on constants 2b) This is the first non-constant table 2c) There is a limit of rows to read that is lower than the fanout for this table, predicate filters included (i.e., the estimated number of rows that will be produced for this table per row combination of previous tables) 2d) The query is NOT run with FOUND_ROWS() (because in that case we have to scan through all rows to count them anyway) */ enum { DONT_RECHECK, NOT_FIRST_TABLE, LOW_LIMIT } recheck_reason = DONT_RECHECK; DBUG_ASSERT(tab->const_keys.is_subset(tab->keys())); const join_type orig_join_type = tab->type(); const QUICK_SELECT_I *const orig_quick = tab->quick(); if (cond && // 1a (tab->keys() != tab->const_keys) && // 1b (i > 0 || // 1c (join->select_lex->master_unit()->item && cond->used_tables() & OUTER_REF_TABLE_BIT))) recheck_reason = NOT_FIRST_TABLE; else if (!tab->const_keys.is_clear_all() && // 2a i == join->const_tables && // 2b (join->unit->select_limit_cnt < (tab->position()->rows_fetched * tab->position()->filter_effect)) && // 2c !join->calc_found_rows) // 2d recheck_reason = LOW_LIMIT; // Don't recheck if the storage engine does not support index access. if ((tab->table()->file->ha_table_flags() & HA_NO_INDEX_ACCESS) != 0) recheck_reason = DONT_RECHECK; if (tab->position()->sj_strategy == SJ_OPT_LOOSE_SCAN) { /* Semijoin loose scan has settled for a certain index-based access method with suitable characteristics, don't substitute it. */ recheck_reason = DONT_RECHECK; } if (recheck_reason != DONT_RECHECK) { Opt_trace_object trace_one_table(trace); trace_one_table.add_utf8_table(tab->table_ref); Opt_trace_object trace_table(trace, "rechecking_index_usage"); if (recheck_reason == NOT_FIRST_TABLE) trace_table.add_alnum("recheck_reason", "not_first_table"); else trace_table.add_alnum("recheck_reason", "low_limit") .add("limit", join->unit->select_limit_cnt) .add("row_estimate", tab->position()->rows_fetched * tab->position()->filter_effect); /* Join with outer join condition */ Item *orig_cond = tab->condition(); tab->and_with_condition(tab->join_cond()); /* We can't call sel->cond->fix_fields, as it will break tab->join_cond() if it's AND condition (fix_fields currently removes extra AND/OR levels). Yet attributes of the just built condition are not needed. Thus we call sel->cond->quick_fix_field for safety. */ if (tab->condition() && !tab->condition()->fixed) tab->condition()->quick_fix_field(); Key_map usable_keys = tab->keys(); enum_order interesting_order = ORDER_NOT_RELEVANT; if (recheck_reason == LOW_LIMIT) { int read_direction = 0; /* If the current plan is to use range, then check if the already selected index provides the order dictated by the ORDER BY clause. */ if (tab->quick() && tab->quick()->index != MAX_KEY) { const uint ref_key = tab->quick()->index; bool skip_quick; read_direction = test_if_order_by_key( &join->order, tab->table(), ref_key, NULL, &skip_quick); if (skip_quick) read_direction = 0; /* If the index provides order there is no need to recheck index usage; we already know from the former call to test_quick_select() that a range scan on the chosen index is cheapest. Note that previous calls to test_quick_select() did not take order direction (ASC/DESC) into account, so in case of DESC ordering we still need to recheck. */ if ((read_direction == 1) || (read_direction == -1 && tab->quick()->reverse_sorted())) { recheck_reason = DONT_RECHECK; } } if (recheck_reason != DONT_RECHECK) { int best_key = -1; ha_rows select_limit = join->unit->select_limit_cnt; /* Use index specified in FORCE INDEX FOR ORDER BY, if any. */ if (tab->table()->force_index) usable_keys.intersect(tab->table()->keys_in_use_for_order_by); /* Do a cost based search on the indexes that give sort order */ test_if_cheaper_ordering( tab, &join->order, tab->table(), usable_keys, -1, select_limit, &best_key, &read_direction, &select_limit); if (best_key < 0) recheck_reason = DONT_RECHECK; // No usable keys else { // Only usable_key is the best_key chosen usable_keys.clear_all(); usable_keys.set_bit(best_key); interesting_order = (read_direction == -1 ? ORDER_DESC : ORDER_ASC); } } } bool search_if_impossible = recheck_reason != DONT_RECHECK; if (search_if_impossible) { if (tab->quick()) { delete tab->quick(); tab->set_type(JT_ALL); } QUICK_SELECT_I *qck; search_if_impossible = test_quick_select( thd, usable_keys, used_tables & ~tab->table_ref->map(), join->calc_found_rows ? HA_POS_ERROR : join->unit->select_limit_cnt, false, // don't force quick range interesting_order, tab, tab->condition(), &tab->needed_reg, &qck, tab->table()->force_index) < 0; tab->set_quick(qck); } tab->set_condition(orig_cond); if (search_if_impossible) { /* Before reporting "Impossible WHERE" for the whole query we have to check isn't it only "impossible ON" instead */ if (!tab->join_cond()) return 1; // No ON, so it's really "impossible WHERE" Opt_trace_object trace_without_on(trace, "without_ON_clause"); if (tab->quick()) { delete tab->quick(); tab->set_type(JT_ALL); } QUICK_SELECT_I *qck; const bool impossible_where = test_quick_select( thd, tab->keys(), used_tables & ~tab->table_ref->map(), join->calc_found_rows ? HA_POS_ERROR : join->unit->select_limit_cnt, false, // don't force quick range ORDER_NOT_RELEVANT, tab, tab->condition(), &tab->needed_reg, &qck, tab->table()->force_index) < 0; tab->set_quick(qck); if (impossible_where) return 1; // Impossible WHERE } /* Access method changed. This is after deciding join order and access method for all other tables so the info updated below will not have any effect on the execution plan. */ if (tab->quick()) tab->set_type(calc_join_type(tab->quick()->get_type())); } // end of "if (recheck_reason != DONT_RECHECK)" if (!tab->table()->quick_keys.is_subset(tab->checked_keys) || !tab->needed_reg.is_subset(tab->checked_keys)) { tab->keys().merge(tab->table()->quick_keys); tab->keys().merge(tab->needed_reg); /* The logic below for assigning tab->use_quick is strange. It bases the decision of which access method to use (dynamic range, range, scan) based on seemingly unrelated information like the presense of another index with too bad selectivity to be used. Consider the following scenario: The join optimizer has decided to use join order (t1,t2), and 'tab' is currently t2. Further, assume that there is a join condition between t1 and t2 using some range operator (e.g. "t1.x < t2.y"). It has been decided that a table scan is best for t2. make_join_select() then reran the range optimizer a few lines up because there is an index 't2.good_idx' covering the t2.y column. If 'good_idx' is the only index in t2, the decision below will be to use dynamic range. However, if t2 also has another index 't2.other' which the range access method can be used on but selectivity is bad (#rows estimate is high), then table scan is chosen instead. Thus, the choice of DYNAMIC RANGE vs SCAN depends on the presense of an index that has so bad selectivity that it will not be used anyway. */ if (!tab->needed_reg.is_clear_all() && (tab->table()->quick_keys.is_clear_all() || (tab->quick() && (tab->quick()->records >= 100L)))) { tab->use_quick = QS_DYNAMIC_RANGE; tab->set_type(JT_ALL); } else tab->use_quick = QS_RANGE; } if (tab->type() != orig_join_type || tab->quick() != orig_quick) // Access method changed tab->position()->filter_effect = COND_FILTER_STALE; } } if (join->attach_join_conditions(i)) return true; } trace_attached_comp.end(); /* In outer joins the loop above, in iteration for table #i, may push conditions to a table before #i. Thus, the processing below has to be in a separate loop: */ Opt_trace_array trace_attached_summary(trace, "attached_conditions_summary"); for (uint i = join->const_tables; i < join->tables; i++) { JOIN_TAB *const tab = join->best_ref[i]; if (!tab->table()) continue; Item *const cond = tab->condition(); Opt_trace_object trace_one_table(trace); trace_one_table.add_utf8_table(tab->table_ref).add("attached", cond); if (cond && cond->has_subquery()) // traverse only if needed { /* Why we pass walk_subquery=false: imagine WHERE t1.col IN (SELECT * FROM t2 WHERE t2.col IN (SELECT * FROM t3) and tab==t1. The grandchild subquery (SELECT * FROM t3) should not be marked as "in condition of t1" but as "in condition of t2", for correct calculation of the number of its executions. */ std::pair pair_object(join->select_lex, i); cond->walk(&Item::inform_item_in_cond_of_tab, enum_walk::POSTFIX, pointer_cast(&pair_object)); } } } return 0; } /** Remove the following expressions from ORDER BY and GROUP BY: Constant expressions @n Expression that only uses tables that are of type EQ_REF and the reference is in the ORDER list or if all refereed tables are of the above type. In the following, the X field can be removed: @code SELECT * FROM t1,t2 WHERE t1.a=t2.a ORDER BY t1.a,t2.X SELECT * FROM t1,t2,t3 WHERE t1.a=t2.a AND t2.b=t3.b ORDER BY t1.a,t3.X @endcode These can't be optimized: @code SELECT * FROM t1,t2 WHERE t1.a=t2.a ORDER BY t2.X,t1.a SELECT * FROM t1,t2 WHERE t1.a=t2.a AND t1.b=t2.b ORDER BY t1.a,t2.c SELECT * FROM t1,t2 WHERE t1.a=t2.a ORDER BY t2.b,t1.a @endcode @param join join object @param start_order clause being analyzed (ORDER BY, GROUP BY...) @param tab table @param cached_eq_ref_tables bitmap: bit Z is set if the table of map Z was already the subject of an eq_ref_table() call for the same clause; then the return value of this previous call can be found at bit Z of 'eq_ref_tables' @param eq_ref_tables see above. */ static bool eq_ref_table(JOIN *join, ORDER *start_order, JOIN_TAB *tab, table_map *cached_eq_ref_tables, table_map *eq_ref_tables) { /* We can skip const tables only if not an outer table */ if (tab->type() == JT_CONST && tab->first_inner() == NO_PLAN_IDX) return true; if (tab->type() != JT_EQ_REF || tab->table()->is_nullable()) return false; const table_map map = tab->table_ref->map(); uint found = 0; for (Item **ref_item = tab->ref().items, **end = ref_item + tab->ref().key_parts; ref_item != end; ref_item++) { if (!(*ref_item)->const_item()) { // Not a const ref ORDER *order; for (order = start_order; order; order = order->next) { if ((*ref_item)->eq(order->item[0], 0)) break; } if (order) { if (!(order->used & map)) { found++; order->used |= map; } continue; // Used in ORDER BY } if (!only_eq_ref_tables(join, start_order, (*ref_item)->used_tables(), cached_eq_ref_tables, eq_ref_tables)) return false; } } /* Check that there was no reference to table before sort order */ for (; found && start_order; start_order = start_order->next) { if (start_order->used & map) { found--; continue; } if (start_order->depend_map & map) return false; } return true; } /// @see eq_ref_table() static bool only_eq_ref_tables(JOIN *join, ORDER *order, table_map tables, table_map *cached_eq_ref_tables, table_map *eq_ref_tables) { tables &= ~PSEUDO_TABLE_BITS; for (JOIN_TAB **tab = join->map2table; tables; tab++, tables >>= 1) { if (tables & 1) { const table_map map = (*tab)->table_ref->map(); bool is_eq_ref; if (*cached_eq_ref_tables & map) // then there exists a cached bit is_eq_ref = *eq_ref_tables & map; else { is_eq_ref = eq_ref_table(join, order, *tab, cached_eq_ref_tables, eq_ref_tables); if (is_eq_ref) *eq_ref_tables |= map; else *eq_ref_tables &= ~map; *cached_eq_ref_tables |= map; // now there exists a cached bit } if (!is_eq_ref) return false; } } return true; } /** Check if an expression in ORDER BY or GROUP BY is a duplicate of a preceding expression. @param first_order the first expression in the ORDER BY or GROUP BY clause @param possible_dup the expression that might be a duplicate of another expression preceding it the ORDER BY or GROUP BY clause @returns true if possible_dup is a duplicate, false otherwise */ static bool duplicate_order(const ORDER *first_order, const ORDER *possible_dup) { const ORDER *order; for (order = first_order; order; order = order->next) { if (order == possible_dup) { // all expressions preceding possible_dup have been checked. return false; } else { const Item *it1 = order->item[0]->real_item(); const Item *it2 = possible_dup->item[0]->real_item(); if (it1->eq(it2, 0)) return true; } } return false; } /** Remove all constants and check if ORDER only contains simple expressions. simple_order is set to 1 if sort_order only uses fields from head table and the head table is not a LEFT JOIN table. @param first_order List of SORT or GROUP order @param cond WHERE statement @param change_list Set to 1 if we should remove things from list. If this is not set, then only simple_order is calculated. @param simple_order Set to 1 if we are only using simple expressions @param group_by True if order represents a grouping operation @return Returns new sort order */ ORDER *JOIN::remove_const(ORDER *first_order, Item *cond, bool change_list, bool *simple_order, bool group_by) { DBUG_TRACE; ASSERT_BEST_REF_IN_JOIN_ORDER(this); if (plan_is_const()) return change_list ? 0 : first_order; // No need to sort Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_disable_I_S trace_disabled(trace, first_order == NULL); Opt_trace_object trace_simpl( trace, group_by ? "simplifying_group_by" : "simplifying_order_by"); if (trace->is_started()) { String str; SELECT_LEX::print_order( thd, &str, first_order, enum_query_type(QT_TO_SYSTEM_CHARSET | QT_SHOW_SELECT_NUMBER | QT_NO_DEFAULT_DB)); trace_simpl.add_utf8("original_clause", str.ptr(), str.length()); } Opt_trace_array trace_each_item(trace, "items"); ORDER *order, **prev_ptr; JOIN_TAB *const first_tab = best_ref[const_tables]; table_map first_table = first_tab->table_ref->map(); table_map not_const_tables = ~const_table_map; table_map ref; // Caches to avoid repeating eq_ref_table() calls, @see eq_ref_table() table_map eq_ref_tables = 0, cached_eq_ref_tables = 0; prev_ptr = &first_order; *simple_order = !first_tab->join_cond(); // De-optimization in conjunction with window functions if (group_by && m_windows.elements > 0) *simple_order = false; /* NOTE: A variable of not_const_tables ^ first_table; breaks gcc 2.7 */ update_depend_map(first_order); for (order = first_order; order; order = order->next) { Opt_trace_object trace_one_item(trace); trace_one_item.add("item", order->item[0]); table_map order_tables = order->item[0]->used_tables(); if (order->item[0]->has_aggregation() || order->item[0]->has_wf() || /* If the outer table of an outer join is const (either by itself or after applying WHERE condition), grouping on a field from such a table will be optimized away and filesort without temporary table will be used unless we prevent that now. Filesort is not fit to handle joins and the join condition is not applied. We can't detect the case without an expensive test, however, so we force temporary table for all queries containing more than one table, ROLLUP, and an outer join. */ (primary_tables > 1 && rollup.state == ROLLUP::STATE_INITED && select_lex->outer_join)) *simple_order = 0; // Must do a temp table to sort else if ((order_tables & not_const_tables) == 0 && evaluate_during_optimization(order->item[0], select_lex)) { if (order->item[0]->has_subquery()) { if (!thd->lex->is_explain()) { Opt_trace_array trace_subselect(trace, "subselect_evaluation"); String str; order->item[0]->val_str(&str); } order->item[0]->mark_subqueries_optimized_away(); } trace_one_item.add("uses_only_constant_tables", true); continue; // skip const item } else if (duplicate_order(first_order, order)) { /* If 'order' is a duplicate of an expression earlier in the ORDER/GROUP BY sequence, it can be removed from the ORDER BY or GROUP BY clause. */ trace_one_item.add("duplicate_item", true); continue; } else if (order->in_field_list && order->item[0]->has_subquery()) /* If the order item is a subquery that is also in the field list, a temp table should be used to avoid evaluating the subquery for each row both when a) creating a sort index and b) getting the value. Example: "SELECT (SELECT ... ) as a ... GROUP BY a;" */ *simple_order = false; else { if (order_tables & (RAND_TABLE_BIT | OUTER_REF_TABLE_BIT)) *simple_order = 0; else { if (cond && const_expression_in_where(cond, order->item[0])) { trace_one_item.add("equals_constant_in_where", true); continue; } if ((ref = order_tables & (not_const_tables ^ first_table))) { if (!(order_tables & first_table) && only_eq_ref_tables(this, first_order, ref, &cached_eq_ref_tables, &eq_ref_tables)) { trace_one_item.add("eq_ref_to_preceding_items", true); continue; } *simple_order = 0; // Must do a temp table to sort } } } if (change_list) *prev_ptr = order; // use this entry prev_ptr = &order->next; } if (change_list) *prev_ptr = 0; if (prev_ptr == &first_order) // Nothing to sort/group *simple_order = 1; DBUG_PRINT("exit", ("simple_order: %d", (int)*simple_order)); trace_each_item.end(); trace_simpl.add("resulting_clause_is_simple", *simple_order); if (trace->is_started() && change_list) { String str; SELECT_LEX::print_order( thd, &str, first_order, enum_query_type(QT_TO_SYSTEM_CHARSET | QT_SHOW_SELECT_NUMBER | QT_NO_DEFAULT_DB)); trace_simpl.add_utf8("resulting_clause", str.ptr(), str.length()); } return first_order; } /** Optimize conditions by a) applying transitivity to build multiple equality predicates (MEP): if x=y and y=z the MEP x=y=z is built. b) apply constants where possible. If the value of x is known to be 42, x is replaced with a constant of value 42. By transitivity, this also applies to MEPs, so the MEP in a) will become 42=x=y=z. c) remove conditions that are always false or always true @param thd Thread handler @param[in,out] cond WHERE or HAVING condition to optimize @param[out] cond_equal The built multiple equalities @param join_list list of join operations with join conditions = NULL: Called for HAVING condition @param[out] cond_value Not changed if cond was empty COND_TRUE if cond is always true COND_FALSE if cond is impossible COND_OK otherwise @returns false if success, true if error */ bool optimize_cond(THD *thd, Item **cond, COND_EQUAL **cond_equal, List *join_list, Item::cond_result *cond_value) { DBUG_TRACE; Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_cond(trace, "condition_processing"); trace_cond.add_alnum("condition", join_list ? "WHERE" : "HAVING"); trace_cond.add("original_condition", *cond); Opt_trace_array trace_steps(trace, "steps"); /* Enter this function a) For a WHERE condition or a query having outer join. b) For a HAVING condition. */ DBUG_ASSERT(*cond || join_list); /* Build all multiple equality predicates and eliminate equality predicates that can be inferred from these multiple equalities. For each reference of a field included into a multiple equality that occurs in a function set a pointer to the multiple equality predicate. Substitute a constant instead of this field if the multiple equality contains a constant. This is performed for the WHERE condition and any join conditions, but not for the HAVING condition. */ if (join_list) { Opt_trace_object step_wrapper(trace); step_wrapper.add_alnum("transformation", "equality_propagation"); { Opt_trace_disable_I_S disable_trace_wrapper( trace, !(*cond && (*cond)->has_subquery())); Opt_trace_array trace_subselect(trace, "subselect_evaluation"); if (build_equal_items(thd, *cond, cond, NULL, true, join_list, cond_equal)) return true; } step_wrapper.add("resulting_condition", *cond); } /* change field = field to field = const for each found field = const */ if (*cond) { Opt_trace_object step_wrapper(trace); step_wrapper.add_alnum("transformation", "constant_propagation"); { Opt_trace_disable_I_S disable_trace_wrapper(trace, !(*cond)->has_subquery()); Opt_trace_array trace_subselect(trace, "subselect_evaluation"); if (propagate_cond_constants(thd, NULL, *cond, *cond)) return true; } step_wrapper.add("resulting_condition", *cond); } /* Remove all instances of item == item Remove all and-levels where CONST item != CONST item */ DBUG_EXECUTE("where", print_where(thd, *cond, "after const change", QT_ORDINARY);); if (*cond) { Opt_trace_object step_wrapper(trace); step_wrapper.add_alnum("transformation", "trivial_condition_removal"); { Opt_trace_disable_I_S disable_trace_wrapper(trace, !(*cond)->has_subquery()); Opt_trace_array trace_subselect(trace, "subselect_evaluation"); if (remove_eq_conds(thd, *cond, cond, cond_value)) return true; } step_wrapper.add("resulting_condition", *cond); } if (thd->is_error()) return true; return false; } /** Checks if a condition can be evaluated during constant folding. It can be evaluated if it is constant during execution and not expensive to evaluate. If it contains a subquery, it should not be evaluated if the option OPTION_NO_SUBQUERY_DURING_OPTIMIZATION is active. */ static bool can_evaluate_condition(THD *thd, Item *condition) { return condition->const_for_execution() && !condition->is_expensive() && evaluate_during_optimization(condition, thd->lex->current_select()); } /** Calls fold_condition. If that made the condition constant for execution, simplify and fold again. @see fold_condition() for arguments. */ static bool fold_condition_exec(THD *thd, Item *cond, Item **retcond, Item::cond_result *cond_value) { if (fold_condition(thd, cond, retcond, cond_value)) return true; if (*retcond != nullptr && can_evaluate_condition(thd, *retcond)) // simplify further maybe return remove_eq_conds(thd, *retcond, retcond, cond_value); return false; } /** Removes const and eq items. Returns the new item, or nullptr if no condition. @param thd thread handler @param cond the condition to handle @param[out] retcond condition after const removal @param[out] cond_value resulting value of the condition =COND_OK condition must be evaluated (e.g. field = constant) =COND_TRUE always true (e.g. 1 = 1) =COND_FALSE always false (e.g. 1 = 2) @returns false if success, true if error */ bool remove_eq_conds(THD *thd, Item *cond, Item **retcond, Item::cond_result *cond_value) { DBUG_ASSERT(cond->real_item()->is_bool_func()); if (cond->type() == Item::COND_ITEM) { Item_cond *const item_cond = down_cast(cond); const bool and_level = item_cond->functype() == Item_func::COND_AND_FUNC; List_iterator li(*item_cond->argument_list()); bool should_fix_fields = false; *cond_value = Item::COND_UNDEF; Item *item; while ((item = li++)) { Item *new_item; Item::cond_result tmp_cond_value; if (remove_eq_conds(thd, item, &new_item, &tmp_cond_value)) return true; if (new_item == NULL) li.remove(); else if (item != new_item) { (void)li.replace(new_item); should_fix_fields = true; } if (*cond_value == Item::COND_UNDEF) *cond_value = tmp_cond_value; switch (tmp_cond_value) { case Item::COND_OK: // Not true or false if (and_level || *cond_value == Item::COND_FALSE) *cond_value = tmp_cond_value; break; case Item::COND_FALSE: if (and_level) // Always false { *cond_value = tmp_cond_value; *retcond = NULL; return false; } break; case Item::COND_TRUE: if (!and_level) // Always true { *cond_value = tmp_cond_value; *retcond = NULL; return false; } break; case Item::COND_UNDEF: // Impossible DBUG_ASSERT(false); /* purecov: deadcode */ } } if (should_fix_fields) item_cond->update_used_tables(); if (item_cond->argument_list()->elements == 0 || *cond_value != Item::COND_OK) { *retcond = NULL; return false; } if (item_cond->argument_list()->elements == 1) { /* BUG#11765699: We're dealing with an AND or OR item that has only one argument. However, it is not an option to empty the list because: - this function is called for either JOIN::conds or JOIN::having, but these point to the same condition as SELECT_LEX::where and SELECT_LEX::having do. - The return value of remove_eq_conds() is assigned to JOIN::conds and JOIN::having, so emptying the list and returning the only remaining item "replaces" the AND or OR with item for the variables in JOIN. However, the return value is not assigned to the SELECT_LEX counterparts. Thus, if argument_list is emptied, SELECT_LEX forgets the item in argument_list()->head(). item is therefore returned, but argument_list is not emptied. */ item = item_cond->argument_list()->head(); /* Consider reenabling the line below when the optimizer has been split into properly separated phases. item_cond->argument_list()->empty(); */ *retcond = item; return false; } } else if (can_evaluate_condition(thd, cond)) { bool value; if (eval_const_cond(thd, cond, &value)) return true; *cond_value = value ? Item::COND_TRUE : Item::COND_FALSE; *retcond = NULL; return false; } else { // Boolean compare function *cond_value = cond->eq_cmp_result(); if (*cond_value == Item::COND_OK) { return fold_condition_exec(thd, cond, retcond, cond_value); } Item *left_item = down_cast(cond)->arguments()[0]; Item *right_item = down_cast(cond)->arguments()[1]; if (left_item->eq(right_item, 1)) { if (!left_item->maybe_null || down_cast(cond)->functype() == Item_func::EQUAL_FUNC) { *retcond = NULL; return false; // Compare of identical items } } } return fold_condition_exec(thd, cond, retcond, cond_value); } /** Check if GROUP BY/DISTINCT can be optimized away because the set is already known to be distinct. Used in removing the GROUP BY/DISTINCT of the following types of statements: @code SELECT [DISTINCT] ... FROM [GROUP BY ,...] @endcode If (a,b,c is distinct) then ,{whatever} is also distinct This function checks if all the key parts of any of the unique keys of the table are referenced by a list : either the select list through find_field_in_item_list or GROUP BY list through find_field_in_order_list. If the above holds and the key parts cannot contain NULLs then we can safely remove the GROUP BY/DISTINCT, as no result set can be more distinct than an unique key. @param tab The join table to operate on. @param find_func function to iterate over the list and search for a field @param data @retval 1 found @retval 0 not found. @note The function assumes that make_outerjoin_info() has been called in order for the check for outer tables to work. */ static bool list_contains_unique_index(JOIN_TAB *tab, bool (*find_func)(Field *, void *), void *data) { TABLE *table = tab->table(); if (tab->is_inner_table_of_outer_join()) return 0; for (uint keynr = 0; keynr < table->s->keys; keynr++) { if (keynr == table->s->primary_key || (table->key_info[keynr].flags & HA_NOSAME)) { KEY *keyinfo = table->key_info + keynr; KEY_PART_INFO *key_part, *key_part_end; for (key_part = keyinfo->key_part, key_part_end = key_part + keyinfo->user_defined_key_parts; key_part < key_part_end; key_part++) { if (key_part->field->real_maybe_null() || !find_func(key_part->field, data)) break; } if (key_part == key_part_end) return 1; } } return 0; } /** Helper function for list_contains_unique_index. Find a field reference in a list of ORDER structures. Finds a direct reference of the Field in the list. @param field The field to search for. @param data ORDER *.The list to search in @retval 1 found @retval 0 not found. */ static bool find_field_in_order_list(Field *field, void *data) { ORDER *group = (ORDER *)data; bool part_found = 0; for (ORDER *tmp_group = group; tmp_group; tmp_group = tmp_group->next) { Item *item = (*tmp_group->item)->real_item(); if (item->type() == Item::FIELD_ITEM && ((Item_field *)item)->field->eq(field)) { part_found = 1; break; } } return part_found; } /** Helper function for list_contains_unique_index. Find a field reference in a dynamic list of Items. Finds a direct reference of the Field in the list. @param[in] field The field to search for. @param[in] data List *.The list to search in @retval 1 found @retval 0 not found. */ static bool find_field_in_item_list(Field *field, void *data) { List *fields = (List *)data; bool part_found = 0; List_iterator li(*fields); Item *item; while ((item = li++)) { if (item->type() == Item::FIELD_ITEM && ((Item_field *)item)->field->eq(field)) { part_found = 1; break; } } return part_found; } ORDER *create_order_from_distinct(THD *thd, Ref_item_array ref_item_array, ORDER *order_list, List &fields, bool skip_aggregates, bool convert_bit_fields_to_long, bool *all_order_by_fields_used) { List_iterator li(fields); Item *item; ORDER *order, *group, **prev; *all_order_by_fields_used = 1; prev = &group; group = 0; for (order = order_list; order; order = order->next) { if (order->in_field_list) { ORDER *ord = (ORDER *)thd->memdup((char *)order, sizeof(ORDER)); if (!ord) return 0; *prev = ord; prev = &ord->next; (*ord->item)->marker = Item::MARKER_DISTINCT_GROUP; } else *all_order_by_fields_used = 0; } li.rewind(); while ((item = li++)) { if (!item->const_item() && (!skip_aggregates || !item->has_aggregation()) && item->marker != Item::MARKER_DISTINCT_GROUP) { /* Don't put duplicate columns from the SELECT list into the GROUP BY list. */ ORDER *ord_iter; for (ord_iter = group; ord_iter; ord_iter = ord_iter->next) if ((*ord_iter->item)->eq(item, 1)) goto next_item; ORDER *ord = (ORDER *)thd->mem_calloc(sizeof(ORDER)); if (!ord) return 0; if (item->type() == Item::FIELD_ITEM && item->data_type() == MYSQL_TYPE_BIT && convert_bit_fields_to_long) { /* Because HEAP tables can't index BIT fields we need to use an additional hidden field for grouping because later it will be converted to a LONG field. Original field will remain of the BIT type and will be returned to a client. @note setup_ref_array() needs to account for the extra space. */ Item_field *new_item = new Item_field(thd, (Item_field *)item); ord->item = thd->lex->current_select()->add_hidden_item(new_item); } else { /* We have here only field_list (not all_field_list), so we can use simple indexing of ref_item_array (order in the array and in the list are same) */ ord->item = &ref_item_array[0]; } ord->direction = ORDER_ASC; *prev = ord; prev = &ord->next; } next_item: ref_item_array.pop_front(); } *prev = 0; return group; } /** Return table number if there is only one table in sort order and group and order is compatible, else return 0. */ static TABLE *get_sort_by_table(ORDER *a, ORDER *b, TABLE_LIST *tables) { DBUG_TRACE; table_map map = (table_map)0; if (!a) a = b; // Only one need to be given else if (!b) b = a; for (; a && b; a = a->next, b = b->next) { if (!(*a->item)->eq(*b->item, 1)) return 0; map |= a->item[0]->used_tables(); } map &= ~INNER_TABLE_BIT; if (!map || (map & (RAND_TABLE_BIT | OUTER_REF_TABLE_BIT))) return 0; for (; !(map & tables->map()); tables = tables->next_leaf) ; if (map != tables->map()) return 0; // More than one table DBUG_PRINT("exit", ("sort by table: %d", tables->tableno())); return tables->table; } /** Update some values in keyuse for faster choose_table_order() loop. @todo Check if this is the real meaning of ref_table_rows. */ void JOIN::optimize_keyuse() { for (size_t ix = 0; ix < keyuse_array.size(); ++ix) { Key_use *keyuse = &keyuse_array.at(ix); table_map map; /* If we find a ref, assume this table matches a proportional part of this table. For example 100 records matching a table with 5000 records gives 5000/100 = 50 records per key Constant tables are ignored. To avoid bad matches, we don't make ref_table_rows less than 100. */ keyuse->ref_table_rows = ~(ha_rows)0; // If no ref if (keyuse->used_tables & (map = (keyuse->used_tables & ~(const_table_map | OUTER_REF_TABLE_BIT)))) { uint tableno; for (tableno = 0; !(map & 1); map >>= 1, tableno++) { } if (map == 1) // Only one table { TABLE *tmp_table = join_tab[tableno].table(); keyuse->ref_table_rows = max(tmp_table->file->stats.records, 100); } } /* Outer reference (external field) is constant for single executing of subquery */ if (keyuse->used_tables == OUTER_REF_TABLE_BIT) keyuse->ref_table_rows = 1; } } /** Function sets FT hints, initializes FT handlers and checks if FT index can be used as covered. */ bool JOIN::optimize_fts_query() { ASSERT_BEST_REF_IN_JOIN_ORDER(this); DBUG_ASSERT(select_lex->has_ft_funcs()); for (uint i = const_tables; i < tables; i++) { JOIN_TAB *tab = best_ref[i]; if (tab->type() != JT_FT) continue; Item_func_match *ifm; Item_func_match *ft_func = down_cast(tab->position()->key->val); List_iterator li(*(select_lex->ftfunc_list)); while ((ifm = li++)) { if (!(ifm->used_tables() & tab->table_ref->map()) || ifm->master) continue; if (ifm != ft_func) { if (ifm->can_skip_ranking()) ifm->set_hints(this, FT_NO_RANKING, HA_POS_ERROR, false); } } /* Check if internal sorting is needed. FT_SORTED flag is set if no ORDER BY clause or ORDER BY MATCH function is the same as the function that is used for FT index and FT table is the first non-constant table in the JOIN. */ if (i == const_tables && !(ft_func->get_hints()->get_flags() & FT_BOOL) && (!order || ft_func == test_if_ft_index_order(order))) ft_func->set_hints(this, FT_SORTED, m_select_limit, false); /* Check if ranking is not needed. FT_NO_RANKING flag is set if MATCH function is used only in WHERE condition and MATCH function is not part of an expression. */ if (ft_func->can_skip_ranking()) ft_func->set_hints(this, FT_NO_RANKING, !order ? m_select_limit : HA_POS_ERROR, false); } return init_ftfuncs(thd, select_lex); } /** Check if FTS index only access is possible. @param tab pointer to JOIN_TAB structure. @return true if index only access is possible, false otherwise. */ bool JOIN::fts_index_access(JOIN_TAB *tab) { DBUG_ASSERT(tab->type() == JT_FT); TABLE *table = tab->table(); if ((table->file->ha_table_flags() & HA_CAN_FULLTEXT_EXT) == 0) return false; // Optimizations requires extended FTS support by table // engine /* This optimization does not work with filesort nor GROUP BY */ if (grouped || (order && m_ordered_index_usage != ORDERED_INDEX_ORDER_BY)) return false; /* Check whether the FTS result is covering. If only document id and rank is needed, there is no need to access table rows. */ for (uint i = bitmap_get_first_set(table->read_set); i < table->s->fields; i = bitmap_get_next_set(table->read_set, i)) { if (table->field[i] != table->fts_doc_id_field || !tab->ft_func()->docid_in_result()) return false; } return true; } /** For {semijoin,subquery} materialization: calculates various cost information, based on a plan in join->best_positions covering the to-be-materialized query block and only this. @param join JOIN where plan can be found @param sj_nest sj materialization nest (NULL if subquery materialization) @param n_tables number of to-be-materialized tables @param[out] sjm where computed costs will be stored @note that this function modifies join->map2table, which has to be filled correctly later. */ static void calculate_materialization_costs(JOIN *join, TABLE_LIST *sj_nest, uint n_tables, Semijoin_mat_optimize *sjm) { double mat_cost; // Estimated cost of materialization double mat_rowcount; // Estimated row count before duplicate removal double distinct_rowcount; // Estimated rowcount after duplicate removal List *inner_expr_list; if (sj_nest) { /* get_partial_join_cost() assumes a regular join, which is correct when we optimize a sj-materialization nest (always executed as regular join). */ get_partial_join_cost(join, n_tables, &mat_cost, &mat_rowcount); n_tables += join->const_tables; inner_expr_list = &sj_nest->nested_join->sj_inner_exprs; } else { mat_cost = join->best_read; mat_rowcount = static_cast(join->best_rowcount); inner_expr_list = &join->select_lex->item_list; } /* Adjust output cardinality estimates. If the subquery has form ... oe IN (SELECT t1.colX, t2.colY, func(X,Y,Z) ) then the number of distinct output record combinations has an upper bound of product of number of records matching the tables that are used by the SELECT clause. TODO: We can get a more precise estimate if we - use rec_per_key cardinality estimates. For simple cases like "oe IN (SELECT t.key ...)" it is trivial. - Functional dependencies between the tables in the semi-join nest (the payoff is probably less here?) */ { for (uint i = 0; i < n_tables; i++) { JOIN_TAB *const tab = join->best_positions[i].table; join->map2table[tab->table_ref->tableno()] = tab; } List_iterator it(*inner_expr_list); Item *item; table_map map = 0; while ((item = it++)) map |= item->used_tables(); map &= ~PSEUDO_TABLE_BITS; Table_map_iterator tm_it(map); int tableno; double rows = 1.0; while ((tableno = tm_it.next_bit()) != Table_map_iterator::BITMAP_END) rows *= join->map2table[tableno]->table()->quick_condition_rows; distinct_rowcount = min(mat_rowcount, rows); } /* Calculate temporary table parameters and usage costs */ const uint rowlen = get_tmp_table_rec_length(*inner_expr_list); const Cost_model_server *cost_model = join->cost_model(); Cost_model_server::enum_tmptable_type tmp_table_type; if (rowlen * distinct_rowcount < join->thd->variables.max_heap_table_size) tmp_table_type = Cost_model_server::MEMORY_TMPTABLE; else tmp_table_type = Cost_model_server::DISK_TMPTABLE; /* Let materialization cost include the cost to create the temporary table and write the rows into it: */ mat_cost += cost_model->tmptable_create_cost(tmp_table_type); mat_cost += cost_model->tmptable_readwrite_cost(tmp_table_type, mat_rowcount, 0.0); sjm->materialization_cost.reset(); sjm->materialization_cost.add_io(mat_cost); sjm->expected_rowcount = distinct_rowcount; /* Set the cost to do a full scan of the temptable (will need this to consider doing sjm-scan): */ sjm->scan_cost.reset(); if (distinct_rowcount > 0.0) { const double scan_cost = cost_model->tmptable_readwrite_cost( tmp_table_type, 0.0, distinct_rowcount); sjm->scan_cost.add_io(scan_cost); } // The cost to lookup a row in temp. table const double row_cost = cost_model->tmptable_readwrite_cost(tmp_table_type, 0.0, 1.0); sjm->lookup_cost.reset(); sjm->lookup_cost.add_io(row_cost); } /** Decides between EXISTS and materialization; performs last steps to set up the chosen strategy. @returns 'false' if no error @note If UNION this is called on each contained JOIN. */ bool JOIN::decide_subquery_strategy() { DBUG_ASSERT(unit->item); switch (unit->item->substype()) { case Item_subselect::IN_SUBS: case Item_subselect::ALL_SUBS: case Item_subselect::ANY_SUBS: // All of those are children of Item_in_subselect and may use EXISTS break; default: return false; } Item_in_subselect *const in_pred = static_cast(unit->item); Item_exists_subselect::enum_exec_method chosen_method = in_pred->exec_method; // Materialization does not allow UNION so this can't happen: DBUG_ASSERT(chosen_method != Item_exists_subselect::EXEC_MATERIALIZATION); if ((chosen_method == Item_exists_subselect::EXEC_EXISTS_OR_MAT) && compare_costs_of_subquery_strategies(&chosen_method)) return true; switch (chosen_method) { case Item_exists_subselect::EXEC_EXISTS: if (select_lex->m_windows.elements > 0) // grep for WL#10431 { my_error(ER_NOT_SUPPORTED_YET, MYF(0), "the combination of this ALL/ANY/SOME/IN subquery with this" " comparison operator and with contained window functions"); return true; } return in_pred->finalize_exists_transform(thd, select_lex); case Item_exists_subselect::EXEC_MATERIALIZATION: return in_pred->finalize_materialization_transform(thd, this); default: DBUG_ASSERT(false); return true; } } /** Tells what is the cheapest between IN->EXISTS and subquery materialization, in terms of cost, for the subquery's JOIN. Input: - join->{best_positions,best_read,best_rowcount} must contain the execution plan of EXISTS (where 'join' is the subquery's JOIN) - join2->{best_positions,best_read,best_rowcount} must be correctly set (where 'join2' is the parent join, the grandparent join, etc). Output: join->{best_positions,best_read,best_rowcount} contain the cheapest execution plan (where 'join' is the subquery's JOIN). This plan choice has to happen before calling functions which set up execution structures, like JOIN::get_best_combination(). @param[out] method chosen method (EXISTS or materialization) will be put here. @returns false if success */ bool JOIN::compare_costs_of_subquery_strategies( Item_exists_subselect::enum_exec_method *method) { *method = Item_exists_subselect::EXEC_EXISTS; Item_exists_subselect::enum_exec_method allowed_strategies = select_lex->subquery_strategy(thd); /* A non-deterministic subquery should not use materialization, unless forced. For a detailed explanation, see SELECT_LEX::decorrelate_where_cond(). Here, the same logic is applied also for subqueries that are not converted to semi-join. */ if (allowed_strategies == Item_exists_subselect::EXEC_EXISTS_OR_MAT && (unit->uncacheable & UNCACHEABLE_RAND)) allowed_strategies = Item_exists_subselect::EXEC_EXISTS; if (allowed_strategies == Item_exists_subselect::EXEC_EXISTS) return false; DBUG_ASSERT(allowed_strategies == Item_exists_subselect::EXEC_EXISTS_OR_MAT || allowed_strategies == Item_exists_subselect::EXEC_MATERIALIZATION); const JOIN *parent_join = unit->outer_select()->join; if (!parent_join || !parent_join->child_subquery_can_materialize) return false; Item_in_subselect *const in_pred = static_cast(unit->item); /* Testing subquery_allows_etc() at each optimization is necessary as each execution of a prepared statement may use a different type of parameter. */ if (!in_pred->subquery_allows_materialization(thd, select_lex, select_lex->outer_select())) return false; Opt_trace_context *const trace = &thd->opt_trace; Opt_trace_object trace_wrapper(trace); Opt_trace_object trace_subqmat( trace, "execution_plan_for_potential_materialization"); const double saved_best_read = best_read; const ha_rows saved_best_rowcount = best_rowcount; POSITION *const saved_best_pos = best_positions; if (in_pred->in2exists_added_to_where()) { Opt_trace_array trace_subqmat_steps(trace, "steps"); // Up to one extra slot per semi-join nest is needed (if materialized) const uint sj_nests = select_lex->sj_nests.elements; if (!(best_positions = new (thd->mem_root) POSITION[tables + sj_nests])) return true; // Compute plans which do not use outer references DBUG_ASSERT(allow_outer_refs); allow_outer_refs = false; if (optimize_semijoin_nests_for_materialization(this)) return true; if (Optimize_table_order(thd, this, NULL).choose_table_order()) return true; } else { /* If IN->EXISTS didn't add any condition to WHERE (only to HAVING, which can happen if subquery has aggregates) then the plan for materialization will be the same as for EXISTS - don't compute it again. */ trace_subqmat.add("surely_same_plan_as_EXISTS", true) .add_alnum("cause", "EXISTS_did_not_change_WHERE"); } Semijoin_mat_optimize sjm; calculate_materialization_costs(this, NULL, primary_tables, &sjm); /* The number of evaluations of the subquery influences costs, we need to compute it. */ Opt_trace_object trace_subq_mat_decision(trace, "subq_mat_decision"); const double subq_executions = calculate_subquery_executions(in_pred, trace); const double cost_exists = subq_executions * saved_best_read; const double cost_mat_table = sjm.materialization_cost.total_cost(); const double cost_mat = cost_mat_table + subq_executions * sjm.lookup_cost.total_cost(); const bool mat_chosen = (allowed_strategies == Item_exists_subselect::EXEC_EXISTS_OR_MAT) ? (cost_mat < cost_exists) : true; trace_subq_mat_decision .add("cost_to_create_and_fill_materialized_table", cost_mat_table) .add("cost_of_one_EXISTS", saved_best_read) .add("number_of_subquery_evaluations", subq_executions) .add("cost_of_materialization", cost_mat) .add("cost_of_EXISTS", cost_exists) .add("chosen", mat_chosen); if (mat_chosen) { *method = Item_exists_subselect::EXEC_MATERIALIZATION; } else { best_read = saved_best_read; best_rowcount = saved_best_rowcount; best_positions = saved_best_pos; /* Don't restore JOIN::positions or best_ref, they're not used afterwards. best_positions is (like: by get_sj_strategy()). */ } return false; } double calculate_subquery_executions(const Item_subselect *subquery, Opt_trace_context *trace) { Opt_trace_array trace_parents(trace, "parent_fanouts"); double subquery_executions = 1.0; for (;;) { const SELECT_LEX *const parent_select = subquery->unit->outer_select(); const JOIN *const parent_join = parent_select->join; if (parent_join == nullptr) { /* May be single-table UPDATE/DELETE, has no join. @todo we should find how many rows it plans to UPDATE/DELETE, taking inspiration in Explain_table::explain_rows_and_filtered(). This is not a priority as it applies only to UPDATE - child(non-mat-subq) - grandchild(may-be-mat-subq). And it will autosolve the day UPDATE gets a JOIN. */ break; } Opt_trace_object trace_parent(trace); trace_parent.add_select_number(parent_select->select_number); double parent_fanout; if ( // safety, not sure needed parent_join->plan_is_const() || // if subq is in condition on constant table: !parent_join->child_subquery_can_materialize) { parent_fanout = 1.0; trace_parent.add("subq_attached_to_const_table", true); } else { if (subquery->in_cond_of_tab != NO_PLAN_IDX) { /* Subquery is attached to a certain 'pos', pos[-1].prefix_rowcount is the number of times we'll start a loop accessing 'pos'; each such loop will read pos->rows_fetched rows of 'pos', so subquery will be evaluated pos[-1].prefix_rowcount * pos->rows_fetched times. Exceptions: - if 'pos' is first, use 1.0 instead of pos[-1].prefix_rowcount - if 'pos' is first of a sj-materialization nest, same. If in a sj-materialization nest, pos->rows_fetched and pos[-1].prefix_rowcount are of the "nest materialization" plan (copied back in fix_semijoin_strategies()), which is appropriate as it corresponds to evaluations of our subquery. pos->prefix_rowcount is not suitable because if we have: select ... from ot1 where ot1.col in (select it1.col1 from it1 where it1.col2 not in (subq)); and subq does subq-mat, and plan is ot1 - it1+firstmatch(ot1), then: - t1.prefix_rowcount==1 (due to firstmatch) - subq is attached to it1, and is evaluated for each row read from t1, potentially way more than 1. */ const uint idx = subquery->in_cond_of_tab; DBUG_ASSERT((int)idx >= 0 && idx < parent_join->tables); trace_parent.add("subq_attached_to_table", true); QEP_TAB *const parent_tab = &parent_join->qep_tab[idx]; trace_parent.add_utf8_table(parent_tab->table_ref); parent_fanout = parent_tab->position()->rows_fetched; if ((idx > parent_join->const_tables) && !sj_is_materialize_strategy(parent_tab->position()->sj_strategy)) parent_fanout *= parent_tab[-1].position()->prefix_rowcount; } else { /* Subquery is SELECT list, GROUP BY, ORDER BY, HAVING: it is evaluated at the end of the parent join's execution. It can be evaluated once per row-before-grouping: SELECT SUM(t1.col IN (subq)) FROM t1 GROUP BY expr; or once per row-after-grouping: SELECT SUM(t1.col) AS s FROM t1 GROUP BY expr HAVING s IN (subq), SELECT SUM(t1.col) IN (subq) FROM t1 GROUP BY expr It's hard to tell. We simply assume 'once per row-before-grouping'. Another approximation: SELECT ... HAVING x IN (subq) LIMIT 1 best_rowcount=1 due to LIMIT, though HAVING (and thus the subquery) may be evaluated many times before HAVING becomes true and the limit is reached. */ trace_parent.add("subq_attached_to_join_result", true); parent_fanout = static_cast(parent_join->best_rowcount); } } subquery_executions *= parent_fanout; trace_parent.add("fanout", parent_fanout); const bool cacheable = parent_select->is_cacheable(); trace_parent.add("cacheable", cacheable); if (cacheable) { // Parent executed only once break; } /* Parent query is executed once per outer row => go up to find number of outer rows. Example: SELECT ... IN(subq-with-in2exists WHERE ... IN (subq-with-mat)) */ subquery = parent_join->unit->item; if (subquery == nullptr) { // derived table, materialized only once break; } } // for(;;) return subquery_executions; } /** Optimize rollup specification. Allocate objects needed for rollup processing. @returns false if success, true if error. */ bool JOIN::optimize_rollup() { tmp_table_param.allow_group_via_temp_table = false; rollup.state = ROLLUP::STATE_INITED; /* Create pointers to the different sum function groups These are updated by rollup_make_fields() */ tmp_table_param.group_parts = send_group_parts; /* substitute_gc() might substitute an expression in the GROUP BY list with a generated column. In such case the GC is added to the all_fields as a hidden field. In total, all_fields list could be grown by up to send_group_parts columns. Reserve space for them here. */ const uint ref_array_size = all_fields.elements + send_group_parts; Item_null_result **null_items = static_cast( thd->alloc(sizeof(Item *) * send_group_parts)); rollup.null_items = Item_null_array(null_items, send_group_parts); rollup.ref_item_arrays = static_cast( thd->alloc((sizeof(Ref_item_array) + ref_array_size * sizeof(Item *)) * send_group_parts)); rollup.all_fields = static_cast *>( thd->alloc(sizeof(List) * send_group_parts)); rollup.fields_list = static_cast *>( thd->alloc(sizeof(List) * send_group_parts)); if (!null_items || !rollup.ref_item_arrays || !rollup.all_fields || !rollup.fields_list) return true; Item **ref_array = (Item **)(rollup.ref_item_arrays + send_group_parts); /* Prepare space for field list for the different levels These will be filled up in rollup_make_fields() */ ORDER *group = group_list; for (uint i = 0; i < send_group_parts; i++, group = group->next) { rollup.null_items[i] = new (thd->mem_root) Item_null_result( (*group->item)->data_type(), (*group->item)->result_type()); if (rollup.null_items[i] == NULL) return true; /* purecov: inspected */ rollup.fields_list[i].empty(); rollup.all_fields[i].empty(); rollup.ref_item_arrays[i] = Ref_item_array(ref_array, ref_array_size); ref_array += ref_array_size; } for (uint i = 0; i < send_group_parts; i++) { for (uint j = 0; j < fields_list.elements; j++) rollup.fields_list[i].push_back(rollup.null_items[i]); for (uint j = 0; j < all_fields.elements; j++) rollup.all_fields[i].push_back(rollup.null_items[i]); } return false; } /** Refine the best_rowcount estimation based on what happens after tables have been joined: LIMIT and type of result sink. */ void JOIN::refine_best_rowcount() { // If plan is const, 0 or 1 rows should be returned DBUG_ASSERT(!plan_is_const() || best_rowcount <= 1); if (plan_is_const()) return; /* If a derived table, or a member of a UNION which itself forms a derived table: setting estimate to 0 or 1 row would mark the derived table as const. The row count is bumped to the nearest higher value, so that the query block will not be evaluated during optimization. */ if (best_rowcount <= 1 && select_lex->master_unit()->first_select()->linkage == DERIVED_TABLE_TYPE) best_rowcount = PLACEHOLDER_TABLE_ROW_ESTIMATE; /* There will be no more rows than defined in the LIMIT clause. Use it as an estimate. If LIMIT 1 is specified, the query block will be considered "const", with actual row count 0 or 1. */ set_if_smaller(best_rowcount, unit->select_limit_cnt); } List *JOIN::get_current_fields() { DBUG_ASSERT((int)current_ref_item_slice >= 0); if (current_ref_item_slice == REF_SLICE_SAVED_BASE) return fields; return &tmp_fields_list[current_ref_item_slice]; } /** @} (end of group Query_Optimizer) */ /** This function is used to get the key length of Item object on which one tmp field will be created during create_tmp_table. This function references KEY_PART_INFO::init_from_field(). @param item A inner item of outer join @return The length of a item to be as a key of a temp table */ static uint32 get_key_length_tmp_table(Item *item) { uint32 len = 0; item = item->real_item(); if (item->type() == Item::FIELD_ITEM) len = ((Item_field *)item)->field->key_length(); else len = item->max_length; if (item->maybe_null) len += HA_KEY_NULL_LENGTH; // references KEY_PART_INFO::init_from_field() enum_field_types type = item->data_type(); if (type == MYSQL_TYPE_BLOB || type == MYSQL_TYPE_VARCHAR || type == MYSQL_TYPE_GEOMETRY) len += HA_KEY_BLOB_LENGTH; return len; } bool evaluate_during_optimization(const Item *item, const SELECT_LEX *select) { /* Should only be called on items that are const_for_execution(), as those items are the only ones that are allowed to be evaluated during optimization in the first place. Additionally, allow items that only access tables in JOIN::const_table_map. This should not be necessary, but the const_for_execution() property is not always updated correctly by update_used_tables() for certain subqueries. */ DBUG_ASSERT(item->const_for_execution() || (item->used_tables() & ~select->join->const_table_map) == 0); return !item->has_subquery() || (select->active_options() & OPTION_NO_SUBQUERY_DURING_OPTIMIZATION) == 0; }