[4.2] Smart Search: Prevent "#__finder_tokens full" error

[joomlapl-bot]

PR w związku ze zmianą oryginału https://github.com/joomla/joomla-cms/pull/36749 Poniżej zmiany w oryginale:

Click to expand the diff!

```diff diff --git a/administrator/components/com_finder/config.xml b/administrator/components/com_finder/config.xml index fb25b31de20a..261b7d7d6049 100644 --- a/administrator/components/com_finder/config.xml +++ b/administrator/components/com_finder/config.xml @@ -308,14 +308,6 @@ J300 - - force = false; // Load the default configuration options. $data->options = ComponentHelper::getParams('com_finder'); + $db = Factory::getDbo(); + + if ($db->getServerType() == 'mysql') { + /** + * Try to calculate the heapsize for the memory table for indexing. If this fails, + * we fall back on a reasonable small size. We want to prevent the system to fail + * and block saving content. + */ + try { + $db->setQuery('SHOW VARIABLES LIKE ' . $db->quote('max_heap_table_size')); + $heapsize = $db->loadObject(); + + /** + * In tests, the size of a row seems to have been around 720 bytes. + * We take 800 to be on the safe side. + */ + $memory_table_limit = (int) ($heapsize->Value / 800); + $data->options->set('memory_table_limit', $memory_table_limit); + } catch (Exception $e) { + // Something failed. We fall back to a reasonable guess. + $data->options->set('memory_table_limit', 7500); + } + } else { + // We are running on PostgreSQL and don't have this issue, so we set a rather high number. + $data->options->set('memory_table_limit', 50000); + } // Setup the weight lookup information. $data->weights = array( @@ -384,10 +411,10 @@ public function index($item, $format = 'html') } // Tokenize a string of content and add it to the database. - $count += $this->tokenizeToDb($ip, $group, $item->language, $format); + $count += $this->tokenizeToDb($ip, $group, $item->language, $format, $count); // Check if we're approaching the memory limit of the token table. - if ($count > static::$state->options->get('memory_table_limit', 30000)) { + if ($count > static::$state->options->get('memory_table_limit', 7500)) { $this->toggleTables(false); } } @@ -404,7 +431,7 @@ public function index($item, $format = 'html') } // Tokenize a string of content and add it to the database. - $count += $this->tokenizeToDb($item->$property, $group, $item->language, $format); + $count += $this->tokenizeToDb($item->$property, $group, $item->language, $format, $count); // Check if we're approaching the memory limit of the token table. if ($count > static::$state->options->get('memory_table_limit', 30000)) { @@ -759,8 +786,8 @@ protected static function getSignature($item) // Get the relevant configuration variables. $config = array( $state->weights, - $state->options->get('stem', 1), - $state->options->get('stemmer', 'porter_en') + $state->options->get('tuplecount', 1), + $state->options->get('language_default', '') ); return md5(serialize(array($item, $config))); @@ -774,14 +801,14 @@ protected static function getSignature($item) * @param integer $context The context of the input. See context constants. * @param string $lang The language of the input. * @param string $format The format of the input. + * @param integer $count Number of words indexed so far. * * @return integer The number of tokens extracted from the input. * * @since 2.5 */ - protected function tokenizeToDb($input, $context, $lang, $format) + protected function tokenizeToDb($input, $context, $lang, $format, $count = 0) { - $count = 0; $buffer = null; if (empty($input)) { @@ -850,6 +877,14 @@ protected function tokenizeToDb($input, $context, $lang, $format) */ private function tokenizeToDbShort($input, $context, $lang, $format, $count) { + static $filterCommon, $filterNumeric; + + if (is_null($filterCommon)) { + $params = ComponentHelper::getParams('com_finder'); + $filterCommon = $params->get('filter_commonwords', false); + $filterNumeric = $params->get('filter_numerics', false); + } + // Parse the input. $input = Helper::parse($input, $format); @@ -865,49 +900,8 @@ private function tokenizeToDbShort($input, $context, $lang, $format, $count) return $count; } - // Add the tokens to the database. - $count += $this->addTokensToDb($tokens, $context); - - // Check if we're approaching the memory limit of the token table. - if ($count > static::$state->options->get('memory_table_limit', 10000)) { - $this->toggleTables(false); - } - - return $count; - } - - /** - * Method to add a set of tokens to the database. - * - * @param Token[]|Token $tokens An array or single Token object. - * @param mixed $context The context of the tokens. See context constants. [optional] - * - * @return integer The number of tokens inserted into the database. - * - * @since 2.5 - * @throws Exception on database error. - */ - protected function addTokensToDb($tokens, $context = '') - { - static $filterCommon, $filterNumeric; - - if (is_null($filterCommon)) { - $params = ComponentHelper::getParams('com_finder'); - $filterCommon = $params->get('filter_commonwords', false); - $filterNumeric = $params->get('filter_numerics', false); - } - - // Get the database object. - $db = $this->db; - $query = clone $this->addTokensToDbQueryTemplate; - // Check if a single FinderIndexerToken object was given and make it to be an array of FinderIndexerToken objects - $tokens = is_array($tokens) ? $tokens : array($tokens); - - // Count the number of token values. - $values = 0; - // Break into chunks of no more than 128 items $chunks = array_chunk($tokens, 128); @@ -929,29 +923,29 @@ protected function addTokensToDb($tokens, $context = '') } $query->values( - $db->quote($token->term) . ', ' - . $db->quote($token->stem) . ', ' + $this->db->quote($token->term) . ', ' + . $this->db->quote($token->stem) . ', ' . (int) $token->common . ', ' . (int) $token->phrase . ', ' - . $db->quote($token->weight) . ', ' + . $this->db->quote($token->weight) . ', ' . (int) $context . ', ' - . $db->quote($token->language) + . $this->db->quote($token->language) ); - ++$values; - } - - // Only execute the query if there are tokens to insert - if ($query->values !== null) { - $db->setQuery($query)->execute(); + $count++; } // Check if we're approaching the memory limit of the token table. - if ($values > static::$state->options->get('memory_table_limit', 10000)) { + if ($count > static::$state->options->get('memory_table_limit', 7500)) { $this->toggleTables(false); } + + // Only execute the query if there are tokens to insert + if ($query->values !== null) { + $this->db->setQuery($query)->execute(); + } } - return $values; + return $count; } /** diff --git a/administrator/language/en-GB/com_finder.ini b/administrator/language/en-GB/com_finder.ini index 220677e1efbf..950b8b80ba32 100644 --- a/administrator/language/en-GB/com_finder.ini +++ b/administrator/language/en-GB/com_finder.ini @@ -24,7 +24,6 @@ COM_FINDER_CONFIG_LANGUAGE_DEFAULT_DESC="Set the language to be used for non-mul COM_FINDER_CONFIG_LANGUAGE_DEFAULT_LABEL="Default Language" COM_FINDER_CONFIG_LANGUAGE_DEFAULT_NONE="None" COM_FINDER_CONFIG_LINKED_IMAGE_LABEL="Linked Image" -COM_FINDER_CONFIG_MEMORY_TABLE_LIMIT_LABEL="Memory Table Limit" COM_FINDER_CONFIG_META_MULTIPLIER_LABEL="Metadata Weight Multiplier" COM_FINDER_CONFIG_MISC_MULTIPLIER_LABEL="Misc. Text Weight Multiplier" COM_FINDER_CONFIG_OPENSEARCH_LABEL="Enable OpenSearch" diff --git a/installation/sql/mysql/base.sql b/installation/sql/mysql/base.sql index 81e2198d2e6a..dc7e9a94bfc2 100644 --- a/installation/sql/mysql/base.sql +++ b/installation/sql/mysql/base.sql @@ -170,7 +170,7 @@ INSERT INTO `#__extensions` (`package_id`, `name`, `type`, `element`, `folder`, (0, 'com_config', 'component', 'com_config', '', 1, 1, 0, 1, 1, '', '{"filters":{"1":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"9":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"6":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"7":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"2":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"3":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"4":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"5":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"8":{"filter_type":"NONE","filter_tags":"","filter_attributes":""}}}', ''), (0, 'com_redirect', 'component', 'com_redirect', '', 1, 1, 0, 0, 1, '', '', ''), (0, 'com_users', 'component', 'com_users', '', 1, 1, 0, 1, 1, '', '{"allowUserRegistration":"0","new_usertype":"2","guest_usergroup":"9","sendpassword":"0","useractivation":"2","mail_to_admin":"1","captcha":"","frontend_userparams":"1","site_language":"0","change_login_name":"0","reset_count":"10","reset_time":"1","minimum_length":"12","minimum_integers":"0","minimum_symbols":"0","minimum_uppercase":"0","save_history":"1","history_limit":5,"mailSubjectPrefix":"","mailBodySuffix":""}', ''), -(0, 'com_finder', 'component', 'com_finder', '', 1, 1, 0, 0, 1, '', '{"enabled":"0","show_description":"1","description_length":255,"allow_empty_query":"0","show_url":"1","show_autosuggest":"1","show_suggested_query":"1","show_explained_query":"1","show_advanced":"1","show_advanced_tips":"1","expand_advanced":"0","show_date_filters":"0","sort_order":"relevance","sort_direction":"desc","highlight_terms":"1","opensearch_name":"","opensearch_description":"","batch_size":"50","memory_table_limit":30000,"title_multiplier":"1.7","text_multiplier":"0.7","meta_multiplier":"1.2","path_multiplier":"2.0","misc_multiplier":"0.3","stem":"1","stemmer":"snowball","enable_logging":"0"}', ''), +(0, 'com_finder', 'component', 'com_finder', '', 1, 1, 0, 0, 1, '', '{"enabled":"0","show_description":"1","description_length":255,"allow_empty_query":"0","show_url":"1","show_autosuggest":"1","show_suggested_query":"1","show_explained_query":"1","show_advanced":"1","show_advanced_tips":"1","expand_advanced":"0","show_date_filters":"0","sort_order":"relevance","sort_direction":"desc","highlight_terms":"1","opensearch_name":"","opensearch_description":"","batch_size":"50","title_multiplier":"1.7","text_multiplier":"0.7","meta_multiplier":"1.2","path_multiplier":"2.0","misc_multiplier":"0.3","stem":"1","stemmer":"snowball","enable_logging":"0"}', ''), (0, 'com_joomlaupdate', 'component', 'com_joomlaupdate', '', 1, 1, 0, 1, 1, '', '{"updatesource":"default","customurl":""}', ''), (0, 'com_tags', 'component', 'com_tags', '', 1, 1, 1, 0, 1, '', '{"tag_layout":"_:default","save_history":"1","history_limit":5,"show_tag_title":"0","tag_list_show_tag_image":"0","tag_list_show_tag_description":"0","tag_list_image":"","tag_list_orderby":"title","tag_list_orderby_direction":"ASC","show_headings":"0","tag_list_show_date":"0","tag_list_show_item_image":"0","tag_list_show_item_description":"0","tag_list_item_maximum_characters":0,"return_any_or_all":"1","include_children":"0","maximum":200,"tag_list_language_filter":"all","tags_layout":"_:default","all_tags_orderby":"title","all_tags_orderby_direction":"ASC","all_tags_show_tag_image":"0","all_tags_show_tag_description":"0","all_tags_tag_maximum_characters":20,"all_tags_show_tag_hits":"0","filter_field":"1","show_pagination_limit":"1","show_pagination":"2","show_pagination_results":"1","tag_field_ajax_mode":"1","show_feed_link":"1"}', ''), (0, 'com_contenthistory', 'component', 'com_contenthistory', '', 1, 1, 1, 0, 1, '', '', ''), diff --git a/installation/sql/postgresql/base.sql b/installation/sql/postgresql/base.sql index cb53de8fc109..beebddfbb49f 100644 --- a/installation/sql/postgresql/base.sql +++ b/installation/sql/postgresql/base.sql @@ -176,7 +176,7 @@ INSERT INTO "#__extensions" ("package_id", "name", "type", "element", "folder", (0, 'com_config', 'component', 'com_config', '', 1, 1, 0, 1, 1, '', '{"filters":{"1":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"9":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"6":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"7":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"2":{"filter_type":"NH","filter_tags":"","filter_attributes":""},"3":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"4":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"5":{"filter_type":"BL","filter_tags":"","filter_attributes":""},"8":{"filter_type":"NONE","filter_tags":"","filter_attributes":""}}}', '', 0, 0), (0, 'com_redirect', 'component', 'com_redirect', '', 1, 1, 0, 0, 1, '', '', '', 0, 0), (0, 'com_users', 'component', 'com_users', '', 1, 1, 0, 1, 1, '', '{"allowUserRegistration":"0","new_usertype":"2","guest_usergroup":"9","sendpassword":"0","useractivation":"2","mail_to_admin":"1","captcha":"","frontend_userparams":"1","site_language":"0","change_login_name":"0","reset_count":"10","reset_time":"1","minimum_length":"12","minimum_integers":"0","minimum_symbols":"0","minimum_uppercase":"0","save_history":"1","history_limit":5,"mailSubjectPrefix":"","mailBodySuffix":""}', '', 0, 0), -(0, 'com_finder', 'component', 'com_finder', '', 1, 1, 0, 0, 1, '', '{"enabled":"0","show_description":"1","description_length":255,"allow_empty_query":"0","show_url":"1","show_autosuggest":"1","show_suggested_query":"1","show_explained_query":"1","show_advanced":"1","show_advanced_tips":"1","expand_advanced":"0","show_date_filters":"0","sort_order":"relevance","sort_direction":"desc","highlight_terms":"1","opensearch_name":"","opensearch_description":"","batch_size":"50","memory_table_limit":30000,"title_multiplier":"1.7","text_multiplier":"0.7","meta_multiplier":"1.2","path_multiplier":"2.0","misc_multiplier":"0.3","stem":"1","stemmer":"snowball","enable_logging":"0"}', '', 0, 0), +(0, 'com_finder', 'component', 'com_finder', '', 1, 1, 0, 0, 1, '', '{"enabled":"0","show_description":"1","description_length":255,"allow_empty_query":"0","show_url":"1","show_autosuggest":"1","show_suggested_query":"1","show_explained_query":"1","show_advanced":"1","show_advanced_tips":"1","expand_advanced":"0","show_date_filters":"0","sort_order":"relevance","sort_direction":"desc","highlight_terms":"1","opensearch_name":"","opensearch_description":"","batch_size":"50","title_multiplier":"1.7","text_multiplier":"0.7","meta_multiplier":"1.2","path_multiplier":"2.0","misc_multiplier":"0.3","stem":"1","stemmer":"snowball","enable_logging":"0"}', '', 0, 0), (0, 'com_joomlaupdate', 'component', 'com_joomlaupdate', '', 1, 1, 0, 1, 1, '', '{"updatesource":"default","customurl":""}', '', 0, 0), (0, 'com_tags', 'component', 'com_tags', '', 1, 1, 1, 0, 1, '', '{"tag_layout":"_:default","save_history":"1","history_limit":5,"show_tag_title":"0","tag_list_show_tag_image":"0","tag_list_show_tag_description":"0","tag_list_image":"","tag_list_orderby":"title","tag_list_orderby_direction":"ASC","show_headings":"0","tag_list_show_date":"0","tag_list_show_item_image":"0","tag_list_show_item_description":"0","tag_list_item_maximum_characters":0,"return_any_or_all":"1","include_children":"0","maximum":200,"tag_list_language_filter":"all","tags_layout":"_:default","all_tags_orderby":"title","all_tags_orderby_direction":"ASC","all_tags_show_tag_image":"0","all_tags_show_tag_description":"0","all_tags_tag_maximum_characters":20,"all_tags_show_tag_hits":"0","filter_field":"1","show_pagination_limit":"1","show_pagination":"2","show_pagination_results":"1","tag_field_ajax_mode":"1","show_feed_link":"1"}', '', 0, 0), (0, 'com_contenthistory', 'component', 'com_contenthistory', '', 1, 1, 1, 0, 1, '', '', '', 0, 0), ```