['weight' => 10, 'type' => 'text', 'deps' => ['title']], 'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'], 'title.keyword' => ['weight' => 0, 'type' => 'text_keyword'], 'annotation' => ['weight' => 4, 'type' => 'text', 'deps' => ['short_descr']], 'description' => ['weight' => 2, 'type' => 'text', 'deps' => ['long_descr']], 'description2' => ['weight' => 2, 'type' => 'text', 'deps' => ['parameters']], 'description_plus' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Blocek]], 'variations' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Variations]], 'sections' => ['weight' => 4, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Sections]], 'parent_sections' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Sections]], 'code' => ['weight' => 7, 'type' => 'keyword', 'deps' => [ElasticUpdateGroup::Code]], 'code.no_spaces' => ['weight' => 7, 'type' => 'no_spaces'], 'ean' => ['weight' => 5, 'type' => 'keyword', 'deps' => [ElasticUpdateGroup::Ean]], 'ean.no_spaces' => ['weight' => 5, 'type' => 'no_spaces'], 'producer' => ['weight' => 4, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Producer]], 'producer.no_spaces' => ['weight' => 4, 'type' => 'no_spaces'], 'parameters' => ['weight' => 1, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Parameters]], 'price' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]], 'delivery' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::DeliveryTime]], 'sold' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => ['pieces_sold']], 'weight' => ['weight' => 0, 'type' => 'sort_float', 'deps' => ['weight', 'pieces_sold', ElasticUpdateGroup::Position, ElasticUpdateGroup::DeliveryTime, ElasticUpdateGroup::InStore]], 'position' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::Position]], 'sort_position_in_sections' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::SectionPosition]], 'sort_position_in_producers' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::SectionPosition]], 'filter_section' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Sections]], 'filter_section_recursive' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Sections]], 'filter_producer' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Producer]], 'filter_in_sale' => ['weight' => 0, 'type' => 'filter_boolean', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]], 'filter_show_in_search' => ['weight' => 0, 'type' => 'filter_boolean', 'deps' => ['show_in_search']], 'filter_labels' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Labels]], 'filter_campaigns' => ['weight' => 0, 'type' => 'keyword', 'deps' => ['campaign']], // will be removed, use variations_data 'filter_variations' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Variations]], 'filter_parameters_'.ParameterType::LIST => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]], 'filter_convertors_parameters' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]], 'filter_parameters_'.ParameterType::STRING => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]], 'filter_parameters_'.ParameterType::FLOAT => ['weight' => 0, 'type' => 'nested', 'properties' => [ 'id' => ['type' => 'keyword'], 'value' => ['type' => 'float'], ], 'deps' => [ElasticUpdateGroup::Parameters]], 'filter_stores_in_store' => ['weight' => 0, 'type' => 'nested', 'properties' => [ 'id' => ['type' => 'keyword'], 'in_store' => ['type' => 'integer'], ], 'deps' => [ElasticUpdateGroup::StoresInStore]], // will be removed, use variations_data 'variations_in_store' => ['weight' => 0, 'type' => 'nested', 'properties' => [ 'id' => ['type' => 'keyword'], 'id_label' => ['type' => 'keyword'], 'in_store' => ['type' => 'integer'], ], 'deps' => [ElasticUpdateGroup::InStore]], 'variations_data' => ['weight' => 0, 'type' => 'nested', 'properties' => [ 'id' => ['type' => 'keyword'], 'combination' => ['type' => 'flattened'], 'in_store' => ['type' => 'integer'], ], 'deps' => [ElasticUpdateGroup::InStore, ElasticUpdateGroup::Variations]], 'in_store' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::InStore]], 'sort_position' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::Position]], 'sort_section_positions' => ['weight' => 0, 'type' => 'filter_positions', 'deps' => [ElasticUpdateGroup::SectionPosition]], 'sort_discount' => ['weight' => 0, 'type' => 'sort_float', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]], 'date_added' => ['weight' => 0, 'type' => 'sort_timestamp'], 'date_updated' => ['weight' => 0, 'type' => 'sort_timestamp', 'deps' => ['updated']], 'date_stock_in' => ['weight' => 0, 'type' => 'sort_timestamp'], 'store_value' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::InStore, 'price_buy']], 'delivery_time_index' => ['deps' => [ElasticUpdateGroup::DeliveryTime]], 'delivery_time_text' => ['deps' => [ElasticUpdateGroup::DeliveryTime]], 'visible' => ['deps' => ['figure']], 'id_photo' => ['deps' => [ElasticUpdateGroup::Photo]], 'price_original' => ['deps' => [ElasticUpdateGroup::Price, 'price_orig']], 'discount' => ['deps' => [ElasticUpdateGroup::Discount]], ]; protected array $fields_sections = [ 'title' => ['weight' => 10, 'type' => 'text'], 'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'], 'description' => ['weight' => 2, 'type' => 'text'], 'path' => ['weight' => 1, 'type' => 'text'], ]; protected array $fields_producers = [ 'title' => ['weight' => 10, 'type' => 'text'], 'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'], 'description' => ['weight' => 2, 'type' => 'text'], ]; protected array $fields_articles = [ 'title' => ['weight' => 10, 'type' => 'text'], 'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'], 'keywords' => ['weight' => 8, 'type' => 'text'], 'lead_in' => ['weight' => 7, 'type' => 'text'], 'section' => ['weight' => 6, 'type' => 'text'], 'tags' => ['weight' => 6, 'type' => 'text'], 'content' => ['weight' => 2, 'type' => 'text'], ]; protected array $fields_pages = GenericObjectsIndex::FIELDS; /** * @var \Doctrine\DBAL\Connection */ private $connection; protected $settings; /** @var LanguageContext */ private $languageContext; private $sentryLogger; /** @var ArticleList */ private $articleList; /** @var SectionTree */ private $sectionTree; protected $filterParams; protected $curlTimeout = 60; public array $multiSearchResultTotals = []; protected GenericObjectsIndex $genericObjectsIndex; public ?ProductIndexExtension $productIndexExtension = null; #[Required] public ElasticDataFetchUtil $elasticDataFetchUtil; public function __construct(LanguageContext $languageContext, SentryLogger $sentryLogger) { if (findModule(\Modules::PRODUCTS_SECTIONS, \Modules::SUB_ELASTICSEARCH)) { $this->productIndexExtension = ServiceContainer::getService(ProductIndexExtension::class); } global $cfg; $default = [ 'index' => $cfg['Connection']['database'], 'rows' => 0, 'synonyms' => [''], ]; $this->languageContext = $languageContext; $this->sentryLogger = $sentryLogger; if (array_key_exists('fulltext_search', $cfg['Modules'])) { $this->settings = array_merge($default, (array) $cfg['Modules']['fulltext_search']); } else { $this->settings = $default; } } /** @var bool Indicates which index to use */ private bool $_useNextIndex = false; public const INDEX_ALIAS_CURRENT = 'current'; public const INDEX_ALIAS_NEXT = 'next'; /** * Creates a new index of specified type and tries to import data via callback. * If import fails the newly created index won't be used. * * @param callable $func callback, if this fails the new index won't be * aliased as current and therefore won't be used * * @throws FulltextException */ protected function withNewIndex(string $indexType, callable $func): bool { $this->_useNextIndex = true; $newIndexName = null; try { $newIndexName = $this->createNewIndex($indexType); $func(); } catch (\Exception $e) { if ($e instanceof FulltextException) { throw $e; } throw new FulltextException($e->getMessage(), [], $e); } finally { $this->_useNextIndex = false; } $this->aliasOneAsCurrent($indexType, $newIndexName); return true; } /** * Creates a new index with specified alias. * * @return string Index name * * @throws FulltextException */ protected function createNewIndex(string $type, bool $useAlias = true): string { $lang = ''; $opts = $this->returnIndexSettings($type); $serverUrl = $this->getServerUrl(); // Temporary fix: Delete index created using alias name $alias = $this->getIndex($type, false); $this->curlInitSession("{$serverUrl}/{$alias}", 'DELETE', ''); if (findModule(\Modules::TRANSLATIONS)) { $lang = $this->languageContext->getActiveId(); } $indexName = $this->settings['index'].'_'.time()."_{$lang}.{$type}"; if ($useAlias) { $opts['aliases'] = [ $this->getIndex($type) => (object) null, ]; } $url = $this->getServerUrl().'/'.$indexName; $this->curlInitSession($url, 'DELETE', ''); $res = $this->curlInitSession($url, 'PUT', json_encode($opts)); if (array_key_exists('error', $res) || !$res['acknowledged']) { throw new FulltextException('Failed to create index.', ['url' => $url, 'data' => $opts, 'result' => $res]); } return $indexName; } /** * Deletes all indices, that are not aliased as current; * removes next aliases. * * @throws FulltextException */ private function clearIndices(): void { $sUrl = $this->getServerUrl(); $prefix = $this->settings['index']; // remove next aliases $req = [ 'actions' => [ 'remove' => [ 'index' => "{$prefix}_*", 'alias' => $this->getIndexPrefix(true, false).'*', ], ], ]; $this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($req)); $allIndices = $this->getAllIndices(); if (empty($allIndices)) { return; } $currentIndices = $this->getCurrentIndices(); foreach (array_diff($allIndices, $currentIndices) as $i) { $this->curlInitSession("{$sUrl}/{$i}", 'DELETE', ''); } } public function getAllIndices() { $sUrl = $this->getServerUrl(); $prefix = $this->settings['index']; $allIndices = $this->curlInitSession("{$sUrl}/_cat/indices/{$prefix}_*?format=JSON", 'GET', ''); return empty($allIndices) ? [] : array_column($allIndices, 'index'); } public function getCurrentIndices() { $sUrl = $this->getServerUrl(); $prefix = $this->settings['index']; $curQuery = $this->getIndexPrefix(false, false).'*'; $currentIndices = $this->curlInitSession("{$sUrl}/{$prefix}_*/_alias/{$curQuery}?format=JSON", 'GET', ''); return array_keys($currentIndices); } /** * Removes current alias from old index and gives it to new one. * * @param string $type index type * @param string $indexName index, that will receive the current alias * * @throws FulltextException */ private function aliasOneAsCurrent(string $type, string $indexName): void { $prefix = $this->settings['index']; $alias = $this->getIndex($type, false); $sUrl = $this->getServerUrl(); // check if the new index exists before current alias is deleted $exists = $this->curlInitSession("{$sUrl}/{$indexName}", 'GET', ''); if (!empty($exists['error'])) { throw new FulltextException('Failed to alias index as current - the index does not exist', ['url' => "{$sUrl}/{$indexName}", 'data' => $indexName, 'result' => $exists]); } $removeReq = [ 'actions' => [ 'remove' => [ 'index' => "{$prefix}*", 'alias' => $alias, ], ], ]; // allowed to fail, when alias does not exist $this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($removeReq)); $addReq = [ 'actions' => [ 'add' => [ 'index' => $indexName, 'alias' => $alias, ], ], ]; $res = $this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($addReq)); if (!empty($res['error']) || !$res['acknowledged']) { throw new FulltextException('Failed to assign index alias.', ['url' => "{$sUrl}/_aliases", 'data' => $addReq, 'result' => $res]); } } /** * @required */ public function setSectionTree(SectionTree $sectionTree): void { $this->sectionTree = $sectionTree; } /** * @required */ public function setArticleList(ArticleList $articleList): void { $this->articleList = $articleList; } public function setCurlTimeout($timeout): self { $this->curlTimeout = $timeout; return $this; } public function getIndexTypes(): array { return static::INDEX_TYPES; } protected function getIndexMapping(string $type): array { $mapping = []; foreach ($this->getSearchFields($type) as $key => $options) { switch ($options['type']) { case 'text': $mapping[$key] = [ 'type' => 'text', 'analyzer' => self::INDEX_ANALYZER_NAME, 'search_analyzer' => self::SEARCH_ANALYZER_NAME, ]; break; case 'keyword': $mapping[$key] = [ 'type' => 'keyword', ]; break; case 'text_keyword': $key = str_replace('.keyword', '', $key); if (!isset($mapping[$key]['fields'])) { $mapping[$key]['fields'] = []; } $mapping[str_replace('.keyword', '', $key)]['fields']['keyword'] = [ 'type' => 'keyword', ]; break; case 'no_spaces': $mapping[str_replace('.no_spaces', '', $key)]['fields'] = [ 'no_spaces' => [ 'type' => 'text', 'analyzer' => 'ngram_analyze', 'search_analyzer' => 'ngram_search', ], ]; break; case 'integer': case 'float': $mapping[$key] = [ 'type' => $options['type'], ]; break; case 'sort_integer': $mapping[$key] = [ 'type' => 'integer', 'index' => false, ]; break; case 'sort_float': $mapping[$key] = [ 'type' => 'float', 'index' => false, ]; break; case 'sort_timestamp': $mapping[$key] = [ 'type' => 'date', 'format' => 'epoch_second', 'index' => false, ]; break; case 'filter_integer': $mapping[$key] = [ 'type' => 'integer', ]; break; case 'filter_boolean': $mapping[$key] = [ 'type' => 'boolean', ]; break; case 'filter_positions': $mapping[$key] = [ 'type' => 'rank_features', ]; break; case 'filter_nested': $mapping[$key] = [ 'type' => 'flattened', ]; break; case 'nested': $mapping[$key] = [ 'type' => 'nested', ]; if ($options['properties']) { $mapping[$key]['properties'] = $options['properties']; } break; default: throw new \Exception('Unknown field type: '.$options['type']); } } return $mapping; } public function returnIndexSettings(string $type): array { $mappingSettings = $this->getIndexMapping($type); $this->loadSynonyms(); $settings = [ 'settings' => [ 'analysis' => [ 'analyzer' => [ // Fulltext analyze self::INDEX_ANALYZER_NAME => [ 'tokenizer' => 'standard', 'filter' => [ 'custom_stemmer', 'icu_folding', 'shingle', ], 'char_filter' => [ 'icu_normalizer', ], ], // Fulltext search self::SEARCH_ANALYZER_NAME => [ 'tokenizer' => 'standard', 'filter' => [ 'synonymsFilter', 'custom_stemmer', 'icu_folding', 'remove_duplicities', ], 'char_filter' => [ 'icu_normalizer', ], ], // Without spaces - analyze // Když analyzuju, chci odstranit i mezery, abych dostal jeden velký keyword 'ngram_analyze' => [ 'tokenizer' => 'ngram_short', 'filter' => ['icu_folding'], 'char_filter' => ['icu_normalizer', 'dash_space_filter'], ], // Without spaces - search - když hledám, chci zachovat mezery, abych měl víc tokenů 'ngram_search' => [ 'tokenizer' => 'ngram_short', 'filter' => ['icu_folding'], 'char_filter' => ['icu_normalizer', 'dash_filter'], ], ], 'char_filter' => [ 'dash_space_filter' => [ 'type' => 'pattern_replace', 'pattern' => '(\-|\040|\056|\052|\057)', 'replacement' => '', ], 'dash_filter' => [ 'type' => 'pattern_replace', 'pattern' => '(\-|\056|\052|\057)', 'replacement' => '', ], ], 'filter' => array_merge( $this->getHunspellFilter(), [ 'remove_duplicities' => [ 'type' => 'unique', 'only_on_same_position' => true, ], 'synonymsFilter' => [ 'type' => 'synonym', 'synonyms' => $this->getFormattedSynonyms($this->settings['synonyms']), 'tokenizer' => 'standard', 'ignore_case' => true, 'expand' => true, ], 'shingle' => [ 'type' => 'shingle', 'min_shingle_size' => 2, 'max_shingle_size' => 3, ], ]), 'tokenizer' => [ 'ngram_short' => [ 'type' => 'ngram', 'min_gram' => '2', 'max_gram' => '5', 'token_chars' => [ 'letter', 'digit', ], ], ], ], 'max_ngram_diff' => 10, ], 'mappings' => [ 'properties' => $mappingSettings, ], ]; return $settings; } private function updateIndexSettings(string $type): void { $url = $this->getServerUrl().'/'.$this->getIndex($type); if ($this->processResult($url.'/_settings', '', 'GET', 'indexExists')) { $this->curlInitSession($url.'/_close', 'POST', ''); $this->curlInitSession($url.'/_settings', 'PUT', json_encode($this->returnIndexSettings($type))); $this->curlInitSession($url.'/_open', 'POST', ''); } } public function updateSynonyms(?array $synonyms, bool $merge = false): void { if (!$synonyms) { $synonyms = []; } $currentSynonyms = $this->loadSynonyms(); if ($merge) { $synonyms = array_merge($currentSynonyms, $synonyms); } $this->saveSynonyms($synonyms); foreach ($this->getIndexTypes() as $type) { $this->updateIndexSettings($type); } } public function saveSynonyms(array $synonyms): void { $dbcfg = \Settings::getDefault(); $dbcfg->saveValue('fulltext_synonyms', $synonyms, false); } public function loadSynonyms(): array { $dbcfg = \Settings::getDefault(); $synonyms = $dbcfg->loadValue('fulltext_synonyms'); $this->settings['synonyms'] = $synonyms ?? []; return $this->settings['synonyms']; } public function loadSynonymsFromIndex(): ?array { $url = $this->getServerUrl().'/'.$this->getIndex(self::INDEX_PRODUCTS).'/_settings'; $url = str_replace('elasticsearch.wpj.cz:80', 'kupshop:kupshop@kibana.wpj.cz:9200', $url); $result = $this->curlInitSession($url, 'GET', ''); $synonyms = reset($result)['settings']['index']['analysis']['filter']['synonymsFilter']['synonyms'] ?? null; if (isset($synonyms)) { $result = array_map(function ($val) { if (empty($val[0])) { return false; } $val = explode('=>', $val); return ['from' => $val[0], 'to' => str_replace($val[0].',', '', $val[1])]; }, $synonyms); } else { $result = null; } return $result; } /** * Formats synonyms for use in Elastic index settings. * * @return string[] formatted synonyms */ protected function getFormattedSynonyms(?array $synonyms, $expand = true): array { if (empty($synonyms) || empty($synonyms[0]['from'])) { return ['']; } return array_map(function ($val) use ($expand) { if (empty($val['from'])) { return ''; } $self = $expand ? "{$val['from']}," : ''; return "{$val['from']}=>{$self}{$val['to']}"; }, $synonyms); } public function search(string $term, array $config, ?array $types = null, $returnRaw = false): array { $queries = []; $types ??= $this->getIndexTypes(); // prepare queries foreach ($types as $type) { $typeConfig = $config[$type] ?? []; // header $queries[] = json_encode(['index' => $this->getIndex($type)]); // body switch ($type) { case self::INDEX_PRODUCTS: [$order, $order_dir] = $this->getOrder($typeConfig['order'] ?? null); $queries[] = $this->productsSearchParameter($term, $typeConfig['count'], $typeConfig['offset'] ?? 0, $order, $order_dir, $typeConfig['exact'] ?? false); break; default: $queries[] = $this->createSearchParameter($term, $type, $typeConfig['count'], $typeConfig['offset'] ?? 0); } } // execute queries $searchResult = $this->multiSearch($queries); $result = []; $totals = []; // process responses foreach ($searchResult['responses'] ?? [] as $key => $item) { $type = $types[$key] ?? null; if ($type === null) { throw new FulltextException('Multi-search invalid response!', ['type' => $type, 'item' => $item]); } if (!in_array($item['status'], [200, 400, 404])) { // 404 - index does not exists, 400 - index error throw new FulltextException("Multi-search invalid response status: {$type}: {$item['status']} != 200", ['type' => $type, 'item' => $item]); } // TODO: Temporary HACK $result[$type] = $this->processSearchResults($returnRaw ? null : $type, $item); $totals[$type] = $item['hits']['total']['value'] ?? 0; } $this->multiSearchResultTotals = $totals; return $result; } public function searchProducts($term, $count, $offset, $order = null, $filter = '', $exact = false) { [$order, $order_dir] = $this->getOrder($order); $urlEnd = '_search?filter_path=hits.total,hits.hits._id'; return $this->executeCurl(self::INDEX_PRODUCTS, $this->productsSearchParameter($term, $count, $offset, $order, $order_dir, $exact), 'GET', $urlEnd); } public function searchProductsExact($term, $count, $offset, $order = null, $filter = '') { return $this->searchProducts($term, $count, $offset, $order, $filter, true); } private function processProductsSearch($result): array { $this->settings['rows'] = ($result['hits']['total']['value'] ?? 0); return Mapping::mapKeys( $result['hits']['hits'] ?? [], function ($index, $row) { return [$row['_id'], $row['_source'] ?? true]; }, ); } private function processSectionsSearch($result): array { $sections = []; foreach ($result['hits']['hits'] ?? [] as $line) { $sections[] = [ 'id' => $line['_id'], 'name' => $line['_source']['title'], 'path' => $line['_source']['path'], 'photo' => getImage($line['_id'], null, null, 'section', 1), 'photo_src' => $line['_source']['photo'] ?? null, ]; } return $sections; } private function processProducersSearch($result): array { $producers = []; foreach ($result['hits']['hits'] ?? [] as $line) { $producers[] = [ 'id' => $line['_id'], 'name' => $line['_source']['title'], ]; } return $producers; } public function getRowsCount() { return $this->settings['rows']; } public function executeCurl($type, $param, $customRequest, $urlEnd) { $url = $this->getServerUrl().'/'.$this->getIndex($type).'/_doc/'.$urlEnd; $result = $this->processResult($url, $param, $customRequest, $type); return $result; } private function returnSearchBody($term, $fields): array { return [ 'query' => [ 'function_score' => [ 'query' => [ 'multi_match' => [ 'query' => $term, 'type' => 'cross_fields', 'operator' => 'and', 'fields' => $fields, ], ], ], ], ]; } private function processSuggestion($term) { $parameterSuggestion = [ 'suggest' => [ 'text' => $term, 'simple_phrase' => [ 'phrase' => [ 'field' => 'title', 'size' => 1, 'gram_size' => 3, 'direct_generator' => [ [ 'field' => 'title', 'suggest_mode' => 'always', ], ], ], ], ], ]; $parameterSuggestion = json_encode($parameterSuggestion); $url = $this->getServerUrl().'/'.$this->getIndex(self::INDEX_PRODUCTS).'/_search?&filter_path=suggest.simple_phrase.options'; return $this->processResult($url, $parameterSuggestion, 'GET', 'suggestion'); } public function suggestTerm($term) { return $this->processSuggestion($term); } protected function createSearchParameter($term, $type, $count, $offset) { $fields = $this->formatQueryFields($this->getSearchFields($type)); $param = [ 'size' => $count, 'from' => $offset, 'query' => $this->returnSearchBody($term, $fields)['query'], ]; return json_encode($param); } public function getSearchFields(string $type): array { return array_filter($this->{"fields_{$type}"}, fn ($item) => isset($item['weight']) && isset($item['type'])); } public function getElasticFieldsDeps(string $type): array { return array_map(fn ($f) => $f['deps'] ?? [], $this->{"fields_{$type}"}); } public function getElasticFieldsAllDeps(string $type): array { return array_unique(array_reduce($this->getElasticFieldsDeps($type), fn ($allDeps, $fieldDeps) => array_merge($allDeps, $fieldDeps), [])); } public function formatQueryFields(array $fields, ?array $types = null): array { return array_values(array_filter(Mapping::withKeys($fields, function ($key, $value) use ($types) { if (is_null($types) || in_array($value['type'], $types)) { return "{$key}^{$value['weight']}"; } return null; }))); } public function productsSearchParameter($term, $count, $offset, $order, $order_dir, $exact = false) { $fields = $this->getSearchFields(self::INDEX_PRODUCTS); $multiMatches = []; foreach (['text', 'keyword', 'no_spaces'] as $fieldType) { if (!($matchFields = $this->formatQueryFields($fields, [$fieldType]))) { continue; } $multiMatches[] = [ 'multi_match' => [ 'query' => $term, 'fields' => $matchFields, 'type' => 'cross_fields', 'operator' => 'and', ], ]; } $query = [ 'function_score' => [ 'query' => [ 'bool' => [ 'should' => $multiMatches, ], ], 'field_value_factor' => [ 'field' => 'asfnboadnb', 'missing' => 1, ], 'boost_mode' => 'max', ], ]; if ($exact === true) { // returns exact matches $query = [ 'match_phrase' => [ 'title' => [ 'query' => $term, 'analyzer' => self::SEARCH_ANALYZER_NAME, ], ], ]; } switch ($order) { case 'weight': $param = [ 'size' => $count, 'from' => $offset, 'query' => [ 'function_score' => [ 'query' => $query, 'field_value_factor' => [ 'field' => 'weight', ], ], ], ]; break; default: $param = [ 'size' => $count, 'from' => $offset, 'sort' => [ [ $order => [ 'order' => $order_dir, ], ], ], 'query' => [ 'function_score' => [ 'query' => $query, 'field_value_factor' => [ 'field' => 'weight', ], ], ], ]; break; } return json_encode($param); } public function processResult($url, $param, $customRequest, $category) { $result = $this->curlInitSession($url, $customRequest, $param); if ($customRequest == 'GET' && !empty($result['hits']['hits'])) { return $this->processSearchResults($category, $result); } if ($customRequest == 'GET' && $category == 'indexExists') { if (key_exists('error', $result)) { return false; } return true; } if ($category == 'suggestion' && key_exists('suggest', $result) && !empty($result['suggest']['simple_phrase'][0]['options'][0])) { return $result['suggest']['simple_phrase'][0]['options'][0]['text']; } else { $this->settings['rows'] = 0; return $result = []; } } public function deleteProducts($products) { foreach ($products as $product) { $this->executeCurl(self::INDEX_PRODUCTS, '', 'DELETE', $product['id']); } } /** * @param int|array $id_product * * @return bool */ public function updateProduct($id_product) { /* Ugly hack - increase memory limit by 1GB */ increaseMemoryLimit(1024); $id_product = is_array($id_product) ? $id_product : [$id_product]; $this->updateProductFulltext($id_product); $this->updateProductsAttributes($id_product); return true; } public function updateProductFulltext($id_products) { if (!is_array($id_products)) { $id_products = [$id_products]; } [$products, $delete] = $this->elasticDataFetchUtil->fetchProducts($id_products); // Update if ($products) { $this->bulkUpdate($products, self::INDEX_PRODUCTS, 'put'); } // Delete missing products from index if ($delete) { $this->bulkUpdate($delete, self::INDEX_PRODUCTS, 'delete'); } return true; } /** * @param array $productsChangesMap mapping of products changed values in format [idProduct => ['title', etc...], ...] * * @throws FulltextException */ public function partialProductsUpdate(array $productsChangesMap): void { $searchFieldsDeps = $this->getElasticFieldsDeps(self::INDEX_PRODUCTS); $this->partialUpdateInBatches($productsChangesMap, function ($productsChangesMapChunk) use ($searchFieldsDeps) { [$products, $delete] = $this->elasticDataFetchUtil->fetchProducts($productsChangesMapChunk, true); foreach ($products as &$product) { $productChanges = $productsChangesMapChunk[$product['id']] ?? false; if (empty($productChanges)) { continue; } $product = array_filter($product, function ($key) use ($searchFieldsDeps, $productChanges) { if ($key == 'id') { return true; } $fieldDeps = $searchFieldsDeps[$key] ?? false; return $fieldDeps && !empty(array_intersect($fieldDeps, $productChanges)); }, ARRAY_FILTER_USE_KEY); } if ($products) { $this->bulkUpdate($products, self::INDEX_PRODUCTS, 'update_partial'); } if ($delete) { $this->bulkUpdate($delete, self::INDEX_PRODUCTS, 'delete'); } }); } public function bulkUpdate($values, $type, $method) { $finalParam = []; switch ($method) { case 'put': foreach ($values as $line) { $finalParam[] = [ 'index' => [ '_index' => $this->getIndex($type), '_type' => '_doc', '_id' => $line['id'], ], ]; $finalParam[] = $line; } break; case 'update': foreach ($values as $line) { $finalParam[] = [ 'update' => [ '_id' => $line['id'], '_type' => '_doc', '_index' => $this->getIndex($type), ], ]; $finalParam[] = [ 'doc' => [ 'weight' => $line['weight'], 'sold' => intval($line['sold']), 'delivery' => intval($line['delivery']), ], 'upsert' => new \stdClass(), ]; } break; case 'update_partial': foreach ($values as $line) { $finalParam[] = [ 'update' => [ '_id' => $line['id'], '_type' => '_doc', '_index' => $this->getIndex($type), ], ]; $finalParam[] = [ 'doc' => $line, ]; } break; case 'delete': foreach ($values as $line) { $finalParam[] = [ 'delete' => [ '_index' => $this->getIndex($type), '_type' => '_doc', '_id' => $line['id'], ], ]; } break; } $url = $this->getServerUrl().'/'.$this->getIndex($type).'/_bulk'; $data = join( "\n", array_map( function ($x) { return json_encode($x); }, $finalParam ) ); try { $response = $this->curlInitSession($url, 'POST', $data."\n"); $errors = isset($response['errors']) && $response['errors']; if ($errors && $method === 'update_partial') { // ignore when update of non-existing product is attempted $errors = !empty(array_filter($response['items'] ?? [], fn ($item) => empty($item['update']) || !in_array($item['update']['status'] ?? '', [404, 200]))); } if ($errors) { throw new FulltextException('Failed to update fulltext!', [ 'response' => $response, ]); } } catch (FulltextException|\JsonException $e) { $this->sentryLogger->captureException($e); if (isDevelopment()) { throw $e; } } } public function curlInitSession($url, $customRequest, $param) { $header = ['content-type: application/json; charset=UTF-8']; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); if (!empty($param)) { curl_setopt($ch, CURLOPT_POSTFIELDS, $param); } curl_setopt($ch, CURLOPT_HTTPHEADER, $header); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $customRequest); curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_TIMEOUT, $this->curlTimeout); $result = curl_exec($ch); if ($result === false) { throw new FulltextException('Elasticsearch server vrátil neplatnou odpověď!', ['server' => $this->getServerUrl(), 'curl_error' => curl_error($ch), 'curl_errno' => curl_errno($ch)]); } $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($http_code == 413) { // 413 Request Entity Too Large throw new FulltextException('Elasticsearch server vrátil neplatnou odpověď! 413 Request Entity Too Large', ['server' => $this->getServerUrl(), 'data' => $param, 'result' => $result]); } $result = json_decode_strict($result, true); curl_close($ch); return $result; } private function updatePages(): void { $this->withNewIndex(self::INDEX_PAGES, function () { foreach ($this->genericObjectsIndex->getBatchedObjects() as $batch) { $this->bulkUpdate($batch, self::INDEX_PAGES, 'put'); } }); } private function updateArticles() { if (!findModule(\Modules::ARTICLES)) { return; } $query = sqlQueryBuilder()->select('id')->from('articles'); $this->withNewIndex(self::INDEX_ARTICLES, function () use ($query) { $this->updateInBatches( $query->execute(), function ($chunk) { $this->updateArticle($chunk); } ); } ); } public function updateArticle($id_article) { if (!is_array($id_article)) { $id_article = [$id_article]; } $specs = [ Operator::inIntArray($id_article, 'a.id'), Operator::equals(['a.show_in_search' => 'Y']), function (QueryBuilder $qb) { $qb->addSelect('GROUP_CONCAT(b.content SEPARATOR "") as content') ->leftJoin('a', 'blocks', 'b', 'b.id_root = a.id_block'); }, ]; $photoTypeId = findModule(\Modules::COMPONENTS) ? 11 : 12; $cfg = Config::get(); if (isset($cfg['Photo']['id_to_type'][$photoTypeId])) { $this->articleList->setImage($cfg['Photo']['id_to_type'][$photoTypeId]); } $articles = $this->articleList ->getArticles(Operator::andX($specs)); $bulkValues = []; foreach ($articles as $article) { $tags = array_map(function ($x) { return $x['tag']; }, $article['tags'] ?? []); $bulkValues[] = [ 'id' => $article['id'], 'title' => $article['title'], 'keywords' => $article['keywords'] ?? '', 'lead_in' => $article['lead_in'], 'section' => $article['section']['name'] ?? '', 'tags' => join(',', $tags), 'content' => StringUtil::htmlToCleanText($article['content'] ?? ''), 'photo' => $article['image']['src'] ?? null, ]; } $this->bulkUpdate($bulkValues, self::INDEX_ARTICLES, 'put'); return true; } /** * Generates new index with products. * * @return void * * @throws FulltextException */ private function updateProducts() { increaseMemoryLimit(1024); $query = $this->createProductQueryBuilder(); $this->withNewIndex(self::INDEX_PRODUCTS, function () use ($query) { $this->updateInBatches( $query->execute(), function ($chunk) { $this->updateProductFulltext($chunk); } ); } ); } public function updateInBatches($rows, $callback) { foreach (array_chunk($rows->fetchAll(), 100) as $chunk) { try { $callback( array_map( function ($x) { return $x['id']; }, $chunk ) ); } catch (FulltextException $e) { $this->sentryLogger->captureException($e, ['extra' => $e->getData()]); if (isDevelopment()) { throw $e; } } } } protected function partialUpdateInBatches($items, $callback) { foreach (array_chunk($items, 100, true) as $chunk) { try { $callback($chunk); } catch (FulltextException $e) { $this->sentryLogger->captureException(new \Exception('Kafka - Elastic index update failed (chunk)', 0, $e), ['extra' => [ 'chunk' => $chunk, ]]); if (isDevelopment()) { throw $e; } } } } public function updateSection($id_section) { if (!is_array($id_section)) { $id_section = [$id_section]; } $dbcfg = \Settings::getDefault(); // Texts $SQL = sqlQueryBuilder()->select('s.id, s.name, CONCAT_WS(" ", GROUP_CONCAT(DISTINCT b.content ORDER BY b.position ASC SEPARATOR "") , GROUP_CONCAT(DISTINCT pr.name)) AS description') ->from('sections', 's') ->leftJoin('s', 'products_in_sections', 'pis', 'pis.id_section = s.id') ->leftJoin('s', 'blocks', 'b', 'b.id_root = s.id_block') ->andWhere(Operator::inIntArray($id_section, 's.id')) ->groupBy('s.id'); if ($dbcfg->cat_show_empty == 'N') { $productList = new \ProductList(); $fp = $productList->applyDefaultFilterParams(); $fp->setSections($id_section); $productList->andSpec(function (QueryBuilder $qb) { $qb->select('p.id'); }); $productListQB = $productList->getQueryBuilder(); $SQL->leftjoin('pis', 'products', 'p', 'p.id = pis.id_product AND p.id IN ('.$productListQB->getSQL().')') ->addQueryBuilderParameters($productListQB); $SQL->andWhere("p.figure = 'Y' OR s.virtual = 'Y'"); } else { $SQL->leftJoin('pis', 'products', 'p', 'p.id = pis.id_product'); } $SQL->leftJoin('p', 'producers', 'pr', 'p.producer = pr.id'); // index only visible sections $SQL->andWhere( Translation::joinTranslatedFields( SectionsTranslation::class, function (QueryBuilder $qb, $columnName, $translatedField) { if ($columnName == 'figure') { $qb->andWhere(Operator::coalesce($translatedField, 's.figure')." != 'N' AND s.show_in_search = 'Y'"); return false; } return true; }, ['name', 'figure'] ) ); $results = $SQL->execute()->fetchAll(); $bulkValues = []; foreach ($results as $values) { $values['title'] = $values['name']; // Path $navArray = getReturnNavigation($values['id']); $navArray = reset($navArray); if ($navArray) { array_shift($navArray); $values['path'] = join( ' / ', array_map( function ($x) { return $x['text']; }, $navArray ) ); $values['path_str'] = $values['path']; } $values['title_str'] = $values['title']; try { // v try-catch, protoze nechci, aby to behem reindexace zuchlo treba na selhani nacteni cache $section = $this->sectionTree->getSectionById($values['id']); } catch (\Throwable $e) { $section = null; } $bulkValues[] = [ 'id' => $values['id'], 'title' => $values['title'], 'description' => $values['description'], 'path' => $values['path'] ?? '', 'photo' => $section ? $section->getPhoto()['src'] ?? null : null, ]; } // Update $this->bulkUpdate($bulkValues, self::INDEX_SECTIONS, 'put'); return true; } private function updateSections() { $query = sqlQueryBuilder() ->select('id') ->from('sections') ->where('id > 0'); $this->withNewIndex(self::INDEX_SECTIONS, function () use ($query) { $this->updateInBatches( $query->execute(), function ($chunk) { $this->updateSection($chunk); } ); } ); } public function updateProducer($id_producer) { $dbcfg = \Settings::getDefault(); // Texts $SQL = sqlQueryBuilder()->select('pr.id, pr.name, 1 AS visible, CONCAT_WS(" ", GROUP_CONCAT(DISTINCT b.content ORDER BY b.position ASC SEPARATOR ""), GROUP_CONCAT(DISTINCT s.name)) AS long_description') ->from('producers', 'pr') ->leftJoin('pr', 'blocks', 'b', 'b.id_root = pr.id_block') ->leftJoin('p', 'products_in_sections', 'pis', 'pis.id_product = p.id') ->leftJoin('pis', 'sections', 's', 's.id = pis.id_section') ->where('pr.id = :id_producer AND pr.active = :active') ->setParameters( [ 'id_producer' => $id_producer, 'active' => 'Y', ] )->groupBy('pr.id'); if ($dbcfg->cat_show_empty == 'N') { $productList = new \ProductList(); $fp = $productList->applyDefaultFilterParams(); $fp->setProducers([$id_producer]); $productList->andSpec(function (QueryBuilder $qb) { $qb->select('p.id'); }); $productListQB = $productList->getQueryBuilder(); $SQL->join('pr', 'products', 'p', 'p.id IN ('.$productListQB->getSQL().')') ->addQueryBuilderParameters($productListQB); } else { $SQL->leftJoin('pr', 'products', 'p', 'pr.id = p.producer'); } if (findModule(\Modules::TRANSLATIONS)) { $SQL->andWhere( \Query\Translation::coalesceTranslatedFields( \KupShop\I18nBundle\Translations\ProducersTranslation::class ) ); } $values = $SQL->execute()->fetch(); if (!$values) { return false; } $values['title'] = $values['name']; $values['description'] = $values['long_description']; // Update $this->executeCurl(self::INDEX_PRODUCERS, $this->createUpdateParameter('producers', $values), 'PUT', $id_producer); return true; } private function updateProducers() { if (!findModule(\Modules::PRODUCERS)) { return; } $this->withNewIndex(self::INDEX_PRODUCERS, function () { foreach (sqlQuery('SELECT id FROM producers') as $producer) { $this->updateProducer($producer['id']); } } ); } public function createUpdateParameter($category, $values) { $param = []; switch ($category) { case 'sections': if (array_key_exists('path', $values)) { $param = [ 'title' => $values['title'], 'description' => $values['description'], 'path' => $values['path'], ]; } break; case 'producers': $param = [ 'title' => $values['title'], 'description' => $values['description'], ]; break; } return json_encode($param); } public function updateProductsAttributes($id_product = null) { $sql = $this->createProductQueryBuilder() ->addSelect('p.position'); if ($id_product) { if (is_array($id_product)) { $sql->andWhere(Operator::inIntArray($id_product, 'p.id')); } else { $sql->andWhere('p.id = :id_product')->setParameter('id_product', $id_product); } } if (findModule('products_suppliers')) { $sql->joinProductsOfSuppliers() ->addSelect('SUM(IF(pos.in_store > 0, pos.in_store, 0)) as in_store_suppliers'); } $updatedValues = []; foreach ($sql->execute() as $row) { $product = new \Product(); $product->createFromArray($row); $product->prepareDeliveryText(); $values = $this->elasticDataFetchUtil->getProductsAttributesValues($product); $updatedValues[] = array_merge($values, ['id' => $product->id]); if (count($updatedValues) > 99) { $this->bulkUpdate($updatedValues, self::INDEX_PRODUCTS, 'update'); unset($updatedValues); $updatedValues = []; } } if (!empty($updatedValues)) { $this->bulkUpdate($updatedValues, self::INDEX_PRODUCTS, 'update'); } } public function multiSearch(array $queries) { return $this->curlInitSession( $this->getServerUrl().'/_msearch', 'GET', implode("\n", $queries)."\n" ); } /** * Adds hunspell filter settings into provided $filters array; or return hunspell filter settings. */ protected function getHunspellFilter(array $filters = []): array { $locale = null; switch ($this->languageContext->getActiveId()) { case 'cs': $locale = 'cs_CZ'; break; case 'sk': $locale = 'sk_SK'; break; } if ($locale) { $filters['hunspell'] = [ 'type' => 'hunspell', 'locale' => $locale, ]; } $filters['custom_stemmer'] = [ 'type' => 'multiplexer', 'filters' => $locale ? ['hunspell'] : [], ]; return $filters; } protected function createProductQueryBuilder() { // Nepředefinovávaj mě, předefinuj getProductsSpec, která se použije na více místech než já $qb = sqlQueryBuilder()->select('p.id, p.title, p.discount, p.vat, p.delivery_time, p.pieces_sold, p.code, p.in_store') ->fromProducts() ->joinVariationsOnProducts() ->andWhere($this->getProductsSpec()) ->andWhere('p.id > 0') ->groupBy('p.id'); return $qb; } /** * @return \FilterParams */ protected function getFilterParams() { if ($this->filterParams) { return $this->filterParams; } return $this->filterParams = new \FilterParams(); } public function getIndex(string $type, ?bool $useNextIndex = null) { return $this->getIndexPrefix($useNextIndex).'.'.$type; } public function getIndexPrefix(?bool $useNextIndex = null, bool $appendLanguage = true): string { if ($useNextIndex === null) { $useNextIndex = $this->_useNextIndex; } $alias = $useNextIndex ? self::INDEX_ALIAS_NEXT : self::INDEX_ALIAS_CURRENT; if ($appendLanguage && findModule(\Modules::TRANSLATIONS)) { $alias .= '_'.$this->languageContext->getActiveId(); } return $this->settings['index'].'_'.$alias; } private function getOrder(?string $order): array { $order = $order ?: '-weight'; $order_dir = 'asc'; if (substr($order, 0, 1) == '-') { $order_dir = 'desc'; $order = substr($order, 1); } return [$order, $order_dir]; } public function getServerUrl(): string { return getenv('ELASTIC_SERVER') ?: 'elasticsearch.wpj.cz:80'; } public function updateIndex(string $type = 'all', $clean = true, array $exceptTypes = []): void { if ($clean) { $this->clearIndices(); } switch ($type) { case self::INDEX_PRODUCTS: $this->updateProducts(); $this->updateProductsAttributes(); break; case 'all': foreach ($this->getIndexTypes() as $iType) { if (!in_array($iType, $exceptTypes)) { $this->updateIndex($iType, false); } } break; default: $titleType = ucfirst($type); $this->{"update{$titleType}"}(); } if ($clean) { $this->clearIndices(); } } /** * @return array */ public function getFulltextLanguages(): array { $languages = []; $languageContext = Contexts::get(LanguageContext::class); if (!findModule(\Modules::TRANSLATIONS)) { if ($default = $languageContext->getAll()[$languageContext->getDefaultId()] ?? null) { $languages[$default->getId()] = $default; } return $languages; } foreach ($languageContext->getAll() as $language) { if (!$language->isActive()) { continue; } $languages[$language->getId()] = $language; } return $languages; } protected function processSearchResults($type, $result) { switch ($type) { case static::INDEX_PRODUCTS: return $this->processProductsSearch($result); case self::INDEX_SECTIONS: return $this->processSectionsSearch($result); case self::INDEX_PRODUCERS: return $this->processProducersSearch($result); default: // Combine source with id $results = $result['hits']['hits'] ?? []; return array_map(function ($x) {return ['id' => $x['_id']] + $x['_source']; }, $results); } } public function getProductsSpec(): callable { return $this->getFilterParams()->getSpec(); } public function getFilters(): array { return []; } public function setDynamicFilters(array $filters): void { } #[Required] final public function setGenericObjectsIndex(GenericObjectsIndex $genericObjectsIndex): void { $this->genericObjectsIndex = $genericObjectsIndex; $this->fields_pages = $genericObjectsIndex->getFields(); } public function supportsFilters(): bool { return false; } public static function isServerAvailable() { return !getCache(Client::ELASTIC_UNAVAILABLE_KEY); } }