1852 lines
62 KiB
PHP
1852 lines
62 KiB
PHP
<?php
|
|
|
|
namespace KupShop\CatalogBundle\Search;
|
|
|
|
use KupShop\CatalogBundle\Parameters\ParameterType;
|
|
use KupShop\CatalogBundle\Search\Exception\FulltextException;
|
|
use KupShop\CatalogBundle\Section\SectionTree;
|
|
use KupShop\CatalogBundle\Util\ElasticDataFetchUtil;
|
|
use KupShop\CatalogElasticBundle\Util\Client;
|
|
use KupShop\CatalogElasticBundle\Util\ProductIndexExtension;
|
|
use KupShop\ContentBundle\Util\ArticleList;
|
|
use KupShop\I18nBundle\Entity\Language;
|
|
use KupShop\I18nBundle\Translations\SectionsTranslation;
|
|
use KupShop\KupShopBundle\Config;
|
|
use KupShop\KupShopBundle\Context\LanguageContext;
|
|
use KupShop\KupShopBundle\Util\Compat\ServiceContainer;
|
|
use KupShop\KupShopBundle\Util\Contexts;
|
|
use KupShop\KupShopBundle\Util\Functional\Mapping;
|
|
use KupShop\KupShopBundle\Util\Logging\SentryLogger;
|
|
use KupShop\KupShopBundle\Util\StringUtil;
|
|
use Query\Operator;
|
|
use Query\QueryBuilder;
|
|
use Query\Translation;
|
|
use Symfony\Contracts\Service\Attribute\Required;
|
|
|
|
class FulltextElastic implements FulltextInterface
|
|
{
|
|
use \DatabaseCommunication;
|
|
|
|
public const INDEX_PRODUCTS = 'products';
|
|
public const INDEX_SECTIONS = 'sections';
|
|
public const INDEX_PRODUCERS = 'producers';
|
|
public const INDEX_ARTICLES = 'articles';
|
|
public const INDEX_PAGES = 'pages';
|
|
|
|
public const INDEX_TYPES = [
|
|
self::INDEX_PRODUCTS,
|
|
self::INDEX_SECTIONS,
|
|
self::INDEX_PRODUCERS,
|
|
self::INDEX_ARTICLES,
|
|
self::INDEX_PAGES,
|
|
];
|
|
|
|
private const INDEX_ANALYZER_NAME = 'icu_folding';
|
|
private const SEARCH_ANALYZER_NAME = 'query';
|
|
|
|
protected array $fields_products = [
|
|
'title' => ['weight' => 10, 'type' => 'text', 'deps' => ['title']],
|
|
'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'],
|
|
'title.keyword' => ['weight' => 0, 'type' => 'text_keyword'],
|
|
'annotation' => ['weight' => 4, 'type' => 'text', 'deps' => ['short_descr']],
|
|
'description' => ['weight' => 2, 'type' => 'text', 'deps' => ['long_descr']],
|
|
'description2' => ['weight' => 2, 'type' => 'text', 'deps' => ['parameters']],
|
|
'description_plus' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Blocek]],
|
|
'variations' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Variations]],
|
|
'sections' => ['weight' => 4, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Sections]],
|
|
'parent_sections' => ['weight' => 2, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Sections]],
|
|
'code' => ['weight' => 7, 'type' => 'keyword', 'deps' => [ElasticUpdateGroup::Code]],
|
|
'code.no_spaces' => ['weight' => 7, 'type' => 'no_spaces'],
|
|
'ean' => ['weight' => 5, 'type' => 'keyword', 'deps' => [ElasticUpdateGroup::Ean]],
|
|
'ean.no_spaces' => ['weight' => 5, 'type' => 'no_spaces'],
|
|
'producer' => ['weight' => 4, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Producer]],
|
|
'producer.no_spaces' => ['weight' => 4, 'type' => 'no_spaces'],
|
|
'parameters' => ['weight' => 1, 'type' => 'text', 'deps' => [ElasticUpdateGroup::Parameters]],
|
|
'price' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]],
|
|
'delivery' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::DeliveryTime]],
|
|
'sold' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => ['pieces_sold']],
|
|
'weight' => ['weight' => 0, 'type' => 'sort_float', 'deps' => ['weight', 'pieces_sold', ElasticUpdateGroup::Position, ElasticUpdateGroup::DeliveryTime, ElasticUpdateGroup::InStore]],
|
|
'position' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::Position]],
|
|
|
|
'sort_position_in_sections' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::SectionPosition]],
|
|
'sort_position_in_producers' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::SectionPosition]],
|
|
|
|
'filter_section' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Sections]],
|
|
'filter_section_recursive' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Sections]],
|
|
'filter_producer' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Producer]],
|
|
'filter_in_sale' => ['weight' => 0, 'type' => 'filter_boolean', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]],
|
|
'filter_show_in_search' => ['weight' => 0, 'type' => 'filter_boolean', 'deps' => ['show_in_search']],
|
|
|
|
'filter_labels' => ['weight' => 0, 'type' => 'filter_integer', 'deps' => [ElasticUpdateGroup::Labels]],
|
|
'filter_campaigns' => ['weight' => 0, 'type' => 'keyword', 'deps' => ['campaign']],
|
|
|
|
// will be removed, use variations_data
|
|
'filter_variations' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Variations]],
|
|
|
|
'filter_parameters_'.ParameterType::LIST => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]],
|
|
'filter_convertors_parameters' => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]],
|
|
'filter_parameters_'.ParameterType::STRING => ['weight' => 0, 'type' => 'filter_nested', 'deps' => [ElasticUpdateGroup::Parameters]],
|
|
'filter_parameters_'.ParameterType::FLOAT => ['weight' => 0, 'type' => 'nested', 'properties' => [
|
|
'id' => ['type' => 'keyword'],
|
|
'value' => ['type' => 'float'],
|
|
], 'deps' => [ElasticUpdateGroup::Parameters]],
|
|
|
|
'filter_stores_in_store' => ['weight' => 0, 'type' => 'nested', 'properties' => [
|
|
'id' => ['type' => 'keyword'],
|
|
'in_store' => ['type' => 'integer'],
|
|
], 'deps' => [ElasticUpdateGroup::StoresInStore]],
|
|
|
|
// will be removed, use variations_data
|
|
'variations_in_store' => ['weight' => 0, 'type' => 'nested', 'properties' => [
|
|
'id' => ['type' => 'keyword'],
|
|
'id_label' => ['type' => 'keyword'],
|
|
'in_store' => ['type' => 'integer'],
|
|
], 'deps' => [ElasticUpdateGroup::InStore]],
|
|
|
|
'variations_data' => ['weight' => 0, 'type' => 'nested', 'properties' => [
|
|
'id' => ['type' => 'keyword'],
|
|
'combination' => ['type' => 'flattened'],
|
|
'in_store' => ['type' => 'integer'],
|
|
], 'deps' => [ElasticUpdateGroup::InStore, ElasticUpdateGroup::Variations]],
|
|
|
|
'in_store' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::InStore]],
|
|
'sort_position' => ['weight' => 0, 'type' => 'sort_integer', 'deps' => [ElasticUpdateGroup::Position]],
|
|
'sort_section_positions' => ['weight' => 0, 'type' => 'filter_positions', 'deps' => [ElasticUpdateGroup::SectionPosition]],
|
|
'sort_discount' => ['weight' => 0, 'type' => 'sort_float', 'deps' => [ElasticUpdateGroup::Price, ElasticUpdateGroup::Discount]],
|
|
|
|
'date_added' => ['weight' => 0, 'type' => 'sort_timestamp'],
|
|
'date_updated' => ['weight' => 0, 'type' => 'sort_timestamp', 'deps' => ['updated']],
|
|
'date_stock_in' => ['weight' => 0, 'type' => 'sort_timestamp'],
|
|
'store_value' => ['weight' => 0, 'type' => 'integer', 'deps' => [ElasticUpdateGroup::InStore, 'price_buy']],
|
|
|
|
'delivery_time_index' => ['deps' => [ElasticUpdateGroup::DeliveryTime]],
|
|
'delivery_time_text' => ['deps' => [ElasticUpdateGroup::DeliveryTime]],
|
|
'visible' => ['deps' => ['figure']],
|
|
'id_photo' => ['deps' => [ElasticUpdateGroup::Photo]],
|
|
'price_original' => ['deps' => [ElasticUpdateGroup::Price, 'price_orig']],
|
|
'discount' => ['deps' => [ElasticUpdateGroup::Discount]],
|
|
];
|
|
|
|
protected array $fields_sections = [
|
|
'title' => ['weight' => 10, 'type' => 'text'],
|
|
'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'],
|
|
'description' => ['weight' => 2, 'type' => 'text'],
|
|
'path' => ['weight' => 1, 'type' => 'text'],
|
|
];
|
|
|
|
protected array $fields_producers = [
|
|
'title' => ['weight' => 10, 'type' => 'text'],
|
|
'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'],
|
|
'description' => ['weight' => 2, 'type' => 'text'],
|
|
];
|
|
|
|
protected array $fields_articles = [
|
|
'title' => ['weight' => 10, 'type' => 'text'],
|
|
'title.no_spaces' => ['weight' => 2, 'type' => 'no_spaces'],
|
|
'keywords' => ['weight' => 8, 'type' => 'text'],
|
|
'lead_in' => ['weight' => 7, 'type' => 'text'],
|
|
'section' => ['weight' => 6, 'type' => 'text'],
|
|
'tags' => ['weight' => 6, 'type' => 'text'],
|
|
'content' => ['weight' => 2, 'type' => 'text'],
|
|
];
|
|
|
|
protected array $fields_pages = GenericObjectsIndex::FIELDS;
|
|
|
|
/**
|
|
* @var \Doctrine\DBAL\Connection
|
|
*/
|
|
private $connection;
|
|
protected $settings;
|
|
|
|
/** @var LanguageContext */
|
|
private $languageContext;
|
|
|
|
private $sentryLogger;
|
|
|
|
/** @var ArticleList */
|
|
private $articleList;
|
|
|
|
/** @var SectionTree */
|
|
private $sectionTree;
|
|
|
|
protected $filterParams;
|
|
|
|
protected $curlTimeout = 60;
|
|
|
|
public array $multiSearchResultTotals = [];
|
|
|
|
protected GenericObjectsIndex $genericObjectsIndex;
|
|
public ?ProductIndexExtension $productIndexExtension = null;
|
|
|
|
#[Required]
|
|
public ElasticDataFetchUtil $elasticDataFetchUtil;
|
|
|
|
public function __construct(LanguageContext $languageContext, SentryLogger $sentryLogger)
|
|
{
|
|
if (findModule(\Modules::PRODUCTS_SECTIONS, \Modules::SUB_ELASTICSEARCH)) {
|
|
$this->productIndexExtension = ServiceContainer::getService(ProductIndexExtension::class);
|
|
}
|
|
|
|
global $cfg;
|
|
|
|
$default = [
|
|
'index' => $cfg['Connection']['database'],
|
|
'rows' => 0,
|
|
'synonyms' => [''],
|
|
];
|
|
|
|
$this->languageContext = $languageContext;
|
|
$this->sentryLogger = $sentryLogger;
|
|
if (array_key_exists('fulltext_search', $cfg['Modules'])) {
|
|
$this->settings = array_merge($default, (array) $cfg['Modules']['fulltext_search']);
|
|
} else {
|
|
$this->settings = $default;
|
|
}
|
|
}
|
|
|
|
/** @var bool Indicates which index to use */
|
|
private bool $_useNextIndex = false;
|
|
public const INDEX_ALIAS_CURRENT = 'current';
|
|
public const INDEX_ALIAS_NEXT = 'next';
|
|
|
|
/**
|
|
* Creates a new index of specified type and tries to import data via callback.
|
|
* If import fails the newly created index won't be used.
|
|
*
|
|
* @param callable $func callback, if this fails the new index won't be
|
|
* aliased as current and therefore won't be used
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
protected function withNewIndex(string $indexType, callable $func): bool
|
|
{
|
|
$this->_useNextIndex = true;
|
|
|
|
$newIndexName = null;
|
|
try {
|
|
$newIndexName = $this->createNewIndex($indexType);
|
|
$func();
|
|
} catch (\Exception $e) {
|
|
if ($e instanceof FulltextException) {
|
|
throw $e;
|
|
}
|
|
|
|
throw new FulltextException($e->getMessage(), [], $e);
|
|
} finally {
|
|
$this->_useNextIndex = false;
|
|
}
|
|
|
|
$this->aliasOneAsCurrent($indexType, $newIndexName);
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Creates a new index with specified alias.
|
|
*
|
|
* @return string Index name
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
protected function createNewIndex(string $type, bool $useAlias = true): string
|
|
{
|
|
$lang = '';
|
|
$opts = $this->returnIndexSettings($type);
|
|
$serverUrl = $this->getServerUrl();
|
|
|
|
// Temporary fix: Delete index created using alias name
|
|
$alias = $this->getIndex($type, false);
|
|
$this->curlInitSession("{$serverUrl}/{$alias}", 'DELETE', '');
|
|
|
|
if (findModule(\Modules::TRANSLATIONS)) {
|
|
$lang = $this->languageContext->getActiveId();
|
|
}
|
|
$indexName = $this->settings['index'].'_'.time()."_{$lang}.{$type}";
|
|
|
|
if ($useAlias) {
|
|
$opts['aliases'] = [
|
|
$this->getIndex($type) => (object) null,
|
|
];
|
|
}
|
|
|
|
$url = $this->getServerUrl().'/'.$indexName;
|
|
|
|
$this->curlInitSession($url, 'DELETE', '');
|
|
$res = $this->curlInitSession($url, 'PUT', json_encode($opts));
|
|
|
|
if (array_key_exists('error', $res) || !$res['acknowledged']) {
|
|
throw new FulltextException('Failed to create index.',
|
|
['url' => $url, 'data' => $opts, 'result' => $res]);
|
|
}
|
|
|
|
return $indexName;
|
|
}
|
|
|
|
/**
|
|
* Deletes all indices, that are not aliased as current;
|
|
* removes next aliases.
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
private function clearIndices(): void
|
|
{
|
|
$sUrl = $this->getServerUrl();
|
|
$prefix = $this->settings['index'];
|
|
|
|
// remove next aliases
|
|
$req = [
|
|
'actions' => [
|
|
'remove' => [
|
|
'index' => "{$prefix}_*",
|
|
'alias' => $this->getIndexPrefix(true, false).'*',
|
|
],
|
|
],
|
|
];
|
|
$this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($req));
|
|
|
|
$allIndices = $this->getAllIndices();
|
|
if (empty($allIndices)) {
|
|
return;
|
|
}
|
|
|
|
$currentIndices = $this->getCurrentIndices();
|
|
|
|
foreach (array_diff($allIndices, $currentIndices) as $i) {
|
|
$this->curlInitSession("{$sUrl}/{$i}", 'DELETE', '');
|
|
}
|
|
}
|
|
|
|
public function getAllIndices()
|
|
{
|
|
$sUrl = $this->getServerUrl();
|
|
$prefix = $this->settings['index'];
|
|
$allIndices = $this->curlInitSession("{$sUrl}/_cat/indices/{$prefix}_*?format=JSON", 'GET', '');
|
|
|
|
return empty($allIndices) ? [] : array_column($allIndices, 'index');
|
|
}
|
|
|
|
public function getCurrentIndices()
|
|
{
|
|
$sUrl = $this->getServerUrl();
|
|
$prefix = $this->settings['index'];
|
|
$curQuery = $this->getIndexPrefix(false, false).'*';
|
|
|
|
$currentIndices = $this->curlInitSession("{$sUrl}/{$prefix}_*/_alias/{$curQuery}?format=JSON", 'GET', '');
|
|
|
|
return array_keys($currentIndices);
|
|
}
|
|
|
|
/**
|
|
* Removes current alias from old index and gives it to new one.
|
|
*
|
|
* @param string $type index type
|
|
* @param string $indexName index, that will receive the current alias
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
private function aliasOneAsCurrent(string $type, string $indexName): void
|
|
{
|
|
$prefix = $this->settings['index'];
|
|
$alias = $this->getIndex($type, false);
|
|
$sUrl = $this->getServerUrl();
|
|
|
|
// check if the new index exists before current alias is deleted
|
|
$exists = $this->curlInitSession("{$sUrl}/{$indexName}", 'GET', '');
|
|
|
|
if (!empty($exists['error'])) {
|
|
throw new FulltextException('Failed to alias index as current - the index does not exist',
|
|
['url' => "{$sUrl}/{$indexName}", 'data' => $indexName, 'result' => $exists]);
|
|
}
|
|
|
|
$removeReq = [
|
|
'actions' => [
|
|
'remove' => [
|
|
'index' => "{$prefix}*",
|
|
'alias' => $alias,
|
|
],
|
|
],
|
|
];
|
|
|
|
// allowed to fail, when alias does not exist
|
|
$this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($removeReq));
|
|
|
|
$addReq = [
|
|
'actions' => [
|
|
'add' => [
|
|
'index' => $indexName,
|
|
'alias' => $alias,
|
|
],
|
|
],
|
|
];
|
|
|
|
$res = $this->curlInitSession("{$sUrl}/_aliases", 'POST', json_encode($addReq));
|
|
|
|
if (!empty($res['error']) || !$res['acknowledged']) {
|
|
throw new FulltextException('Failed to assign index alias.',
|
|
['url' => "{$sUrl}/_aliases", 'data' => $addReq, 'result' => $res]);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @required
|
|
*/
|
|
public function setSectionTree(SectionTree $sectionTree): void
|
|
{
|
|
$this->sectionTree = $sectionTree;
|
|
}
|
|
|
|
/**
|
|
* @required
|
|
*/
|
|
public function setArticleList(ArticleList $articleList): void
|
|
{
|
|
$this->articleList = $articleList;
|
|
}
|
|
|
|
public function setCurlTimeout($timeout): self
|
|
{
|
|
$this->curlTimeout = $timeout;
|
|
|
|
return $this;
|
|
}
|
|
|
|
public function getIndexTypes(): array
|
|
{
|
|
return static::INDEX_TYPES;
|
|
}
|
|
|
|
protected function getIndexMapping(string $type): array
|
|
{
|
|
$mapping = [];
|
|
foreach ($this->getSearchFields($type) as $key => $options) {
|
|
switch ($options['type']) {
|
|
case 'text':
|
|
$mapping[$key] = [
|
|
'type' => 'text',
|
|
'analyzer' => self::INDEX_ANALYZER_NAME,
|
|
'search_analyzer' => self::SEARCH_ANALYZER_NAME,
|
|
];
|
|
break;
|
|
|
|
case 'keyword':
|
|
$mapping[$key] = [
|
|
'type' => 'keyword',
|
|
];
|
|
break;
|
|
|
|
case 'text_keyword':
|
|
$key = str_replace('.keyword', '', $key);
|
|
if (!isset($mapping[$key]['fields'])) {
|
|
$mapping[$key]['fields'] = [];
|
|
}
|
|
|
|
$mapping[str_replace('.keyword', '', $key)]['fields']['keyword'] = [
|
|
'type' => 'keyword',
|
|
];
|
|
break;
|
|
|
|
case 'no_spaces':
|
|
$mapping[str_replace('.no_spaces', '', $key)]['fields'] = [
|
|
'no_spaces' => [
|
|
'type' => 'text',
|
|
'analyzer' => 'ngram_analyze',
|
|
'search_analyzer' => 'ngram_search',
|
|
],
|
|
];
|
|
break;
|
|
|
|
case 'integer':
|
|
case 'float':
|
|
$mapping[$key] = [
|
|
'type' => $options['type'],
|
|
];
|
|
break;
|
|
|
|
case 'sort_integer':
|
|
$mapping[$key] = [
|
|
'type' => 'integer',
|
|
'index' => false,
|
|
];
|
|
break;
|
|
|
|
case 'sort_float':
|
|
$mapping[$key] = [
|
|
'type' => 'float',
|
|
'index' => false,
|
|
];
|
|
break;
|
|
|
|
case 'sort_timestamp':
|
|
$mapping[$key] = [
|
|
'type' => 'date',
|
|
'format' => 'epoch_second',
|
|
'index' => false,
|
|
];
|
|
break;
|
|
|
|
case 'filter_integer':
|
|
$mapping[$key] = [
|
|
'type' => 'integer',
|
|
];
|
|
break;
|
|
case 'filter_boolean':
|
|
$mapping[$key] = [
|
|
'type' => 'boolean',
|
|
];
|
|
break;
|
|
|
|
case 'filter_positions':
|
|
$mapping[$key] = [
|
|
'type' => 'rank_features',
|
|
];
|
|
break;
|
|
|
|
case 'filter_nested':
|
|
$mapping[$key] = [
|
|
'type' => 'flattened',
|
|
];
|
|
break;
|
|
|
|
case 'nested':
|
|
$mapping[$key] = [
|
|
'type' => 'nested',
|
|
];
|
|
|
|
if ($options['properties']) {
|
|
$mapping[$key]['properties'] = $options['properties'];
|
|
}
|
|
break;
|
|
|
|
default:
|
|
throw new \Exception('Unknown field type: '.$options['type']);
|
|
}
|
|
}
|
|
|
|
return $mapping;
|
|
}
|
|
|
|
public function returnIndexSettings(string $type): array
|
|
{
|
|
$mappingSettings = $this->getIndexMapping($type);
|
|
|
|
$this->loadSynonyms();
|
|
|
|
$settings = [
|
|
'settings' => [
|
|
'analysis' => [
|
|
'analyzer' => [
|
|
// Fulltext analyze
|
|
self::INDEX_ANALYZER_NAME => [
|
|
'tokenizer' => 'standard',
|
|
'filter' => [
|
|
'custom_stemmer',
|
|
'icu_folding',
|
|
'shingle',
|
|
],
|
|
'char_filter' => [
|
|
'icu_normalizer',
|
|
],
|
|
],
|
|
// Fulltext search
|
|
self::SEARCH_ANALYZER_NAME => [
|
|
'tokenizer' => 'standard',
|
|
'filter' => [
|
|
'synonymsFilter',
|
|
'custom_stemmer',
|
|
'icu_folding',
|
|
'remove_duplicities',
|
|
],
|
|
'char_filter' => [
|
|
'icu_normalizer',
|
|
],
|
|
],
|
|
// Without spaces - analyze
|
|
// Když analyzuju, chci odstranit i mezery, abych dostal jeden velký keyword
|
|
'ngram_analyze' => [
|
|
'tokenizer' => 'ngram_short',
|
|
'filter' => ['icu_folding'],
|
|
'char_filter' => ['icu_normalizer', 'dash_space_filter'],
|
|
],
|
|
// Without spaces - search - když hledám, chci zachovat mezery, abych měl víc tokenů
|
|
'ngram_search' => [
|
|
'tokenizer' => 'ngram_short',
|
|
'filter' => ['icu_folding'],
|
|
'char_filter' => ['icu_normalizer', 'dash_filter'],
|
|
],
|
|
],
|
|
'char_filter' => [
|
|
'dash_space_filter' => [
|
|
'type' => 'pattern_replace',
|
|
'pattern' => '(\-|\040|\056|\052|\057)',
|
|
'replacement' => '',
|
|
],
|
|
'dash_filter' => [
|
|
'type' => 'pattern_replace',
|
|
'pattern' => '(\-|\056|\052|\057)',
|
|
'replacement' => '',
|
|
],
|
|
],
|
|
'filter' => array_merge(
|
|
$this->getHunspellFilter(),
|
|
[
|
|
'remove_duplicities' => [
|
|
'type' => 'unique',
|
|
'only_on_same_position' => true,
|
|
],
|
|
'synonymsFilter' => [
|
|
'type' => 'synonym',
|
|
'synonyms' => $this->getFormattedSynonyms($this->settings['synonyms']),
|
|
'tokenizer' => 'standard',
|
|
'ignore_case' => true,
|
|
'expand' => true,
|
|
],
|
|
'shingle' => [
|
|
'type' => 'shingle',
|
|
'min_shingle_size' => 2,
|
|
'max_shingle_size' => 3,
|
|
],
|
|
]),
|
|
'tokenizer' => [
|
|
'ngram_short' => [
|
|
'type' => 'ngram',
|
|
'min_gram' => '2',
|
|
'max_gram' => '5',
|
|
'token_chars' => [
|
|
'letter',
|
|
'digit',
|
|
],
|
|
],
|
|
],
|
|
],
|
|
'max_ngram_diff' => 10,
|
|
],
|
|
'mappings' => [
|
|
'properties' => $mappingSettings,
|
|
],
|
|
];
|
|
|
|
return $settings;
|
|
}
|
|
|
|
private function updateIndexSettings(string $type): void
|
|
{
|
|
$url = $this->getServerUrl().'/'.$this->getIndex($type);
|
|
|
|
if ($this->processResult($url.'/_settings', '', 'GET', 'indexExists')) {
|
|
$this->curlInitSession($url.'/_close', 'POST', '');
|
|
$this->curlInitSession($url.'/_settings', 'PUT', json_encode($this->returnIndexSettings($type)));
|
|
$this->curlInitSession($url.'/_open', 'POST', '');
|
|
}
|
|
}
|
|
|
|
public function updateSynonyms(?array $synonyms, bool $merge = false): void
|
|
{
|
|
if (!$synonyms) {
|
|
$synonyms = [];
|
|
}
|
|
|
|
$currentSynonyms = $this->loadSynonyms();
|
|
if ($merge) {
|
|
$synonyms = array_merge($currentSynonyms, $synonyms);
|
|
}
|
|
|
|
$this->saveSynonyms($synonyms);
|
|
|
|
foreach ($this->getIndexTypes() as $type) {
|
|
$this->updateIndexSettings($type);
|
|
}
|
|
}
|
|
|
|
public function saveSynonyms(array $synonyms): void
|
|
{
|
|
$dbcfg = \Settings::getDefault();
|
|
$dbcfg->saveValue('fulltext_synonyms', $synonyms, false);
|
|
}
|
|
|
|
public function loadSynonyms(): array
|
|
{
|
|
$dbcfg = \Settings::getDefault();
|
|
$synonyms = $dbcfg->loadValue('fulltext_synonyms');
|
|
$this->settings['synonyms'] = $synonyms ?? [];
|
|
|
|
return $this->settings['synonyms'];
|
|
}
|
|
|
|
public function loadSynonymsFromIndex(): ?array
|
|
{
|
|
$url = $this->getServerUrl().'/'.$this->getIndex(self::INDEX_PRODUCTS).'/_settings';
|
|
$url = str_replace('elasticsearch.wpj.cz:80', 'kupshop:kupshop@kibana.wpj.cz:9200', $url);
|
|
$result = $this->curlInitSession($url, 'GET', '');
|
|
|
|
$synonyms = reset($result)['settings']['index']['analysis']['filter']['synonymsFilter']['synonyms'] ?? null;
|
|
if (isset($synonyms)) {
|
|
$result = array_map(function ($val) {
|
|
if (empty($val[0])) {
|
|
return false;
|
|
}
|
|
$val = explode('=>', $val);
|
|
|
|
return ['from' => $val[0], 'to' => str_replace($val[0].',', '', $val[1])];
|
|
}, $synonyms);
|
|
} else {
|
|
$result = null;
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
/**
|
|
* Formats synonyms for use in Elastic index settings.
|
|
*
|
|
* @return string[] formatted synonyms
|
|
*/
|
|
protected function getFormattedSynonyms(?array $synonyms, $expand = true): array
|
|
{
|
|
if (empty($synonyms) || empty($synonyms[0]['from'])) {
|
|
return [''];
|
|
}
|
|
|
|
return array_map(function ($val) use ($expand) {
|
|
if (empty($val['from'])) {
|
|
return '';
|
|
}
|
|
|
|
$self = $expand ? "{$val['from']}," : '';
|
|
|
|
return "{$val['from']}=>{$self}{$val['to']}";
|
|
}, $synonyms);
|
|
}
|
|
|
|
public function search(string $term, array $config, ?array $types = null, $returnRaw = false): array
|
|
{
|
|
$queries = [];
|
|
|
|
$types ??= $this->getIndexTypes();
|
|
|
|
// prepare queries
|
|
foreach ($types as $type) {
|
|
$typeConfig = $config[$type] ?? [];
|
|
// header
|
|
$queries[] = json_encode(['index' => $this->getIndex($type)]);
|
|
// body
|
|
switch ($type) {
|
|
case self::INDEX_PRODUCTS:
|
|
[$order, $order_dir] = $this->getOrder($typeConfig['order'] ?? null);
|
|
$queries[] = $this->productsSearchParameter($term, $typeConfig['count'], $typeConfig['offset'] ?? 0, $order, $order_dir, $typeConfig['exact'] ?? false);
|
|
break;
|
|
default:
|
|
$queries[] = $this->createSearchParameter($term, $type, $typeConfig['count'], $typeConfig['offset'] ?? 0);
|
|
}
|
|
}
|
|
|
|
// execute queries
|
|
$searchResult = $this->multiSearch($queries);
|
|
|
|
$result = [];
|
|
$totals = [];
|
|
// process responses
|
|
foreach ($searchResult['responses'] ?? [] as $key => $item) {
|
|
$type = $types[$key] ?? null;
|
|
if ($type === null) {
|
|
throw new FulltextException('Multi-search invalid response!',
|
|
['type' => $type, 'item' => $item]);
|
|
}
|
|
|
|
if (!in_array($item['status'], [200, 400, 404])) { // 404 - index does not exists, 400 - index error
|
|
throw new FulltextException("Multi-search invalid response status: {$type}: {$item['status']} != 200",
|
|
['type' => $type, 'item' => $item]);
|
|
}
|
|
|
|
// TODO: Temporary HACK
|
|
$result[$type] = $this->processSearchResults($returnRaw ? null : $type, $item);
|
|
$totals[$type] = $item['hits']['total']['value'] ?? 0;
|
|
}
|
|
|
|
$this->multiSearchResultTotals = $totals;
|
|
|
|
return $result;
|
|
}
|
|
|
|
public function searchProducts($term, $count, $offset, $order = null, $filter = '', $exact = false)
|
|
{
|
|
[$order, $order_dir] = $this->getOrder($order);
|
|
|
|
$urlEnd = '_search?filter_path=hits.total,hits.hits._id';
|
|
|
|
return $this->executeCurl(self::INDEX_PRODUCTS, $this->productsSearchParameter($term, $count, $offset, $order, $order_dir, $exact), 'GET', $urlEnd);
|
|
}
|
|
|
|
public function searchProductsExact($term, $count, $offset, $order = null, $filter = '')
|
|
{
|
|
return $this->searchProducts($term, $count, $offset, $order, $filter, true);
|
|
}
|
|
|
|
private function processProductsSearch($result): array
|
|
{
|
|
$this->settings['rows'] = ($result['hits']['total']['value'] ?? 0);
|
|
|
|
return Mapping::mapKeys(
|
|
$result['hits']['hits'] ?? [],
|
|
function ($index, $row) {
|
|
return [$row['_id'], $row['_source'] ?? true];
|
|
},
|
|
);
|
|
}
|
|
|
|
private function processSectionsSearch($result): array
|
|
{
|
|
$sections = [];
|
|
foreach ($result['hits']['hits'] ?? [] as $line) {
|
|
$sections[] = [
|
|
'id' => $line['_id'],
|
|
'name' => $line['_source']['title'],
|
|
'path' => $line['_source']['path'],
|
|
'photo' => getImage($line['_id'], null, null, 'section', 1),
|
|
'photo_src' => $line['_source']['photo'] ?? null,
|
|
];
|
|
}
|
|
|
|
return $sections;
|
|
}
|
|
|
|
private function processProducersSearch($result): array
|
|
{
|
|
$producers = [];
|
|
foreach ($result['hits']['hits'] ?? [] as $line) {
|
|
$producers[] = [
|
|
'id' => $line['_id'],
|
|
'name' => $line['_source']['title'],
|
|
];
|
|
}
|
|
|
|
return $producers;
|
|
}
|
|
|
|
public function getRowsCount()
|
|
{
|
|
return $this->settings['rows'];
|
|
}
|
|
|
|
public function executeCurl($type, $param, $customRequest, $urlEnd)
|
|
{
|
|
$url = $this->getServerUrl().'/'.$this->getIndex($type).'/_doc/'.$urlEnd;
|
|
$result = $this->processResult($url, $param, $customRequest, $type);
|
|
|
|
return $result;
|
|
}
|
|
|
|
private function returnSearchBody($term, $fields): array
|
|
{
|
|
return [
|
|
'query' => [
|
|
'function_score' => [
|
|
'query' => [
|
|
'multi_match' => [
|
|
'query' => $term,
|
|
'type' => 'cross_fields',
|
|
'operator' => 'and',
|
|
'fields' => $fields,
|
|
],
|
|
],
|
|
],
|
|
],
|
|
];
|
|
}
|
|
|
|
private function processSuggestion($term)
|
|
{
|
|
$parameterSuggestion = [
|
|
'suggest' => [
|
|
'text' => $term,
|
|
'simple_phrase' => [
|
|
'phrase' => [
|
|
'field' => 'title',
|
|
'size' => 1,
|
|
'gram_size' => 3,
|
|
'direct_generator' => [
|
|
[
|
|
'field' => 'title',
|
|
'suggest_mode' => 'always',
|
|
],
|
|
],
|
|
],
|
|
],
|
|
],
|
|
];
|
|
$parameterSuggestion = json_encode($parameterSuggestion);
|
|
$url = $this->getServerUrl().'/'.$this->getIndex(self::INDEX_PRODUCTS).'/_search?&filter_path=suggest.simple_phrase.options';
|
|
|
|
return $this->processResult($url, $parameterSuggestion, 'GET', 'suggestion');
|
|
}
|
|
|
|
public function suggestTerm($term)
|
|
{
|
|
return $this->processSuggestion($term);
|
|
}
|
|
|
|
protected function createSearchParameter($term, $type, $count, $offset)
|
|
{
|
|
$fields = $this->formatQueryFields($this->getSearchFields($type));
|
|
|
|
$param = [
|
|
'size' => $count,
|
|
'from' => $offset,
|
|
'query' => $this->returnSearchBody($term, $fields)['query'],
|
|
];
|
|
|
|
return json_encode($param);
|
|
}
|
|
|
|
public function getSearchFields(string $type): array
|
|
{
|
|
return array_filter($this->{"fields_{$type}"}, fn ($item) => isset($item['weight']) && isset($item['type']));
|
|
}
|
|
|
|
public function getElasticFieldsDeps(string $type): array
|
|
{
|
|
return array_map(fn ($f) => $f['deps'] ?? [], $this->{"fields_{$type}"});
|
|
}
|
|
|
|
public function getElasticFieldsAllDeps(string $type): array
|
|
{
|
|
return array_unique(array_reduce($this->getElasticFieldsDeps($type), fn ($allDeps, $fieldDeps) => array_merge($allDeps, $fieldDeps), []));
|
|
}
|
|
|
|
public function formatQueryFields(array $fields, ?array $types = null): array
|
|
{
|
|
return array_values(array_filter(Mapping::withKeys($fields, function ($key, $value) use ($types) {
|
|
if (is_null($types) || in_array($value['type'], $types)) {
|
|
return "{$key}^{$value['weight']}";
|
|
}
|
|
|
|
return null;
|
|
})));
|
|
}
|
|
|
|
public function productsSearchParameter($term, $count, $offset, $order, $order_dir, $exact = false)
|
|
{
|
|
$fields = $this->getSearchFields(self::INDEX_PRODUCTS);
|
|
|
|
$multiMatches = [];
|
|
|
|
foreach (['text', 'keyword', 'no_spaces'] as $fieldType) {
|
|
if (!($matchFields = $this->formatQueryFields($fields, [$fieldType]))) {
|
|
continue;
|
|
}
|
|
|
|
$multiMatches[] = [
|
|
'multi_match' => [
|
|
'query' => $term,
|
|
'fields' => $matchFields,
|
|
'type' => 'cross_fields',
|
|
'operator' => 'and',
|
|
],
|
|
];
|
|
}
|
|
|
|
$query = [
|
|
'function_score' => [
|
|
'query' => [
|
|
'bool' => [
|
|
'should' => $multiMatches,
|
|
],
|
|
],
|
|
'field_value_factor' => [
|
|
'field' => 'asfnboadnb',
|
|
'missing' => 1,
|
|
],
|
|
'boost_mode' => 'max',
|
|
],
|
|
];
|
|
if ($exact === true) {
|
|
// returns exact matches
|
|
$query = [
|
|
'match_phrase' => [
|
|
'title' => [
|
|
'query' => $term,
|
|
'analyzer' => self::SEARCH_ANALYZER_NAME,
|
|
],
|
|
],
|
|
];
|
|
}
|
|
|
|
switch ($order) {
|
|
case 'weight':
|
|
$param = [
|
|
'size' => $count,
|
|
'from' => $offset,
|
|
'query' => [
|
|
'function_score' => [
|
|
'query' => $query,
|
|
'field_value_factor' => [
|
|
'field' => 'weight',
|
|
],
|
|
],
|
|
],
|
|
];
|
|
break;
|
|
default:
|
|
$param = [
|
|
'size' => $count,
|
|
'from' => $offset,
|
|
'sort' => [
|
|
[
|
|
$order => [
|
|
'order' => $order_dir,
|
|
],
|
|
],
|
|
],
|
|
'query' => [
|
|
'function_score' => [
|
|
'query' => $query,
|
|
'field_value_factor' => [
|
|
'field' => 'weight',
|
|
],
|
|
],
|
|
],
|
|
];
|
|
break;
|
|
}
|
|
|
|
return json_encode($param);
|
|
}
|
|
|
|
public function processResult($url, $param, $customRequest, $category)
|
|
{
|
|
$result = $this->curlInitSession($url, $customRequest, $param);
|
|
|
|
if ($customRequest == 'GET' && !empty($result['hits']['hits'])) {
|
|
return $this->processSearchResults($category, $result);
|
|
}
|
|
if ($customRequest == 'GET' && $category == 'indexExists') {
|
|
if (key_exists('error', $result)) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
if ($category == 'suggestion' && key_exists('suggest', $result) && !empty($result['suggest']['simple_phrase'][0]['options'][0])) {
|
|
return $result['suggest']['simple_phrase'][0]['options'][0]['text'];
|
|
} else {
|
|
$this->settings['rows'] = 0;
|
|
|
|
return $result = [];
|
|
}
|
|
}
|
|
|
|
public function deleteProducts($products)
|
|
{
|
|
foreach ($products as $product) {
|
|
$this->executeCurl(self::INDEX_PRODUCTS, '', 'DELETE', $product['id']);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param int|array $id_product
|
|
*
|
|
* @return bool
|
|
*/
|
|
public function updateProduct($id_product)
|
|
{
|
|
/* Ugly hack - increase memory limit by 1GB */
|
|
increaseMemoryLimit(1024);
|
|
|
|
$id_product = is_array($id_product) ? $id_product : [$id_product];
|
|
|
|
$this->updateProductFulltext($id_product);
|
|
$this->updateProductsAttributes($id_product);
|
|
|
|
return true;
|
|
}
|
|
|
|
public function updateProductFulltext($id_products)
|
|
{
|
|
if (!is_array($id_products)) {
|
|
$id_products = [$id_products];
|
|
}
|
|
|
|
[$products, $delete] = $this->elasticDataFetchUtil->fetchProducts($id_products);
|
|
|
|
// Update
|
|
if ($products) {
|
|
$this->bulkUpdate($products, self::INDEX_PRODUCTS, 'put');
|
|
}
|
|
|
|
// Delete missing products from index
|
|
if ($delete) {
|
|
$this->bulkUpdate($delete, self::INDEX_PRODUCTS, 'delete');
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param array $productsChangesMap mapping of products changed values in format [idProduct => ['title', etc...], ...]
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
public function partialProductsUpdate(array $productsChangesMap): void
|
|
{
|
|
$searchFieldsDeps = $this->getElasticFieldsDeps(self::INDEX_PRODUCTS);
|
|
|
|
$this->partialUpdateInBatches($productsChangesMap, function ($productsChangesMapChunk) use ($searchFieldsDeps) {
|
|
[$products, $delete] = $this->elasticDataFetchUtil->fetchProducts($productsChangesMapChunk, true);
|
|
|
|
foreach ($products as &$product) {
|
|
$productChanges = $productsChangesMapChunk[$product['id']] ?? false;
|
|
if (empty($productChanges)) {
|
|
continue;
|
|
}
|
|
|
|
$product = array_filter($product, function ($key) use ($searchFieldsDeps, $productChanges) {
|
|
if ($key == 'id') {
|
|
return true;
|
|
}
|
|
$fieldDeps = $searchFieldsDeps[$key] ?? false;
|
|
|
|
return $fieldDeps && !empty(array_intersect($fieldDeps, $productChanges));
|
|
}, ARRAY_FILTER_USE_KEY);
|
|
}
|
|
|
|
if ($products) {
|
|
$this->bulkUpdate($products, self::INDEX_PRODUCTS, 'update_partial');
|
|
}
|
|
|
|
if ($delete) {
|
|
$this->bulkUpdate($delete, self::INDEX_PRODUCTS, 'delete');
|
|
}
|
|
});
|
|
}
|
|
|
|
public function bulkUpdate($values, $type, $method)
|
|
{
|
|
$finalParam = [];
|
|
switch ($method) {
|
|
case 'put':
|
|
foreach ($values as $line) {
|
|
$finalParam[] = [
|
|
'index' => [
|
|
'_index' => $this->getIndex($type),
|
|
'_type' => '_doc',
|
|
'_id' => $line['id'],
|
|
],
|
|
];
|
|
$finalParam[] = $line;
|
|
}
|
|
|
|
break;
|
|
case 'update':
|
|
foreach ($values as $line) {
|
|
$finalParam[] = [
|
|
'update' => [
|
|
'_id' => $line['id'],
|
|
'_type' => '_doc',
|
|
'_index' => $this->getIndex($type),
|
|
],
|
|
];
|
|
$finalParam[] = [
|
|
'doc' => [
|
|
'weight' => $line['weight'],
|
|
'sold' => intval($line['sold']),
|
|
'delivery' => intval($line['delivery']),
|
|
],
|
|
'upsert' => new \stdClass(),
|
|
];
|
|
}
|
|
break;
|
|
|
|
case 'update_partial':
|
|
foreach ($values as $line) {
|
|
$finalParam[] = [
|
|
'update' => [
|
|
'_id' => $line['id'],
|
|
'_type' => '_doc',
|
|
'_index' => $this->getIndex($type),
|
|
],
|
|
];
|
|
$finalParam[] = [
|
|
'doc' => $line,
|
|
];
|
|
}
|
|
break;
|
|
case 'delete':
|
|
foreach ($values as $line) {
|
|
$finalParam[] = [
|
|
'delete' => [
|
|
'_index' => $this->getIndex($type),
|
|
'_type' => '_doc',
|
|
'_id' => $line['id'],
|
|
],
|
|
];
|
|
}
|
|
break;
|
|
}
|
|
$url = $this->getServerUrl().'/'.$this->getIndex($type).'/_bulk';
|
|
$data = join(
|
|
"\n",
|
|
array_map(
|
|
function ($x) {
|
|
return json_encode($x);
|
|
},
|
|
$finalParam
|
|
)
|
|
);
|
|
|
|
try {
|
|
$response = $this->curlInitSession($url, 'POST', $data."\n");
|
|
|
|
$errors = isset($response['errors']) && $response['errors'];
|
|
|
|
if ($errors && $method === 'update_partial') {
|
|
// ignore when update of non-existing product is attempted
|
|
$errors = !empty(array_filter($response['items'] ?? [],
|
|
fn ($item) => empty($item['update']) || !in_array($item['update']['status'] ?? '', [404, 200])));
|
|
}
|
|
|
|
if ($errors) {
|
|
throw new FulltextException('Failed to update fulltext!', [
|
|
'response' => $response,
|
|
]);
|
|
}
|
|
} catch (FulltextException|\JsonException $e) {
|
|
$this->sentryLogger->captureException($e);
|
|
|
|
if (isDevelopment()) {
|
|
throw $e;
|
|
}
|
|
}
|
|
}
|
|
|
|
public function curlInitSession($url, $customRequest, $param)
|
|
{
|
|
$header = ['content-type: application/json; charset=UTF-8'];
|
|
$ch = curl_init();
|
|
|
|
curl_setopt($ch, CURLOPT_URL, $url);
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
if (!empty($param)) {
|
|
curl_setopt($ch, CURLOPT_POSTFIELDS, $param);
|
|
}
|
|
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
|
|
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $customRequest);
|
|
curl_setopt($ch, CURLOPT_POST, true);
|
|
curl_setopt($ch, CURLOPT_TIMEOUT, $this->curlTimeout);
|
|
$result = curl_exec($ch);
|
|
if ($result === false) {
|
|
throw new FulltextException('Elasticsearch server vrátil neplatnou odpověď!',
|
|
['server' => $this->getServerUrl(), 'curl_error' => curl_error($ch), 'curl_errno' => curl_errno($ch)]);
|
|
}
|
|
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
if ($http_code == 413) {
|
|
// 413 Request Entity Too Large
|
|
throw new FulltextException('Elasticsearch server vrátil neplatnou odpověď! 413 Request Entity Too Large',
|
|
['server' => $this->getServerUrl(), 'data' => $param, 'result' => $result]);
|
|
}
|
|
$result = json_decode_strict($result, true);
|
|
curl_close($ch);
|
|
|
|
return $result;
|
|
}
|
|
|
|
private function updatePages(): void
|
|
{
|
|
$this->withNewIndex(self::INDEX_PAGES, function () {
|
|
foreach ($this->genericObjectsIndex->getBatchedObjects() as $batch) {
|
|
$this->bulkUpdate($batch, self::INDEX_PAGES, 'put');
|
|
}
|
|
});
|
|
}
|
|
|
|
private function updateArticles()
|
|
{
|
|
if (!findModule(\Modules::ARTICLES)) {
|
|
return;
|
|
}
|
|
|
|
$query = sqlQueryBuilder()->select('id')->from('articles');
|
|
|
|
$this->withNewIndex(self::INDEX_ARTICLES,
|
|
function () use ($query) {
|
|
$this->updateInBatches(
|
|
$query->execute(),
|
|
function ($chunk) {
|
|
$this->updateArticle($chunk);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
|
|
public function updateArticle($id_article)
|
|
{
|
|
if (!is_array($id_article)) {
|
|
$id_article = [$id_article];
|
|
}
|
|
|
|
$specs = [
|
|
Operator::inIntArray($id_article, 'a.id'),
|
|
Operator::equals(['a.show_in_search' => 'Y']),
|
|
function (QueryBuilder $qb) {
|
|
$qb->addSelect('GROUP_CONCAT(b.content SEPARATOR "") as content')
|
|
->leftJoin('a', 'blocks', 'b', 'b.id_root = a.id_block');
|
|
},
|
|
];
|
|
|
|
$photoTypeId = findModule(\Modules::COMPONENTS) ? 11 : 12;
|
|
|
|
$cfg = Config::get();
|
|
if (isset($cfg['Photo']['id_to_type'][$photoTypeId])) {
|
|
$this->articleList->setImage($cfg['Photo']['id_to_type'][$photoTypeId]);
|
|
}
|
|
|
|
$articles = $this->articleList
|
|
->getArticles(Operator::andX($specs));
|
|
|
|
$bulkValues = [];
|
|
foreach ($articles as $article) {
|
|
$tags = array_map(function ($x) { return $x['tag']; }, $article['tags'] ?? []);
|
|
|
|
$bulkValues[] = [
|
|
'id' => $article['id'],
|
|
'title' => $article['title'],
|
|
'keywords' => $article['keywords'] ?? '',
|
|
'lead_in' => $article['lead_in'],
|
|
'section' => $article['section']['name'] ?? '',
|
|
'tags' => join(',', $tags),
|
|
'content' => StringUtil::htmlToCleanText($article['content'] ?? ''),
|
|
'photo' => $article['image']['src'] ?? null,
|
|
];
|
|
}
|
|
|
|
$this->bulkUpdate($bulkValues, self::INDEX_ARTICLES, 'put');
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Generates new index with products.
|
|
*
|
|
* @return void
|
|
*
|
|
* @throws FulltextException
|
|
*/
|
|
private function updateProducts()
|
|
{
|
|
increaseMemoryLimit(1024);
|
|
|
|
$query = $this->createProductQueryBuilder();
|
|
|
|
$this->withNewIndex(self::INDEX_PRODUCTS,
|
|
function () use ($query) {
|
|
$this->updateInBatches(
|
|
$query->execute(),
|
|
function ($chunk) {
|
|
$this->updateProductFulltext($chunk);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
|
|
public function updateInBatches($rows, $callback)
|
|
{
|
|
foreach (array_chunk($rows->fetchAll(), 100) as $chunk) {
|
|
try {
|
|
$callback(
|
|
array_map(
|
|
function ($x) {
|
|
return $x['id'];
|
|
},
|
|
$chunk
|
|
)
|
|
);
|
|
} catch (FulltextException $e) {
|
|
$this->sentryLogger->captureException($e, ['extra' => $e->getData()]);
|
|
|
|
if (isDevelopment()) {
|
|
throw $e;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
protected function partialUpdateInBatches($items, $callback)
|
|
{
|
|
foreach (array_chunk($items, 100, true) as $chunk) {
|
|
try {
|
|
$callback($chunk);
|
|
} catch (FulltextException $e) {
|
|
$this->sentryLogger->captureException(new \Exception('Kafka - Elastic index update failed (chunk)', 0, $e), ['extra' => [
|
|
'chunk' => $chunk,
|
|
]]);
|
|
|
|
if (isDevelopment()) {
|
|
throw $e;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public function updateSection($id_section)
|
|
{
|
|
if (!is_array($id_section)) {
|
|
$id_section = [$id_section];
|
|
}
|
|
|
|
$dbcfg = \Settings::getDefault();
|
|
|
|
// Texts
|
|
$SQL = sqlQueryBuilder()->select('s.id, s.name, CONCAT_WS(" ", GROUP_CONCAT(DISTINCT b.content ORDER BY b.position ASC SEPARATOR "") , GROUP_CONCAT(DISTINCT pr.name)) AS description')
|
|
->from('sections', 's')
|
|
->leftJoin('s', 'products_in_sections', 'pis', 'pis.id_section = s.id')
|
|
->leftJoin('s', 'blocks', 'b', 'b.id_root = s.id_block')
|
|
->andWhere(Operator::inIntArray($id_section, 's.id'))
|
|
->groupBy('s.id');
|
|
|
|
if ($dbcfg->cat_show_empty == 'N') {
|
|
$productList = new \ProductList();
|
|
$fp = $productList->applyDefaultFilterParams();
|
|
$fp->setSections($id_section);
|
|
$productList->andSpec(function (QueryBuilder $qb) {
|
|
$qb->select('p.id');
|
|
});
|
|
|
|
$productListQB = $productList->getQueryBuilder();
|
|
$SQL->leftjoin('pis', 'products', 'p', 'p.id = pis.id_product AND p.id IN ('.$productListQB->getSQL().')')
|
|
->addQueryBuilderParameters($productListQB);
|
|
$SQL->andWhere("p.figure = 'Y' OR s.virtual = 'Y'");
|
|
} else {
|
|
$SQL->leftJoin('pis', 'products', 'p', 'p.id = pis.id_product');
|
|
}
|
|
$SQL->leftJoin('p', 'producers', 'pr', 'p.producer = pr.id');
|
|
|
|
// index only visible sections
|
|
$SQL->andWhere(
|
|
Translation::joinTranslatedFields(
|
|
SectionsTranslation::class,
|
|
function (QueryBuilder $qb, $columnName, $translatedField) {
|
|
if ($columnName == 'figure') {
|
|
$qb->andWhere(Operator::coalesce($translatedField, 's.figure')." != 'N' AND s.show_in_search = 'Y'");
|
|
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
},
|
|
['name', 'figure']
|
|
)
|
|
);
|
|
|
|
$results = $SQL->execute()->fetchAll();
|
|
|
|
$bulkValues = [];
|
|
foreach ($results as $values) {
|
|
$values['title'] = $values['name'];
|
|
|
|
// Path
|
|
$navArray = getReturnNavigation($values['id']);
|
|
$navArray = reset($navArray);
|
|
|
|
if ($navArray) {
|
|
array_shift($navArray);
|
|
$values['path'] = join(
|
|
' / ',
|
|
array_map(
|
|
function ($x) {
|
|
return $x['text'];
|
|
},
|
|
$navArray
|
|
)
|
|
);
|
|
$values['path_str'] = $values['path'];
|
|
}
|
|
|
|
$values['title_str'] = $values['title'];
|
|
|
|
try {
|
|
// v try-catch, protoze nechci, aby to behem reindexace zuchlo treba na selhani nacteni cache
|
|
$section = $this->sectionTree->getSectionById($values['id']);
|
|
} catch (\Throwable $e) {
|
|
$section = null;
|
|
}
|
|
|
|
$bulkValues[] = [
|
|
'id' => $values['id'],
|
|
'title' => $values['title'],
|
|
'description' => $values['description'],
|
|
'path' => $values['path'] ?? '',
|
|
'photo' => $section ? $section->getPhoto()['src'] ?? null : null,
|
|
];
|
|
}
|
|
|
|
// Update
|
|
$this->bulkUpdate($bulkValues, self::INDEX_SECTIONS, 'put');
|
|
|
|
return true;
|
|
}
|
|
|
|
private function updateSections()
|
|
{
|
|
$query = sqlQueryBuilder()
|
|
->select('id')
|
|
->from('sections')
|
|
->where('id > 0');
|
|
|
|
$this->withNewIndex(self::INDEX_SECTIONS,
|
|
function () use ($query) {
|
|
$this->updateInBatches(
|
|
$query->execute(),
|
|
function ($chunk) {
|
|
$this->updateSection($chunk);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
|
|
public function updateProducer($id_producer)
|
|
{
|
|
$dbcfg = \Settings::getDefault();
|
|
|
|
// Texts
|
|
$SQL = sqlQueryBuilder()->select('pr.id, pr.name, 1 AS visible,
|
|
CONCAT_WS(" ", GROUP_CONCAT(DISTINCT b.content ORDER BY b.position ASC SEPARATOR ""), GROUP_CONCAT(DISTINCT s.name)) AS long_description')
|
|
->from('producers', 'pr')
|
|
->leftJoin('pr', 'blocks', 'b', 'b.id_root = pr.id_block')
|
|
->leftJoin('p', 'products_in_sections', 'pis', 'pis.id_product = p.id')
|
|
->leftJoin('pis', 'sections', 's', 's.id = pis.id_section')
|
|
->where('pr.id = :id_producer AND pr.active = :active')
|
|
->setParameters(
|
|
[
|
|
'id_producer' => $id_producer,
|
|
'active' => 'Y',
|
|
]
|
|
)->groupBy('pr.id');
|
|
|
|
if ($dbcfg->cat_show_empty == 'N') {
|
|
$productList = new \ProductList();
|
|
$fp = $productList->applyDefaultFilterParams();
|
|
$fp->setProducers([$id_producer]);
|
|
$productList->andSpec(function (QueryBuilder $qb) {
|
|
$qb->select('p.id');
|
|
});
|
|
|
|
$productListQB = $productList->getQueryBuilder();
|
|
$SQL->join('pr', 'products', 'p', 'p.id IN ('.$productListQB->getSQL().')')
|
|
->addQueryBuilderParameters($productListQB);
|
|
} else {
|
|
$SQL->leftJoin('pr', 'products', 'p', 'pr.id = p.producer');
|
|
}
|
|
|
|
if (findModule(\Modules::TRANSLATIONS)) {
|
|
$SQL->andWhere(
|
|
\Query\Translation::coalesceTranslatedFields(
|
|
\KupShop\I18nBundle\Translations\ProducersTranslation::class
|
|
)
|
|
);
|
|
}
|
|
|
|
$values = $SQL->execute()->fetch();
|
|
|
|
if (!$values) {
|
|
return false;
|
|
}
|
|
|
|
$values['title'] = $values['name'];
|
|
$values['description'] = $values['long_description'];
|
|
|
|
// Update
|
|
$this->executeCurl(self::INDEX_PRODUCERS, $this->createUpdateParameter('producers', $values), 'PUT', $id_producer);
|
|
|
|
return true;
|
|
}
|
|
|
|
private function updateProducers()
|
|
{
|
|
if (!findModule(\Modules::PRODUCERS)) {
|
|
return;
|
|
}
|
|
|
|
$this->withNewIndex(self::INDEX_PRODUCERS,
|
|
function () {
|
|
foreach (sqlQuery('SELECT id FROM producers') as $producer) {
|
|
$this->updateProducer($producer['id']);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
public function createUpdateParameter($category, $values)
|
|
{
|
|
$param = [];
|
|
switch ($category) {
|
|
case 'sections':
|
|
if (array_key_exists('path', $values)) {
|
|
$param = [
|
|
'title' => $values['title'],
|
|
'description' => $values['description'],
|
|
'path' => $values['path'],
|
|
];
|
|
}
|
|
break;
|
|
case 'producers':
|
|
$param = [
|
|
'title' => $values['title'],
|
|
'description' => $values['description'],
|
|
];
|
|
break;
|
|
}
|
|
|
|
return json_encode($param);
|
|
}
|
|
|
|
public function updateProductsAttributes($id_product = null)
|
|
{
|
|
$sql = $this->createProductQueryBuilder()
|
|
->addSelect('p.position');
|
|
|
|
if ($id_product) {
|
|
if (is_array($id_product)) {
|
|
$sql->andWhere(Operator::inIntArray($id_product, 'p.id'));
|
|
} else {
|
|
$sql->andWhere('p.id = :id_product')->setParameter('id_product', $id_product);
|
|
}
|
|
}
|
|
|
|
if (findModule('products_suppliers')) {
|
|
$sql->joinProductsOfSuppliers()
|
|
->addSelect('SUM(IF(pos.in_store > 0, pos.in_store, 0)) as in_store_suppliers');
|
|
}
|
|
|
|
$updatedValues = [];
|
|
foreach ($sql->execute() as $row) {
|
|
$product = new \Product();
|
|
$product->createFromArray($row);
|
|
$product->prepareDeliveryText();
|
|
|
|
$values = $this->elasticDataFetchUtil->getProductsAttributesValues($product);
|
|
|
|
$updatedValues[] = array_merge($values, ['id' => $product->id]);
|
|
|
|
if (count($updatedValues) > 99) {
|
|
$this->bulkUpdate($updatedValues, self::INDEX_PRODUCTS, 'update');
|
|
unset($updatedValues);
|
|
$updatedValues = [];
|
|
}
|
|
}
|
|
|
|
if (!empty($updatedValues)) {
|
|
$this->bulkUpdate($updatedValues, self::INDEX_PRODUCTS, 'update');
|
|
}
|
|
}
|
|
|
|
public function multiSearch(array $queries)
|
|
{
|
|
return $this->curlInitSession(
|
|
$this->getServerUrl().'/_msearch',
|
|
'GET',
|
|
implode("\n", $queries)."\n"
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Adds hunspell filter settings into provided $filters array; or return hunspell filter settings.
|
|
*/
|
|
protected function getHunspellFilter(array $filters = []): array
|
|
{
|
|
$locale = null;
|
|
switch ($this->languageContext->getActiveId()) {
|
|
case 'cs':
|
|
$locale = 'cs_CZ';
|
|
break;
|
|
case 'sk':
|
|
$locale = 'sk_SK';
|
|
break;
|
|
}
|
|
|
|
if ($locale) {
|
|
$filters['hunspell'] = [
|
|
'type' => 'hunspell',
|
|
'locale' => $locale,
|
|
];
|
|
}
|
|
|
|
$filters['custom_stemmer'] = [
|
|
'type' => 'multiplexer',
|
|
'filters' => $locale ? ['hunspell'] : [],
|
|
];
|
|
|
|
return $filters;
|
|
}
|
|
|
|
protected function createProductQueryBuilder()
|
|
{
|
|
// Nepředefinovávaj mě, předefinuj getProductsSpec, která se použije na více místech než já
|
|
$qb = sqlQueryBuilder()->select('p.id, p.title, p.discount, p.vat, p.delivery_time, p.pieces_sold, p.code, p.in_store')
|
|
->fromProducts()
|
|
->joinVariationsOnProducts()
|
|
->andWhere($this->getProductsSpec())
|
|
->andWhere('p.id > 0')
|
|
->groupBy('p.id');
|
|
|
|
return $qb;
|
|
}
|
|
|
|
/**
|
|
* @return \FilterParams
|
|
*/
|
|
protected function getFilterParams()
|
|
{
|
|
if ($this->filterParams) {
|
|
return $this->filterParams;
|
|
}
|
|
|
|
return $this->filterParams = new \FilterParams();
|
|
}
|
|
|
|
public function getIndex(string $type, ?bool $useNextIndex = null)
|
|
{
|
|
return $this->getIndexPrefix($useNextIndex).'.'.$type;
|
|
}
|
|
|
|
public function getIndexPrefix(?bool $useNextIndex = null, bool $appendLanguage = true): string
|
|
{
|
|
if ($useNextIndex === null) {
|
|
$useNextIndex = $this->_useNextIndex;
|
|
}
|
|
$alias = $useNextIndex ? self::INDEX_ALIAS_NEXT : self::INDEX_ALIAS_CURRENT;
|
|
|
|
if ($appendLanguage && findModule(\Modules::TRANSLATIONS)) {
|
|
$alias .= '_'.$this->languageContext->getActiveId();
|
|
}
|
|
|
|
return $this->settings['index'].'_'.$alias;
|
|
}
|
|
|
|
private function getOrder(?string $order): array
|
|
{
|
|
$order = $order ?: '-weight';
|
|
$order_dir = 'asc';
|
|
|
|
if (substr($order, 0, 1) == '-') {
|
|
$order_dir = 'desc';
|
|
$order = substr($order, 1);
|
|
}
|
|
|
|
return [$order, $order_dir];
|
|
}
|
|
|
|
public function getServerUrl(): string
|
|
{
|
|
return getenv('ELASTIC_SERVER') ?: 'elasticsearch.wpj.cz:80';
|
|
}
|
|
|
|
public function updateIndex(string $type = 'all', $clean = true, array $exceptTypes = []): void
|
|
{
|
|
if ($clean) {
|
|
$this->clearIndices();
|
|
}
|
|
|
|
switch ($type) {
|
|
case self::INDEX_PRODUCTS:
|
|
$this->updateProducts();
|
|
$this->updateProductsAttributes();
|
|
break;
|
|
|
|
case 'all':
|
|
foreach ($this->getIndexTypes() as $iType) {
|
|
if (!in_array($iType, $exceptTypes)) {
|
|
$this->updateIndex($iType, false);
|
|
}
|
|
}
|
|
break;
|
|
|
|
default:
|
|
$titleType = ucfirst($type);
|
|
$this->{"update{$titleType}"}();
|
|
}
|
|
|
|
if ($clean) {
|
|
$this->clearIndices();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return array<string, Language>
|
|
*/
|
|
public function getFulltextLanguages(): array
|
|
{
|
|
$languages = [];
|
|
$languageContext = Contexts::get(LanguageContext::class);
|
|
|
|
if (!findModule(\Modules::TRANSLATIONS)) {
|
|
if ($default = $languageContext->getAll()[$languageContext->getDefaultId()] ?? null) {
|
|
$languages[$default->getId()] = $default;
|
|
}
|
|
|
|
return $languages;
|
|
}
|
|
|
|
foreach ($languageContext->getAll() as $language) {
|
|
if (!$language->isActive()) {
|
|
continue;
|
|
}
|
|
|
|
$languages[$language->getId()] = $language;
|
|
}
|
|
|
|
return $languages;
|
|
}
|
|
|
|
protected function processSearchResults($type, $result)
|
|
{
|
|
switch ($type) {
|
|
case static::INDEX_PRODUCTS:
|
|
return $this->processProductsSearch($result);
|
|
|
|
case self::INDEX_SECTIONS:
|
|
return $this->processSectionsSearch($result);
|
|
|
|
case self::INDEX_PRODUCERS:
|
|
return $this->processProducersSearch($result);
|
|
|
|
default:
|
|
// Combine source with id
|
|
$results = $result['hits']['hits'] ?? [];
|
|
|
|
return array_map(function ($x) {return ['id' => $x['_id']] + $x['_source']; }, $results);
|
|
}
|
|
}
|
|
|
|
public function getProductsSpec(): callable
|
|
{
|
|
return $this->getFilterParams()->getSpec();
|
|
}
|
|
|
|
public function getFilters(): array
|
|
{
|
|
return [];
|
|
}
|
|
|
|
public function setDynamicFilters(array $filters): void
|
|
{
|
|
}
|
|
|
|
#[Required]
|
|
final public function setGenericObjectsIndex(GenericObjectsIndex $genericObjectsIndex): void
|
|
{
|
|
$this->genericObjectsIndex = $genericObjectsIndex;
|
|
$this->fields_pages = $genericObjectsIndex->getFields();
|
|
}
|
|
|
|
public function supportsFilters(): bool
|
|
{
|
|
return false;
|
|
}
|
|
|
|
public static function isServerAvailable()
|
|
{
|
|
return !getCache(Client::ELASTIC_UNAVAILABLE_KEY);
|
|
}
|
|
}
|