329 lines
12 KiB
PHP
329 lines
12 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace KupShop\KafkaBundle\DBChanges\MessageHandlers;
|
|
|
|
use Elastica\Query\BoolQuery;
|
|
use KupShop\AdminBundle\FulltextUpdatesTrait;
|
|
use KupShop\CatalogBundle\Search\ElasticUpdateGroup;
|
|
use KupShop\CatalogBundle\Search\FulltextElastic;
|
|
use KupShop\CatalogElasticBundle\Elastica\Query;
|
|
use KupShop\CatalogElasticBundle\Elastica\QueryBuilder;
|
|
use KupShop\CatalogElasticBundle\Util\ElasticaFactory;
|
|
use KupShop\KafkaBundle\DBChanges\DBChangeTypeEnum;
|
|
use KupShop\KafkaBundle\DBChanges\MessageTypes\DBGenericMessage;
|
|
use KupShop\KafkaBundle\DBChanges\MessageTypes\DBMsgProduct;
|
|
use KupShop\KupShopBundle\Context\LanguageContext;
|
|
use KupShop\KupShopBundle\Util\Logging\SentryLogger;
|
|
use Symfony\Contracts\Service\Attribute\Required;
|
|
|
|
class DBMessageElasticHandler extends DBMessageAbstractHandler
|
|
{
|
|
use QueryBuilder;
|
|
use FulltextUpdatesTrait;
|
|
#[Required]
|
|
public FulltextElastic $fulltextElastic;
|
|
#[Required]
|
|
public LanguageContext $languageContext;
|
|
#[Required]
|
|
public SentryLogger $sentryLogger;
|
|
|
|
protected ?ElasticaFactory $searchClientFactory = null;
|
|
|
|
private array $updatedProducts = [];
|
|
private array $insertedProducts = [];
|
|
private array $deletedProducts = [];
|
|
|
|
private array $log = [];
|
|
private $startTime;
|
|
|
|
#[Required]
|
|
final public function setSearchClientFactory(?ElasticaFactory $searchClientFactory): void
|
|
{
|
|
$this->searchClientFactory = $searchClientFactory;
|
|
}
|
|
|
|
protected function handleProducts(DBMsgProduct $message): void
|
|
{
|
|
$productId = $message->getId();
|
|
|
|
switch ($message->getEventType()) {
|
|
case DBChangeTypeEnum::INSERT:
|
|
$this->addInsertedProduct($productId);
|
|
break;
|
|
case DBChangeTypeEnum::DELETE:
|
|
$this->addDeletedProduct($productId);
|
|
break;
|
|
case DBChangeTypeEnum::UPDATE:
|
|
if ($message->hasValueChanged('figure')) {
|
|
$this->log['figureUpdateCount'] = ($this->log['figureUpdateCount'] ?? 0) + 1;
|
|
$this->log['figureUpdateIds'] = array_merge($this->log['figureUpdateIds'] ?? [], [$productId]);
|
|
$this->addInsertedProduct($productId);
|
|
break;
|
|
}
|
|
|
|
$changedFields = $message->getChangedValuesKeys();
|
|
// update in_store only if changed from zero to non-zero or via versa
|
|
if ($message->hasValueBoolChanged('in_store')) {
|
|
$changedFields[] = ElasticUpdateGroup::InStore;
|
|
$changedFields[] = ElasticUpdateGroup::DeliveryTime;
|
|
}
|
|
|
|
$this->addUpdatedProduct($productId, $changedFields);
|
|
break;
|
|
}
|
|
}
|
|
|
|
protected function handleParametersProducts(DBGenericMessage $message): void
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [
|
|
ElasticUpdateGroup::Parameters,
|
|
]);
|
|
}
|
|
|
|
protected function handleProductsInSections(DBGenericMessage $message): void
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [
|
|
ElasticUpdateGroup::Sections,
|
|
]);
|
|
}
|
|
|
|
public function handleProductsVariations(DBGenericMessage $message)
|
|
{
|
|
$changedFields = [];
|
|
|
|
if ($message->hasValueBoolChanged('in_store') || $message->hasValueChanged('delivery_time')) {
|
|
$changedFields[] = ElasticUpdateGroup::InStore;
|
|
$changedFields[] = ElasticUpdateGroup::DeliveryTime;
|
|
}
|
|
|
|
if ($message->startsWithChanged('price')) {
|
|
$changedFields[] = ElasticUpdateGroup::Price;
|
|
}
|
|
|
|
if ($message->hasValueChanged('ean')) {
|
|
$changedFields[] = ElasticUpdateGroup::Ean;
|
|
}
|
|
if ($message->hasValueChanged('code')) {
|
|
$changedFields[] = ElasticUpdateGroup::Code;
|
|
}
|
|
|
|
if ($message->didAnyOfFieldChange(['title'])) {
|
|
$changedFields[] = ElasticUpdateGroup::Variations;
|
|
}
|
|
|
|
$this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
|
|
}
|
|
|
|
public function handleProductsVariationsChoicesCategorization(DBGenericMessage $message)
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [
|
|
ElasticUpdateGroup::Variations,
|
|
]);
|
|
}
|
|
|
|
public function handleProductLabelsRelation(DBGenericMessage $message)
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [
|
|
ElasticUpdateGroup::Labels,
|
|
]);
|
|
}
|
|
|
|
public function handlePriceLevelsProducts(DBGenericMessage $message)
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [
|
|
ElasticUpdateGroup::Price,
|
|
]);
|
|
}
|
|
|
|
public function handleProductsOfSuppliers(DBGenericMessage $message)
|
|
{
|
|
$changedFields = [];
|
|
|
|
if ($message->hasValueBoolChanged('in_store')) {
|
|
$changedFields[] = ElasticUpdateGroup::InStore;
|
|
$changedFields[] = ElasticUpdateGroup::DeliveryTime;
|
|
}
|
|
|
|
$this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
|
|
}
|
|
|
|
public function handlePhotosProductsRelation(DBGenericMessage $message)
|
|
{
|
|
$this->addUpdatedProduct($message->getValue('id_product'), [ElasticUpdateGroup::Photo]);
|
|
}
|
|
|
|
public function handleStoresItems(DBGenericMessage $message)
|
|
{
|
|
$changedFields = [];
|
|
|
|
if ($message->hasValueBoolChanged('quantity')) {
|
|
$changedFields[] = ElasticUpdateGroup::StoresInStore;
|
|
}
|
|
|
|
$this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
|
|
}
|
|
|
|
public function handleProductsInSectionsPositions(DBGenericMessage $message)
|
|
{
|
|
$changedFields = [];
|
|
|
|
if ($message->hasValueChanged('position')) {
|
|
$changedFields[] = ElasticUpdateGroup::SectionPosition;
|
|
}
|
|
|
|
$this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
|
|
}
|
|
|
|
public function handleDefault(DBGenericMessage $message): void
|
|
{
|
|
}
|
|
|
|
public function updateIndices()
|
|
{
|
|
$this->startTime = microtime(true);
|
|
$this->updateProductsIndex();
|
|
}
|
|
|
|
protected function updateProductsIndex(): void
|
|
{
|
|
if (empty($this->insertedProducts) && empty($this->updatedProducts) && empty($this->deletedProducts)) {
|
|
return;
|
|
}
|
|
|
|
// do not update products that were deleted or inserted
|
|
$this->updatedProducts = array_filter($this->updatedProducts, function ($pId) {
|
|
if (isset($this->insertedProducts[$pId]) || isset($this->deletedProducts[$pId])) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}, ARRAY_FILTER_USE_KEY);
|
|
|
|
// filter only product updates that will result in elastic field change
|
|
$allElasticDeps = $this->fulltextElastic->getElasticFieldsAllDeps(FulltextElastic::INDEX_PRODUCTS);
|
|
$this->updatedProducts = array_filter(array_map(fn ($prodDeps) => array_intersect($prodDeps, $allElasticDeps), $this->updatedProducts));
|
|
|
|
// unset updated products that do not exist in elastic
|
|
foreach ($this->checkIfElasticProductsExist(array_keys($this->updatedProducts)) as $pId) {
|
|
unset($this->updatedProducts[$pId]);
|
|
$this->addInsertedProduct($pId);
|
|
$this->log['nonExistingUpdatesCount'] = ($this->log['nonExistingUpdatesCount'] ?? 0) + 1;
|
|
$this->log['nonExistingUpdatesIds'] = array_merge($this->log['nonExistingUpdatesIds'] ?? [], [$pId]);
|
|
}
|
|
|
|
try {
|
|
$this->forEachFulltextLanguage(function () {
|
|
$this->fulltextElastic->deleteProducts(array_map(fn ($pId) => ['id' => $pId], $this->deletedProducts));
|
|
|
|
if (!empty($this->insertedProducts)) {
|
|
$this->fulltextElastic->updateProduct(array_keys($this->insertedProducts));
|
|
}
|
|
if (!empty($this->updatedProducts)) {
|
|
$this->fulltextElastic->partialProductsUpdate($this->updatedProducts);
|
|
}
|
|
});
|
|
|
|
$this->logger->notice('Kafka - elastic products index updated', $this->getLogStats());
|
|
} catch (\Throwable $e) {
|
|
$this->sentryLogger->captureException(new \Exception('Kafka - Elastic index update failed', 0, $e), ['extra' => [
|
|
'stats' => $this->getLogStats(),
|
|
'inserted' => array_keys($this->insertedProducts),
|
|
'updated' => $this->updatedProducts,
|
|
'deleted' => array_keys($this->deletedProducts),
|
|
]]);
|
|
}
|
|
}
|
|
|
|
protected function addUpdatedProduct(int $productId, array $fields)
|
|
{
|
|
if (empty($fields)) {
|
|
return;
|
|
}
|
|
|
|
if ($this->updatedProducts[$productId] ?? false) {
|
|
$this->updatedProducts[$productId] = array_unique(array_merge($this->updatedProducts[$productId], $fields));
|
|
} else {
|
|
$this->updatedProducts[$productId] = $fields;
|
|
}
|
|
}
|
|
|
|
protected function addInsertedProduct(int $productId)
|
|
{
|
|
$this->insertedProducts[$productId] = $productId;
|
|
}
|
|
|
|
protected function addDeletedProduct(int $productId)
|
|
{
|
|
unset($this->insertedProducts[$productId]);
|
|
unset($this->updatedProducts[$productId]);
|
|
$this->deletedProducts[$productId] = $productId;
|
|
}
|
|
|
|
// Get ids of updates that do not exist in elastic
|
|
public function checkIfElasticProductsExist($checkIds)
|
|
{
|
|
$existIds = [];
|
|
|
|
foreach (array_chunk($checkIds, 100) as $idsChunk) {
|
|
$query = new Query();
|
|
$query->setFields(['id']);
|
|
$query->setSource(false);
|
|
$query->setQuery((new BoolQuery())->addFilter($this->query()->terms('id', $idsChunk)));
|
|
$query->setSize(count($idsChunk));
|
|
$search = $this->searchClientFactory->createSearch();
|
|
$res = $search->search($query);
|
|
|
|
$existIds = array_merge($existIds, array_map(fn ($result) => $result->getId(), $res->getResults()));
|
|
}
|
|
|
|
return array_diff($checkIds, $existIds);
|
|
}
|
|
|
|
protected function getLogStats(): array
|
|
{
|
|
$searchFieldsDeps = $this->fulltextElastic->getElasticFieldsDeps(FulltextElastic::INDEX_PRODUCTS);
|
|
|
|
// which elastic fields dependencies were triggered and how many times
|
|
$depsCount = [];
|
|
|
|
// count which fields in elastic were update and how many times
|
|
$fieldsCount = [];
|
|
|
|
foreach ($this->updatedProducts as $fields) {
|
|
foreach ($fields as $field) {
|
|
$depsCount[$field] = ($depsCount[$field] ?? 0) + 1;
|
|
|
|
$elasticFields = [];
|
|
|
|
foreach ($searchFieldsDeps as $elasticField => $searchFieldDeps) {
|
|
if (in_array($field, $searchFieldDeps)) {
|
|
$elasticFields[] = $elasticField;
|
|
}
|
|
}
|
|
|
|
foreach ($elasticFields as $elasticField) {
|
|
$fieldsCount[$elasticField] = ($fieldsCount[$elasticField] ?? 0) + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
$this->log['figureUpdateIds'] = array_slice($this->log['figureUpdateIds'] ?? [], 0, 20);
|
|
$this->log['nonExistingUpdatesIds'] = array_slice($this->log['nonExistingUpdatesIds'] ?? [], 0, 20);
|
|
|
|
return array_merge([
|
|
'insertedIds' => array_keys($this->insertedProducts),
|
|
'insertedCount' => count($this->insertedProducts),
|
|
'deletedCount' => count($this->deletedProducts),
|
|
'deletedIds' => array_keys($this->deletedProducts),
|
|
'updatedCount' => count($this->updatedProducts),
|
|
'updatedIds' => array_keys($this->updatedProducts),
|
|
'updateDepsTotal' => $depsCount,
|
|
'updateFieldsTotal' => $fieldsCount,
|
|
'updateDeps' => array_slice($this->updatedProducts, 0, 20, true),
|
|
'time_seconds' => microtime(true) - $this->startTime,
|
|
], $this->log);
|
|
}
|
|
}
|