first commit

bundles/KupShop/KafkaBundle/Command/ProcessDBChangesCommand.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace KupShop\KafkaBundle\Command;
+
+use KupShop\KafkaBundle\SynchronizationRegister\ConsumersJobsRegister;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+#[AsCommand(name: 'kupshop:elastic:kafka', description: 'Process db changes from kafka and update index')]
+class ProcessDBChangesCommand extends Command
+{
+    public function __construct(
+        protected ConsumersJobsRegister $consumersJobsRegister,
+    ) {
+        parent::__construct();
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $this->consumersJobsRegister->elasticJob();
+
+        $output->writeln('<info>Done - consumed changes from kafka</info>');
+
+        return 0;
+    }
+}

bundles/KupShop/KafkaBundle/Connection/KafkaConnection.php

@@ -0,0 +1,279 @@
+<?php
+
+namespace KupShop\KafkaBundle\Connection;
+
+use KupShop\KafkaBundle\Exception\KafkaConfigException;
+use RdKafka\Conf;
+use RdKafka\KafkaConsumer;
+use RdKafka\Producer;
+use Symfony\Component\Messenger\Exception\RuntimeException;
+
+/**
+ * Zkopírováno z https://github.com/symfony-examples/messenger-kafka a upraveno.
+ */
+class KafkaConnection
+{
+    private const BROKERS_LIST = 'metadata.broker.list';
+    private const GROUP_ID = 'group.id';
+    private const PRODUCER_MESSAGE_FLAGS_BLOCK = 'producer_message_flags_block';
+    private const PRODUCER_PARTITION_ID_ASSIGNMENT = 'producer_partition_id_assignment';
+    private const CONSUMER_TOPICS_NAME = 'consumer_topics';
+    private const CONSUMER_CONSUME_TIMEOUT_MS = 'consumer_consume_timeout_ms';
+    private const PRODUCER_POLL_TIMEOUT_MS = 'producer_poll_timeout_ms';
+    private const PRODUCER_FLUSH_TIMEOUT_MS = 'producer_flush_timeout_ms';
+    private const PRODUCER_TOPIC_NAME = 'producer_topic';
+    private const TRANSPORT_NAME = 'transport_name';
+    private const GLOBAL_OPTIONS = [
+        self::TRANSPORT_NAME,
+        self::CONSUMER_CONSUME_TIMEOUT_MS,
+        self::PRODUCER_POLL_TIMEOUT_MS,
+        self::PRODUCER_FLUSH_TIMEOUT_MS,
+        self::PRODUCER_MESSAGE_FLAGS_BLOCK,
+        self::PRODUCER_PARTITION_ID_ASSIGNMENT,
+        self::CONSUMER_TOPICS_NAME,
+        self::PRODUCER_TOPIC_NAME,
+    ];
+
+    /** @psalm-param array<string, bool|float|int|string|array<string>> $kafkaConfig */
+    public function __construct(
+        private readonly array $kafkaConfig,
+        private readonly string $transportName,
+    ) {
+        if (!\extension_loaded('rdkafka')) {
+            throw new KafkaConfigException(sprintf(
+                'You cannot use the "%s" as the "rdkafka" extension is not installed.',
+                __CLASS__
+            ));
+        }
+    }
+
+    /** @psalm-param array<string, bool|float|int|string|array<string>> $options */
+    public static function builder(array $options = []): self
+    {
+        if (!array_key_exists(self::TRANSPORT_NAME, $options) || !is_string($options[self::TRANSPORT_NAME])) {
+            throw new RuntimeException('Transport name must be exist end type of string.');
+        }
+
+        self::optionsValidator($options, $options[self::TRANSPORT_NAME]);
+
+        return new self($options, $options[self::TRANSPORT_NAME]);
+    }
+
+    public function createConsumer(): KafkaConsumer
+    {
+        if (!array_key_exists(self::GROUP_ID, $this->kafkaConfig)) {
+            throw new KafkaConfigException(sprintf(
+                'The transport "%s" is not configured to consume messages because "%s" option is missing.',
+                $this->transportName,
+                self::GROUP_ID
+            ));
+        }
+
+        $conf = new Conf();
+
+        foreach ($this->kafkaConfig as $key => $value) {
+            if (array_key_exists($key, array_merge(KafkaConnectionOption::global(), KafkaConnectionOption::consumer()))) {
+                if (!is_string($value)) {
+                    continue;
+                }
+                $conf->set($key, $value);
+            }
+        }
+
+        return new KafkaConsumer($conf);
+    }
+
+    public function createProducer(): Producer
+    {
+        $conf = new Conf();
+
+        foreach ($this->kafkaConfig as $key => $value) {
+            if (array_key_exists($key, array_merge(KafkaConnectionOption::global(), KafkaConnectionOption::producer()))) {
+                if (!is_string($value)) {
+                    continue;
+                }
+                $conf->set($key, $value);
+            }
+        }
+
+        return new Producer($conf);
+    }
+
+    /** @psalm-param array<string, string> $headers */
+    public function publish(string $body, array $headers = []): void
+    {
+        $producer = $this->createProducer();
+
+        $topic = $producer->newTopic($this->getTopic());
+        $topic->producev(
+            partition: $this->getPartitionId(), // todo: retrieve from stamp ?
+            msgflags: $this->getMessageFlags(),
+            payload: $body,
+            headers: $headers
+        );
+
+        $producer->poll($this->getProducerPollTimeout());
+        $producer->flush($this->getProducerFlushTimeout());
+    }
+
+    /** @psalm-param array<string, bool|float|int|string|array<string>> $options */
+    private static function optionsValidator(array $options, string $transportName): void
+    {
+        $invalidOptions = array_diff(
+            array_keys($options),
+            array_merge(
+                self::GLOBAL_OPTIONS,
+                array_keys(
+                    array_merge(self::GLOBAL_OPTIONS, KafkaConnectionOption::consumer(), KafkaConnectionOption::producer())
+                )
+            )
+        );
+
+        if (0 < \count($invalidOptions)) {
+            throw new KafkaConfigException(sprintf(
+                'Invalid option(s) "%s" passed to the Kafka Messenger transport "%s".',
+                implode('", "', $invalidOptions),
+                $transportName
+            ));
+        }
+    }
+
+    /** @psalm-return array<string> */
+    public function getTopics(): array
+    {
+        if (!array_key_exists(self::CONSUMER_TOPICS_NAME, $this->kafkaConfig)) {
+            throw new KafkaConfigException(sprintf(
+                'The transport "%s" is not configured to consume messages because "%s" option is missing.',
+                $this->transportName,
+                self::CONSUMER_TOPICS_NAME
+            ));
+        }
+
+        if (!is_array($this->kafkaConfig[self::CONSUMER_TOPICS_NAME])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be array, %s given in "%s" transport.',
+                self::CONSUMER_TOPICS_NAME,
+                gettype($this->kafkaConfig[self::CONSUMER_TOPICS_NAME]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::CONSUMER_TOPICS_NAME];
+    }
+
+    public function getConsumerConsumeTimeout(): int
+    {
+        if (isLocalDevelopment()) {
+            return 1000;
+        }
+
+        if (!array_key_exists(self::CONSUMER_CONSUME_TIMEOUT_MS, $this->kafkaConfig)) {
+            return 10000;
+        }
+
+        if (!is_int($this->kafkaConfig[self::CONSUMER_CONSUME_TIMEOUT_MS])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be integer, %s given in "%s" transport.',
+                self::CONSUMER_CONSUME_TIMEOUT_MS,
+                gettype($this->kafkaConfig[self::CONSUMER_CONSUME_TIMEOUT_MS]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::CONSUMER_CONSUME_TIMEOUT_MS];
+    }
+
+    private function getTopic(): string
+    {
+        if (!array_key_exists(self::PRODUCER_TOPIC_NAME, $this->kafkaConfig)) {
+            throw new KafkaConfigException(sprintf(
+                'The transport "%s" is not configured to dispatch messages because "%s" option is missing.',
+                $this->transportName,
+                self::PRODUCER_TOPIC_NAME
+            ));
+        }
+
+        if (!is_string($this->kafkaConfig[self::PRODUCER_TOPIC_NAME])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be string, %s given in "%s" transport.',
+                self::PRODUCER_TOPIC_NAME,
+                gettype($this->kafkaConfig[self::PRODUCER_TOPIC_NAME]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::PRODUCER_TOPIC_NAME];
+    }
+
+    private function getMessageFlags(): int
+    {
+        if (!array_key_exists(self::PRODUCER_MESSAGE_FLAGS_BLOCK, $this->kafkaConfig)) {
+            return 0;
+        }
+
+        if (!is_bool($this->kafkaConfig[self::PRODUCER_MESSAGE_FLAGS_BLOCK])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be boolean, %s given in "%s" transport.',
+                self::PRODUCER_MESSAGE_FLAGS_BLOCK,
+                gettype($this->kafkaConfig[self::PRODUCER_MESSAGE_FLAGS_BLOCK]),
+                $this->transportName
+            ));
+        }
+
+        return false === $this->kafkaConfig[self::PRODUCER_MESSAGE_FLAGS_BLOCK] ? 0 : RD_KAFKA_MSG_F_BLOCK;
+    }
+
+    private function getPartitionId(): int
+    {
+        if (!array_key_exists(self::PRODUCER_PARTITION_ID_ASSIGNMENT, $this->kafkaConfig)) {
+            return RD_KAFKA_PARTITION_UA;
+        }
+
+        if (!is_int($this->kafkaConfig[self::PRODUCER_PARTITION_ID_ASSIGNMENT])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be integer, %s given in "%s" transport.',
+                self::PRODUCER_PARTITION_ID_ASSIGNMENT,
+                gettype($this->kafkaConfig[self::PRODUCER_PARTITION_ID_ASSIGNMENT]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::PRODUCER_PARTITION_ID_ASSIGNMENT];
+    }
+
+    private function getProducerPollTimeout(): int
+    {
+        if (!array_key_exists(self::PRODUCER_POLL_TIMEOUT_MS, $this->kafkaConfig)) {
+            return 0;
+        }
+
+        if (!is_int($this->kafkaConfig[self::PRODUCER_POLL_TIMEOUT_MS])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be integer, %s given in "%s" transport.',
+                self::PRODUCER_POLL_TIMEOUT_MS,
+                gettype($this->kafkaConfig[self::PRODUCER_POLL_TIMEOUT_MS]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::PRODUCER_POLL_TIMEOUT_MS];
+    }
+
+    private function getProducerFlushTimeout(): int
+    {
+        if (!array_key_exists(self::PRODUCER_FLUSH_TIMEOUT_MS, $this->kafkaConfig)) {
+            return 10000;
+        }
+
+        if (!is_int($this->kafkaConfig[self::PRODUCER_FLUSH_TIMEOUT_MS])) {
+            throw new KafkaConfigException(sprintf(
+                'The "%s" option type must be integer, %s given in "%s" transport.',
+                self::PRODUCER_FLUSH_TIMEOUT_MS,
+                gettype($this->kafkaConfig[self::PRODUCER_FLUSH_TIMEOUT_MS]),
+                $this->transportName
+            ));
+        }
+
+        return $this->kafkaConfig[self::PRODUCER_FLUSH_TIMEOUT_MS];
+    }
+}

bundles/KupShop/KafkaBundle/Connection/KafkaConnectionConfig.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Connection;
+
+use KupShop\KafkaBundle\Util\KafkaUtilTrait;
+
+class KafkaConnectionConfig
+{
+    use KafkaUtilTrait;
+
+    public function __construct(
+        private string $groupId,
+        private string $topic = 'default',
+        private array $config = [],
+    ) {
+    }
+
+    public function getGroupId(): string
+    {
+        return $this->groupId;
+    }
+
+    public function getTopic(): string
+    {
+        return $this->getShopTopicName($this->topic);
+    }
+
+    public function getTopics(): array
+    {
+        return [$this->getTopic()];
+    }
+
+    public function getConfig(): array
+    {
+        return $this->config;
+    }
+}

bundles/KupShop/KafkaBundle/Connection/KafkaConnectionOption.php

@@ -0,0 +1,204 @@
+<?php
+
+namespace KupShop\KafkaBundle\Connection;
+
+/**
+ *  Zkopírováno z https://github.com/symfony-examples/messenger-kafka.
+ *
+ * @SuppressWarnings(PHPMD.ExcessiveMethodLength)
+ *
+ * @see https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+ */
+final class KafkaConnectionOption
+{
+    /** @psalm-return array<string, string> */
+    public static function consumer(): array
+    {
+        return array_merge(
+            self::global(),
+            [
+                'group.id' => 'C',
+                'group.instance.id' => 'C',
+                'partition.assignment.strategy' => 'C',
+                'session.timeout.ms' => 'C',
+                'heartbeat.interval.ms' => 'C',
+                'group.protocol.type' => 'C',
+                'coordinator.query.interval.ms' => 'C',
+                'max.poll.interval.ms' => 'C',
+                'enable.auto.commit' => 'C',
+                'auto.commit.interval.ms' => 'C',
+                'enable.auto.offset.store' => 'C',
+                'queued.min.messages' => 'C',
+                'queued.max.messages.kbytes' => 'C',
+                'fetch.wait.max.ms' => 'C',
+                'fetch.message.max.bytes' => 'C',
+                'max.partition.fetch.bytes' => 'C',
+                'fetch.max.bytes' => 'C',
+                'fetch.min.bytes' => 'C',
+                'fetch.error.backoff.ms' => 'C',
+                'offset.store.method' => 'C',
+                'isolation.level' => 'C',
+                'consume_cb' => 'C',
+                'rebalance_cb' => 'C',
+                'offset_commit_cb' => 'C',
+                'enable.partition.eof' => 'C',
+                'check.crcs' => 'C',
+                'auto.commit.enable' => 'C',
+                'auto.offset.reset' => 'C',
+                'offset.store.path' => 'C',
+                'offset.store.sync.interval.ms' => 'C',
+                'consume.callback.max.messages' => 'C',
+            ]
+        );
+    }
+
+    /** @psalm-return array<string, string> */
+    public static function producer(): array
+    {
+        return array_merge(
+            self::global(),
+            [
+                'transactional.id' => 'P',
+                'transaction.timeout.ms' => 'P',
+                'enable.idempotence' => 'P',
+                'enable.gapless.guarantee' => 'P',
+                'queue.buffering.max.messages' => 'P',
+                'queue.buffering.max.kbytes' => 'P',
+                'queue.buffering.max.ms' => 'P',
+                'linger.ms' => 'P',
+                'message.send.max.retries' => 'P',
+                'retries' => 'P',
+                'retry.backoff.ms' => 'P',
+                'queue.buffering.backpressure.threshold' => 'P',
+                'compression.codec' => 'P',
+                'compression.type' => 'P',
+                'batch.num.messages' => 'P',
+                'batch.size' => 'P',
+                'delivery.report.only.error' => 'P',
+                'dr_cb' => 'P',
+                'dr_msg_cb' => 'P',
+                'sticky.partitioning.linger.ms' => 'P',
+                'request.required.acks' => 'P',
+                'acks' => 'P',
+                'request.timeout.ms' => 'P',
+                'message.timeout.ms' => 'P',
+                'delivery.timeout.ms' => 'P',
+                'queuing.strategy' => 'P',
+                'produce.offset.report' => 'P',
+                'partitioner' => 'P',
+                'partitioner_cb' => 'P',
+                'msg_order_cmp' => 'P',
+                'compression.level' => 'P',
+            ]
+        );
+    }
+
+    /** @psalm-return array<string, string> */
+    public static function global(): array
+    {
+        return [
+            'builtin.features' => '*',
+            'client.id' => '*',
+            'metadata.broker.list' => '*',
+            'bootstrap.servers' => '*',
+            'message.max.bytes' => '*',
+            'message.copy.max.bytes' => '*',
+            'receive.message.max.bytes' => '*',
+            'max.in.flight.requests.per.connection' => '*',
+            'max.in.flight' => '*',
+            'topic.metadata.refresh.interval.ms' => '*',
+            'metadata.max.age.ms' => '*',
+            'topic.metadata.refresh.fast.interval.ms' => '*',
+            'topic.metadata.refresh.fast.cnt' => '*',
+            'topic.metadata.refresh.sparse' => '*',
+            'topic.metadata.propagation.max.ms' => '*',
+            'topic.blacklist' => '*',
+            'debug' => '*',
+            'socket.timeout.ms' => '*',
+            'socket.blocking.max.ms' => '*',
+            'socket.send.buffer.bytes' => '*',
+            'socket.receive.buffer.bytes' => '*',
+            'socket.keepalive.enable' => '*',
+            'socket.nagle.disable' => '*',
+            'socket.max.fails' => '*',
+            'broker.address.ttl' => '*',
+            'broker.address.family' => '*',
+            'socket.connection.setup.timeout.ms' => '*',
+            'connections.max.idle.ms' => '*',
+            'reconnect.backoff.jitter.ms' => '*',
+            'reconnect.backoff.ms' => '*',
+            'reconnect.backoff.max.ms' => '*',
+            'statistics.interval.ms' => '*',
+            'enabled_events' => '*',
+            'error_cb' => '*',
+            'throttle_cb' => '*',
+            'stats_cb' => '*',
+            'log_cb' => '*',
+            'log_level' => '*',
+            'log.queue' => '*',
+            'log.thread.name' => '*',
+            'enable.random.seed' => '*',
+            'log.connection.close' => '*',
+            'background_event_cb' => '*',
+            'socket_cb' => '*',
+            'connect_cb' => '*',
+            'closesocket_cb' => '*',
+            'open_cb' => '*',
+            'resolve_cb' => '*',
+            'opaque' => '*',
+            'default_topic_conf' => '*',
+            'internal.termination.signal' => '*',
+            'api.version.request' => '*',
+            'api.version.request.timeout.ms' => '*',
+            'api.version.fallback.ms' => '*',
+            'broker.version.fallback' => '*',
+            'allow.auto.create.topics' => '*',
+            'security.protocol' => '*',
+            'ssl.cipher.suites' => '*',
+            'ssl.curves.list' => '*',
+            'ssl.sigalgs.list' => '*',
+            'ssl.key.location' => '*',
+            'ssl.key.password' => '*',
+            'ssl.key.pem' => '*',
+            'ssl_key' => '*',
+            'ssl.certificate.location' => '*',
+            'ssl.certificate.pem' => '*',
+            'ssl_certificate' => '*',
+            'ssl.ca.location' => '*',
+            'ssl.ca.pem' => '*',
+            'ssl_ca' => '*',
+            'ssl.ca.certificate.stores' => '*',
+            'ssl.crl.location' => '*',
+            'ssl.keystore.location' => '*',
+            'ssl.keystore.password' => '*',
+            'ssl.providers' => '*',
+            'ssl.engine.location' => '*',
+            'ssl.engine.id' => '*',
+            'ssl_engine_callback_data' => '*',
+            'enable.ssl.certificate.verification' => '*',
+            'ssl.endpoint.identification.algorithm' => '*',
+            'ssl.certificate.verify_cb' => '*',
+            'sasl.mechanisms' => '*',
+            'sasl.mechanism' => '*',
+            'sasl.kerberos.service.name' => '*',
+            'sasl.kerberos.principal' => '*',
+            'sasl.kerberos.kinit.cmd' => '*',
+            'sasl.kerberos.keytab' => '*',
+            'sasl.kerberos.min.time.before.relogin' => '*',
+            'sasl.username' => '*',
+            'sasl.password' => '*',
+            'sasl.oauthbearer.config' => '*',
+            'enable.sasl.oauthbearer.unsecure.jwt' => '*',
+            'oauthbearer_token_refresh_cb' => '*',
+            'sasl.oauthbearer.method' => '*',
+            'sasl.oauthbearer.client.id' => '*',
+            'sasl.oauthbearer.client.secret' => '*',
+            'sasl.oauthbearer.scope' => '*',
+            'sasl.oauthbearer.extensions' => '*',
+            'sasl.oauthbearer.token.endpoint.url' => '*',
+            'plugin.library.paths' => '*',
+            'interceptors' => '*',
+            'client.rack' => '*',
+        ];
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/DBChangeTypeEnum.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges;
+
+enum DBChangeTypeEnum: string
+{
+    case INSERT = 'c';
+    case UPDATE = 'u';
+    case DELETE = 'd';
+    case UNKNOWN = 'unknown';
+
+    public static function tryFromLegacy(string $name): static
+    {
+        return match ($name) {
+            'insert' => DBChangeTypeEnum::INSERT,
+            'update_after' => DBChangeTypeEnum::UPDATE,
+            'delete' => DBChangeTypeEnum::DELETE,
+            default => DBChangeTypeEnum::UNKNOWN,
+        };
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageHandlers/DBMessageAbstractHandler.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageHandlers;
+
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBGenericMessage;
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBMsgProduct;
+use KupShop\KafkaBundle\DBChanges\Util\DBMessageFactory;
+use Psr\Log\LoggerInterface;
+use RdKafka\Message;
+use Symfony\Contracts\Service\Attribute\Required;
+
+/**
+ * @method handleProducts(DBMsgProduct $message)
+ * @method handleProductsVariations(DBGenericMessage $message)
+ * @method handleParametersProducts(DBGenericMessage $message)
+ * @method handleProductsInSections(DBGenericMessage $message)
+ * @method handlePricelistsProducts(DBGenericMessage $message)
+ * @method handlePriceLevelsProducts(DBGenericMessage $message)
+ * @method handleProductsOfSuppliers(DBGenericMessage $message)
+ * @method handleProductLabelsRelation(DBGenericMessage $message)
+ * @method handleStoresItems(DBGenericMessage $message)
+ * @method handleProductsInSectionsPositions(DBGenericMessage $message)
+ * @method handleProductsVariationsChoicesCategorization(DBGenericMessage $message)
+ * @method handlePhotosProductsRelation(DBGenericMessage $message)
+ */
+abstract class DBMessageAbstractHandler implements DBMessageHandlerInterface
+{
+    #[Required]
+    public DBMessageFactory $dbMessageFactory;
+
+    #[Required]
+    public LoggerInterface $logger;
+
+    abstract public function handleDefault(DBGenericMessage $message): void;
+
+    public function createMessageAndHandle(Message $rawMessage): void
+    {
+        $message = $this->dbMessageFactory->create($rawMessage);
+
+        $tableMethod = 'handle'.$this->tableCamelize($message->getTableName());
+
+        if (method_exists($this, $tableMethod)) {
+            $this->{$tableMethod}($message);
+        } else {
+            $this->handleDefault($message);
+        }
+    }
+
+    private function tableCamelize($input, $separator = '_')
+    {
+        return str_replace($separator, '', ucwords($input, $separator));
+    }
+
+    /**
+     * Use in synchronization tasks. Service instances are persisted between runs.
+     *
+     * @return $this
+     */
+    public function newInstance(): self
+    {
+        return clone $this;
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageHandlers/DBMessageElasticHandler.php

@@ -0,0 +1,328 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageHandlers;
+
+use Elastica\Query\BoolQuery;
+use KupShop\AdminBundle\FulltextUpdatesTrait;
+use KupShop\CatalogBundle\Search\ElasticUpdateGroup;
+use KupShop\CatalogBundle\Search\FulltextElastic;
+use KupShop\CatalogElasticBundle\Elastica\Query;
+use KupShop\CatalogElasticBundle\Elastica\QueryBuilder;
+use KupShop\CatalogElasticBundle\Util\ElasticaFactory;
+use KupShop\KafkaBundle\DBChanges\DBChangeTypeEnum;
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBGenericMessage;
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBMsgProduct;
+use KupShop\KupShopBundle\Context\LanguageContext;
+use KupShop\KupShopBundle\Util\Logging\SentryLogger;
+use Symfony\Contracts\Service\Attribute\Required;
+
+class DBMessageElasticHandler extends DBMessageAbstractHandler
+{
+    use QueryBuilder;
+    use FulltextUpdatesTrait;
+    #[Required]
+    public FulltextElastic $fulltextElastic;
+    #[Required]
+    public LanguageContext $languageContext;
+    #[Required]
+    public SentryLogger $sentryLogger;
+
+    protected ?ElasticaFactory $searchClientFactory = null;
+
+    private array $updatedProducts = [];
+    private array $insertedProducts = [];
+    private array $deletedProducts = [];
+
+    private array $log = [];
+    private $startTime;
+
+    #[Required]
+    final public function setSearchClientFactory(?ElasticaFactory $searchClientFactory): void
+    {
+        $this->searchClientFactory = $searchClientFactory;
+    }
+
+    protected function handleProducts(DBMsgProduct $message): void
+    {
+        $productId = $message->getId();
+
+        switch ($message->getEventType()) {
+            case DBChangeTypeEnum::INSERT:
+                $this->addInsertedProduct($productId);
+                break;
+            case DBChangeTypeEnum::DELETE:
+                $this->addDeletedProduct($productId);
+                break;
+            case DBChangeTypeEnum::UPDATE:
+                if ($message->hasValueChanged('figure')) {
+                    $this->log['figureUpdateCount'] = ($this->log['figureUpdateCount'] ?? 0) + 1;
+                    $this->log['figureUpdateIds'] = array_merge($this->log['figureUpdateIds'] ?? [], [$productId]);
+                    $this->addInsertedProduct($productId);
+                    break;
+                }
+
+                $changedFields = $message->getChangedValuesKeys();
+                // update in_store only if changed from zero to non-zero or via versa
+                if ($message->hasValueBoolChanged('in_store')) {
+                    $changedFields[] = ElasticUpdateGroup::InStore;
+                    $changedFields[] = ElasticUpdateGroup::DeliveryTime;
+                }
+
+                $this->addUpdatedProduct($productId, $changedFields);
+                break;
+        }
+    }
+
+    protected function handleParametersProducts(DBGenericMessage $message): void
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [
+            ElasticUpdateGroup::Parameters,
+        ]);
+    }
+
+    protected function handleProductsInSections(DBGenericMessage $message): void
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [
+            ElasticUpdateGroup::Sections,
+        ]);
+    }
+
+    public function handleProductsVariations(DBGenericMessage $message)
+    {
+        $changedFields = [];
+
+        if ($message->hasValueBoolChanged('in_store') || $message->hasValueChanged('delivery_time')) {
+            $changedFields[] = ElasticUpdateGroup::InStore;
+            $changedFields[] = ElasticUpdateGroup::DeliveryTime;
+        }
+
+        if ($message->startsWithChanged('price')) {
+            $changedFields[] = ElasticUpdateGroup::Price;
+        }
+
+        if ($message->hasValueChanged('ean')) {
+            $changedFields[] = ElasticUpdateGroup::Ean;
+        }
+        if ($message->hasValueChanged('code')) {
+            $changedFields[] = ElasticUpdateGroup::Code;
+        }
+
+        if ($message->didAnyOfFieldChange(['title'])) {
+            $changedFields[] = ElasticUpdateGroup::Variations;
+        }
+
+        $this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
+    }
+
+    public function handleProductsVariationsChoicesCategorization(DBGenericMessage $message)
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [
+            ElasticUpdateGroup::Variations,
+        ]);
+    }
+
+    public function handleProductLabelsRelation(DBGenericMessage $message)
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [
+            ElasticUpdateGroup::Labels,
+        ]);
+    }
+
+    public function handlePriceLevelsProducts(DBGenericMessage $message)
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [
+            ElasticUpdateGroup::Price,
+        ]);
+    }
+
+    public function handleProductsOfSuppliers(DBGenericMessage $message)
+    {
+        $changedFields = [];
+
+        if ($message->hasValueBoolChanged('in_store')) {
+            $changedFields[] = ElasticUpdateGroup::InStore;
+            $changedFields[] = ElasticUpdateGroup::DeliveryTime;
+        }
+
+        $this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
+    }
+
+    public function handlePhotosProductsRelation(DBGenericMessage $message)
+    {
+        $this->addUpdatedProduct($message->getValue('id_product'), [ElasticUpdateGroup::Photo]);
+    }
+
+    public function handleStoresItems(DBGenericMessage $message)
+    {
+        $changedFields = [];
+
+        if ($message->hasValueBoolChanged('quantity')) {
+            $changedFields[] = ElasticUpdateGroup::StoresInStore;
+        }
+
+        $this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
+    }
+
+    public function handleProductsInSectionsPositions(DBGenericMessage $message)
+    {
+        $changedFields = [];
+
+        if ($message->hasValueChanged('position')) {
+            $changedFields[] = ElasticUpdateGroup::SectionPosition;
+        }
+
+        $this->addUpdatedProduct($message->getValue('id_product'), $changedFields);
+    }
+
+    public function handleDefault(DBGenericMessage $message): void
+    {
+    }
+
+    public function updateIndices()
+    {
+        $this->startTime = microtime(true);
+        $this->updateProductsIndex();
+    }
+
+    protected function updateProductsIndex(): void
+    {
+        if (empty($this->insertedProducts) && empty($this->updatedProducts) && empty($this->deletedProducts)) {
+            return;
+        }
+
+        // do not update products that were deleted or inserted
+        $this->updatedProducts = array_filter($this->updatedProducts, function ($pId) {
+            if (isset($this->insertedProducts[$pId]) || isset($this->deletedProducts[$pId])) {
+                return false;
+            }
+
+            return true;
+        }, ARRAY_FILTER_USE_KEY);
+
+        // filter only product updates that will result in elastic field change
+        $allElasticDeps = $this->fulltextElastic->getElasticFieldsAllDeps(FulltextElastic::INDEX_PRODUCTS);
+        $this->updatedProducts = array_filter(array_map(fn ($prodDeps) => array_intersect($prodDeps, $allElasticDeps), $this->updatedProducts));
+
+        // unset updated products that do not exist in elastic
+        foreach ($this->checkIfElasticProductsExist(array_keys($this->updatedProducts)) as $pId) {
+            unset($this->updatedProducts[$pId]);
+            $this->addInsertedProduct($pId);
+            $this->log['nonExistingUpdatesCount'] = ($this->log['nonExistingUpdatesCount'] ?? 0) + 1;
+            $this->log['nonExistingUpdatesIds'] = array_merge($this->log['nonExistingUpdatesIds'] ?? [], [$pId]);
+        }
+
+        try {
+            $this->forEachFulltextLanguage(function () {
+                $this->fulltextElastic->deleteProducts(array_map(fn ($pId) => ['id' => $pId], $this->deletedProducts));
+
+                if (!empty($this->insertedProducts)) {
+                    $this->fulltextElastic->updateProduct(array_keys($this->insertedProducts));
+                }
+                if (!empty($this->updatedProducts)) {
+                    $this->fulltextElastic->partialProductsUpdate($this->updatedProducts);
+                }
+            });
+
+            $this->logger->notice('Kafka - elastic products index updated', $this->getLogStats());
+        } catch (\Throwable $e) {
+            $this->sentryLogger->captureException(new \Exception('Kafka - Elastic index update failed', 0, $e), ['extra' => [
+                'stats' => $this->getLogStats(),
+                'inserted' => array_keys($this->insertedProducts),
+                'updated' => $this->updatedProducts,
+                'deleted' => array_keys($this->deletedProducts),
+            ]]);
+        }
+    }
+
+    protected function addUpdatedProduct(int $productId, array $fields)
+    {
+        if (empty($fields)) {
+            return;
+        }
+
+        if ($this->updatedProducts[$productId] ?? false) {
+            $this->updatedProducts[$productId] = array_unique(array_merge($this->updatedProducts[$productId], $fields));
+        } else {
+            $this->updatedProducts[$productId] = $fields;
+        }
+    }
+
+    protected function addInsertedProduct(int $productId)
+    {
+        $this->insertedProducts[$productId] = $productId;
+    }
+
+    protected function addDeletedProduct(int $productId)
+    {
+        unset($this->insertedProducts[$productId]);
+        unset($this->updatedProducts[$productId]);
+        $this->deletedProducts[$productId] = $productId;
+    }
+
+    // Get ids of updates that do not exist in elastic
+    public function checkIfElasticProductsExist($checkIds)
+    {
+        $existIds = [];
+
+        foreach (array_chunk($checkIds, 100) as $idsChunk) {
+            $query = new Query();
+            $query->setFields(['id']);
+            $query->setSource(false);
+            $query->setQuery((new BoolQuery())->addFilter($this->query()->terms('id', $idsChunk)));
+            $query->setSize(count($idsChunk));
+            $search = $this->searchClientFactory->createSearch();
+            $res = $search->search($query);
+
+            $existIds = array_merge($existIds, array_map(fn ($result) => $result->getId(), $res->getResults()));
+        }
+
+        return array_diff($checkIds, $existIds);
+    }
+
+    protected function getLogStats(): array
+    {
+        $searchFieldsDeps = $this->fulltextElastic->getElasticFieldsDeps(FulltextElastic::INDEX_PRODUCTS);
+
+        // which elastic fields dependencies were triggered and how many times
+        $depsCount = [];
+
+        // count which fields in elastic were update and how many times
+        $fieldsCount = [];
+
+        foreach ($this->updatedProducts as $fields) {
+            foreach ($fields as $field) {
+                $depsCount[$field] = ($depsCount[$field] ?? 0) + 1;
+
+                $elasticFields = [];
+
+                foreach ($searchFieldsDeps as $elasticField => $searchFieldDeps) {
+                    if (in_array($field, $searchFieldDeps)) {
+                        $elasticFields[] = $elasticField;
+                    }
+                }
+
+                foreach ($elasticFields as $elasticField) {
+                    $fieldsCount[$elasticField] = ($fieldsCount[$elasticField] ?? 0) + 1;
+                }
+            }
+        }
+
+        $this->log['figureUpdateIds'] = array_slice($this->log['figureUpdateIds'] ?? [], 0, 20);
+        $this->log['nonExistingUpdatesIds'] = array_slice($this->log['nonExistingUpdatesIds'] ?? [], 0, 20);
+
+        return array_merge([
+            'insertedIds' => array_keys($this->insertedProducts),
+            'insertedCount' => count($this->insertedProducts),
+            'deletedCount' => count($this->deletedProducts),
+            'deletedIds' => array_keys($this->deletedProducts),
+            'updatedCount' => count($this->updatedProducts),
+            'updatedIds' => array_keys($this->updatedProducts),
+            'updateDepsTotal' => $depsCount,
+            'updateFieldsTotal' => $fieldsCount,
+            'updateDeps' => array_slice($this->updatedProducts, 0, 20, true),
+            'time_seconds' => microtime(true) - $this->startTime,
+        ], $this->log);
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageHandlers/DBMessageHandlerInterface.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageHandlers;
+
+interface DBMessageHandlerInterface
+{
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageTypes/DBGenericMessage.php

@@ -0,0 +1,165 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageTypes;
+
+use KupShop\KafkaBundle\DBChanges\DBChangeTypeEnum;
+use KupShop\KafkaBundle\DBChanges\MessageHandlers\DBMessageAbstractHandler;
+use KupShop\KupShopBundle\Util\StringUtil;
+
+class DBGenericMessage implements DBMessageInterface
+{
+    public static string $tableName = '_default';
+    public const KEY_TABLE = 'table_name';
+    protected const KEY_MESSAGE_BEFORE = 'before';
+    protected const KEY_MESSAGE_AFTER = 'after';
+    protected const KEY_MESSAGE_SOURCE = 'source';
+    protected const KEY_EVENT_TYPE = 'op';
+    private int $offset;
+    private int $timestamp;
+    private array $values; // Don't you dare to make it protected/public
+    private array $headers;
+
+    public function getOffset(): int
+    {
+        return $this->offset;
+    }
+
+    public function getTimestamp(): int
+    {
+        return $this->timestamp;
+    }
+
+    public function getEventType(): DBChangeTypeEnum
+    {
+        return DBChangeTypeEnum::tryFrom($this->values[self::KEY_EVENT_TYPE]) ?? DBChangeTypeEnum::UNKNOWN;
+    }
+
+    public function hasValueChanged(string $key): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        $prevValues = $this->getPreviousValues();
+
+        if (!array_key_exists($key, $prevValues)) {
+            return false;
+        }
+
+        return $this->getValue($key) !== $prevValues[$key];
+    }
+
+    public function didAnyOfFieldChange(array $fields): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        return !empty(array_intersect($this->getChangedValuesKeys(), $fields));
+    }
+
+    public function getChangedValues()
+    {
+        return array_filter($this->getValues(), fn ($key) => $this->hasValueChanged($key), ARRAY_FILTER_USE_KEY);
+    }
+
+    public function getChangedValuesKeys(): array
+    {
+        return array_keys($this->getChangedValues());
+    }
+
+    public function getValues(): array
+    {
+        switch ($this->getEventType()) {
+            case DBChangeTypeEnum::INSERT:
+            case DBChangeTypeEnum::UPDATE:
+                return $this->values[self::KEY_MESSAGE_AFTER] ?? [];
+            case DBChangeTypeEnum::DELETE:
+                return $this->values[self::KEY_MESSAGE_BEFORE] ?? [];
+        }
+
+        return [];
+    }
+
+    public function getValue($key): mixed
+    {
+        return $this->getValues()[$key] ?? null;
+    }
+
+    public function getPreviousValues(): array
+    {
+        switch ($this->getEventType()) {
+            case DBChangeTypeEnum::UPDATE:
+                return $this->values[self::KEY_MESSAGE_BEFORE] ?? [];
+            case DBChangeTypeEnum::INSERT:
+            case DBChangeTypeEnum::DELETE:
+                return [];
+        }
+
+        return [];
+    }
+
+    public function hasValueBoolChanged(string $key): bool
+    {
+        if (!$this->hasValueChanged($key)) {
+            return false;
+        }
+
+        return (bool) $this->getValue($key) !== (bool) $this->getPreviousValue($key);
+    }
+
+    public function startsWithChanged(string $prefix): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        foreach ($this->getChangedValuesKeys() as $key) {
+            if (StringUtil::startsWith($key, $prefix)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public function getPreviousValue(string $key)
+    {
+        return $this->getPreviousValues()[$key] ?? null;
+    }
+
+    public function getTableName(): string
+    {
+        return $this->values[self::KEY_MESSAGE_SOURCE]['table'];
+    }
+
+    public static function getClassTableName(): string
+    {
+        return static::$tableName;
+    }
+
+    public function setMessage(int $offset, int $timestamp, array $headers, array $values)
+    {
+        $this->clear();
+
+        $this->offset = $offset;
+        $this->timestamp = $timestamp;
+        $this->headers = $headers;
+        $this->values = $values;
+
+        return $this;
+    }
+
+    public function clear(): void
+    {
+        // override
+        // clear extra object properties before next message is set into the service
+    }
+
+    public function visit(DBMessageAbstractHandler $handler): void
+    {
+        $handler->handleDefault($this);
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageTypes/DBLegacyMessage.php

@@ -0,0 +1,157 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageTypes;
+
+use KupShop\KafkaBundle\DBChanges\DBChangeTypeEnum;
+use KupShop\KafkaBundle\DBChanges\MessageHandlers\DBMessageAbstractHandler;
+use KupShop\KupShopBundle\Util\StringUtil;
+
+// dočasný hotfix, aby jsme mohli konzumovat zprávy ve starým maxscale formátu.
+// Potřeba než z topiců zmizí staré zprávy a zbydou jen zorávy v debezium formátu
+class DBLegacyMessage extends DBGenericMessage
+{
+    public static string $tableName = 'legacy';
+    public const KEY_TABLE = 'table_name';
+    protected const KEY_UPDATE_PREV = '_previous';
+    protected const KEY_EVENT_TYPE = 'event_type';
+    private int $offset;
+    private int $timestamp;
+    private array $values;
+    private array $headers;
+
+    public function getOffset(): int
+    {
+        return $this->offset;
+    }
+
+    public function getTimestamp(): int
+    {
+        return $this->timestamp;
+    }
+
+    public function getEventType(): DBChangeTypeEnum
+    {
+        return DBChangeTypeEnum::tryFromLegacy($this->values[self::KEY_EVENT_TYPE]) ?? DBChangeTypeEnum::UNKNOWN;
+    }
+
+    public function hasValueChanged(string $key): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        $prevValues = $this->getPreviousValues();
+
+        if (!array_key_exists($key, $prevValues)) {
+            return false;
+        }
+
+        return $this->getValue($key) !== $prevValues[$key];
+    }
+
+    public function didAnyOfFieldChange(array $fields): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        return !empty(array_intersect($this->getChangedValuesKeys(), $fields));
+    }
+
+    public function getChangedValues()
+    {
+        return array_filter($this->getValues(), fn ($key) => $this->hasValueChanged($key), ARRAY_FILTER_USE_KEY);
+    }
+
+    public function getChangedValuesKeys(): array
+    {
+        return array_keys($this->getChangedValues());
+    }
+
+    public function getValues(): array
+    {
+        return $this->values;
+    }
+
+    public function getValue($key): mixed
+    {
+        return $this->values[$key] ?? null;
+    }
+
+    public function getPreviousValues(): array
+    {
+        switch ($this->getEventType()) {
+            case DBChangeTypeEnum::UPDATE:
+                return $this->values[self::KEY_UPDATE_PREV] ?? [];
+            case DBChangeTypeEnum::INSERT:
+            case DBChangeTypeEnum::DELETE:
+                return [];
+        }
+
+        return [];
+    }
+
+    public function hasValueBoolChanged(string $key): bool
+    {
+        if (!$this->hasValueChanged($key)) {
+            return false;
+        }
+
+        return (bool) $this->getValue($key) !== (bool) $this->getPreviousValue($key);
+    }
+
+    public function startsWithChanged(string $prefix): bool
+    {
+        if ($this->getEventType() != DBChangeTypeEnum::UPDATE) {
+            return true;
+        }
+
+        foreach ($this->getChangedValuesKeys() as $key) {
+            if (StringUtil::startsWith($key, $prefix)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public function getPreviousValue(string $key)
+    {
+        return $this->getPreviousValues()[$key] ?? null;
+    }
+
+    public function getTableName(): string
+    {
+        return $this->getValue(self::KEY_TABLE);
+    }
+
+    public static function getClassTableName(): string
+    {
+        return static::$tableName;
+    }
+
+    public function setMessage(int $offset, int $timestamp, array $headers, array $values)
+    {
+        $this->clear();
+
+        $this->offset = $offset;
+        $this->timestamp = $timestamp;
+        $this->headers = $headers;
+        $this->values = $values;
+
+        return $this;
+    }
+
+    public function clear(): void
+    {
+        // override
+        // clear extra object properties before next message is set into the service
+    }
+
+    public function visit(DBMessageAbstractHandler $handler): void
+    {
+        $handler->handleDefault($this);
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageTypes/DBMessageInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageTypes;
+
+use Symfony\Component\DependencyInjection\Attribute\AutoconfigureTag;
+
+#[AutoconfigureTag('kupshop.kafka.dbmessages')]
+interface DBMessageInterface
+{
+}

bundles/KupShop/KafkaBundle/DBChanges/MessageTypes/DBMsgProduct.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\MessageTypes;
+
+class DBMsgProduct extends DBGenericMessage
+{
+    public static string $tableName = 'products';
+
+    public function getId(): int
+    {
+        return $this->getValue('id');
+    }
+}

bundles/KupShop/KafkaBundle/DBChanges/Util/DBMessageFactory.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\DBChanges\Util;
+
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBGenericMessage;
+use KupShop\KafkaBundle\DBChanges\MessageTypes\DBLegacyMessage;
+use KupShop\KafkaBundle\Exception\KafkaInvalidMessageException;
+use RdKafka\Message;
+use Symfony\Component\DependencyInjection\Attribute\TaggedLocator;
+use Symfony\Component\DependencyInjection\ServiceLocator;
+
+class DBMessageFactory
+{
+    public function __construct(
+        /** @param ServiceLocator<DBGenericMessage> $serviceLocator */
+        #[TaggedLocator(tag: 'kupshop.kafka.dbmessages', defaultIndexMethod: 'getClassTableName')] private ServiceLocator $serviceLocator,
+    ) {
+    }
+
+    /**
+     * @param $rawMessage array
+     *
+     * @throws KafkaInvalidMessageException
+     */
+    public function create(Message $rawMessage): DBGenericMessage
+    {
+        $body = json_decode($rawMessage->payload, true) ?? [];
+
+        if (!isset($body['cdc'])) {
+            // for messages in legacy format
+            $tableName = DBLegacyMessage::$tableName;
+        } else {
+            $tableName = $body['source']['table'] ?? '';
+
+            if (!$this->serviceLocator->has($tableName)) {
+                $tableName = DBGenericMessage::$tableName;
+            }
+        }
+
+        return $this->serviceLocator->get($tableName)->setMessage(
+            $rawMessage->offset,
+            (int) floor($rawMessage->timestamp / 1000),
+            $rawMessage->headers,
+            $body
+        );
+    }
+}

bundles/KupShop/KafkaBundle/Exception/KafkaConfigException.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Exception;
+
+class KafkaConfigException extends \RuntimeException
+{
+}

bundles/KupShop/KafkaBundle/Exception/KafkaException.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Exception;
+
+class KafkaException extends \RuntimeException
+{
+}

bundles/KupShop/KafkaBundle/Exception/KafkaInvalidMessageException.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Exception;
+
+class KafkaInvalidMessageException extends \RuntimeException
+{
+}

bundles/KupShop/KafkaBundle/Inspections/ModuleInspection.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace KupShop\KafkaBundle\Inspections;
+
+use KupShop\SystemInspectionBundle\Inspections\Compile\CompileInspectionInterface;
+use KupShop\SystemInspectionBundle\Inspections\Inspection;
+use KupShop\SystemInspectionBundle\InspectionWriters\MessageTypes\InspectionMessage;
+use KupShop\SystemInspectionBundle\InspectionWriters\MessageTypes\SimpleMessage;
+
+class ModuleInspection extends Inspection implements CompileInspectionInterface
+{
+    /**
+     * @return InspectionMessage[]|null
+     */
+    public function runInspection(): ?array
+    {
+        $errors = [];
+
+        if (!\extension_loaded('rdkafka')) {
+            $errors[] = new SimpleMessage('"rdkafka" extension is not installed.');
+        }
+
+        return $errors;
+    }
+}

bundles/KupShop/KafkaBundle/KafkaBundle.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace KupShop\KafkaBundle;
+
+use Symfony\Component\HttpKernel\Bundle\Bundle;
+
+/**
+ * WIKI: https://gitlab.wpj.cz/kupshop/engine/-/wikis/Kafka.
+ */
+class KafkaBundle extends Bundle
+{
+}

bundles/KupShop/KafkaBundle/Resources/config/config.yml

@@ -0,0 +1,3 @@
+parameters:
+    kupshop.kafka.brokers:
+      - 'cluster-kafka-bootstrap.kafka:9092'

bundles/KupShop/KafkaBundle/Resources/config/services.yml

@@ -0,0 +1,7 @@
+services:
+  _defaults:
+    autowire: true
+    autoconfigure: true
+
+  KupShop\KafkaBundle\:
+    resource: ../../{Util,DBChanges,SynchronizationRegister,Inspections,Command}

bundles/KupShop/KafkaBundle/Resources/upgrade/TopicUpgrade.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace KupShop\KafkaBundle\Resources\upgrade;
+
+use KupShop\KafkaBundle\Util\KafkaAdminUtil;
+use KupShop\KupShopBundle\Util\Compat\ServiceContainer;
+
+class TopicUpgrade extends \UpgradeNew
+{
+    //    public function check_DbChangesTopic()
+    //    {
+    //        $kafkaAdminUtil = ServiceContainer::getService(KafkaAdminUtil::class);
+    //
+    //        return !isLocalDevelopment() && !$kafkaAdminUtil->dbChangesTopicExists();
+    //    }
+    //
+    //    public function upgrade_DbChangesTopic()
+    //    {
+    //        $kafkaAdminUtil = ServiceContainer::getService(KafkaAdminUtil::class);
+    //        $kafkaAdminUtil->createDbChangesTopic();
+    //        $this->upgradeOK();
+    //    }
+}

bundles/KupShop/KafkaBundle/SynchronizationRegister/ConsumersJobsRegister.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\SynchronizationRegister;
+
+use KupShop\CatalogBundle\Section\SectionTree;
+use KupShop\KafkaBundle\Connection\KafkaConnectionConfig;
+use KupShop\KafkaBundle\DBChanges\MessageHandlers\DBMessageElasticHandler;
+use KupShop\KafkaBundle\Util\KafkaConsumer;
+use KupShop\SynchronizationBundle\Synchronization\Job;
+use KupShop\SynchronizationBundle\Synchronization\SynchronizationRegisterInterface;
+
+class ConsumersJobsRegister implements SynchronizationRegisterInterface
+{
+    public function __construct(
+        private KafkaConsumer $kafkaConsumer,
+        private DBMessageElasticHandler $elasticHandler,
+        public SectionTree $sectionTree,
+    ) {
+    }
+
+    public function getJobs(): iterable
+    {
+        if (findModule(\Modules::PRODUCTS_SECTIONS, \Modules::SUB_ELASTICSEARCH)) {
+            yield Job::create('System::elastic:update', fn () => $this->elasticJob(), description: 'Aktualizace vyhledávacího indexu', enableManualRun: false)
+                ->everyMinute(1);
+        }
+    }
+
+    public function elasticJob(): void
+    {
+        $elasticHandler = $this->elasticHandler->newInstance();
+        $this->sectionTree->clearRuntimeCache(); // take menuSectionTree from apcu/remote cache
+
+        $this->kafkaConsumer->processDbChangesMessages(
+            new KafkaConnectionConfig(groupId: 'kupshop-elastic'),
+            function ($rawMessage) use ($elasticHandler) {
+                $elasticHandler->createMessageAndHandle($rawMessage);
+            }
+        );
+
+        $elasticHandler->updateIndices();
+    }
+}

bundles/KupShop/KafkaBundle/Tests/DBMessagesTest.php

@@ -0,0 +1,118 @@
+<?php
+
+namespace KupShop\KafkaBundle\Tests;
+
+use KupShop\KafkaBundle\DBChanges\DBChangeTypeEnum;
+use KupShop\KafkaBundle\DBChanges\Util\DBMessageFactory;
+use RdKafka\Message;
+
+class DBMessagesTest extends \DatabaseTestCase
+{
+    /** @var DBMessageFactory */
+    private $dbMessageFactory;
+    private array $testMessages;
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        $this->dbMessageFactory = $this->get(DBMessageFactory::class);
+        $this->testMessages = json_decode(file_get_contents(__DIR__.'/dbMessagesTestPayloads.json'), true);
+    }
+
+    public function testProductsVariationsUpdate()
+    {
+        $message = $this->dbMessageFactory->create($this->createMessageFromTestData($this->testMessages['variationsUpdate0']));
+
+        self::assertEquals(DBChangeTypeEnum::UPDATE, $message->getEventType());
+
+        self::assertTrue($message->hasValueChanged('in_store'));
+        self::assertFalse($message->hasValueChanged('id'));
+        self::assertFalse($message->hasValueChanged('title'));
+        self::assertTrue($message->hasValueChanged('in_store'));
+
+        self::assertTrue($message->didAnyOfFieldChange(['title', 'in_store']));
+
+        self::assertFalse($message->didAnyOfFieldChange(['aaa', 'bbb']));
+
+        self::assertEquals(309, $message->getValue('in_store'));
+        self::assertEquals(1234, $message->getValue('ean'));
+        self::assertEquals(['in_store' => 309, 'updated' => '2025-04-29T10:52:10Z', 'delivery_time' => 0, 'ean' => 1234], $message->getChangedValues());
+        self::assertEquals(['in_store',  'delivery_time', 'ean', 'updated'], $message->getChangedValuesKeys());
+
+        self::assertFalse($message->hasValueBoolChanged('in_store'));
+        self::assertTrue($message->hasValueBoolChanged('delivery_time'));
+
+        self::assertFalse($message->startsWithChanged('id'));
+        self::assertTrue($message->startsWithChanged('in'));
+
+        self::assertEquals(312, $message->getPreviousValue('in_store'));
+        self::assertEquals('products_variations', $message->getTableName());
+    }
+
+    public function testProductsVariationsInsert()
+    {
+        $message = $this->dbMessageFactory->create($this->createMessageFromTestData($this->testMessages['variationsInsert0']));
+
+        self::assertEquals(DBChangeTypeEnum::INSERT, $message->getEventType());
+        self::assertEquals('products_variations', $message->getTableName());
+
+        self::assertNotNull($message->getChangedValues());
+        self::assertArraySubset($message->getChangedValues(), $message->getValues());
+
+        self::assertTrue($message->hasValueChanged('in_store'));
+        self::assertTrue($message->hasValueChanged('id'));
+        self::assertTrue($message->hasValueChanged('title'));
+
+        self::assertTrue($message->didAnyOfFieldChange(['title', 'in_store']));
+
+        self::assertEquals(5, $message->getValue('in_store'));
+
+        self::assertTrue($message->hasValueBoolChanged('delivery_time'));
+        self::assertTrue($message->startsWithChanged('in'));
+
+        self::assertEquals([], $message->getPreviousValues());
+        self::assertNull($message->getPreviousValue('in_store'));
+    }
+
+    public function testProductsVariationsDelete()
+    {
+        $message = $this->dbMessageFactory->create($this->createMessageFromTestData($this->testMessages['variationsDelete0']));
+
+        self::assertEquals(DBChangeTypeEnum::DELETE, $message->getEventType());
+        self::assertEquals('products_variations', $message->getTableName());
+
+        self::assertNotNull($message->getValues());
+        self::assertArraySubset($message->getChangedValues(), $message->getValues());
+
+        self::assertTrue($message->hasValueChanged('in_store'));
+        self::assertTrue($message->hasValueChanged('id'));
+        self::assertTrue($message->hasValueChanged('title'));
+
+        self::assertTrue($message->didAnyOfFieldChange(['title', 'in_store']));
+
+        self::assertEquals(5, $message->getValue('in_store'));
+
+        self::assertTrue($message->hasValueBoolChanged('delivery_time'));
+        self::assertTrue($message->startsWithChanged('in'));
+
+        self::assertEquals([], $message->getPreviousValues());
+        self::assertNull($message->getPreviousValue('in_store'));
+    }
+
+    private function createMessageFromTestData(array $data)
+    {
+        $message = new Message();
+
+        $message->payload = json_encode($data['payload']);
+        $message->headers = [];
+        $message->key = 'key';
+        $message->offset = 1;
+        $message->timestamp = time() * 1000;
+        $message->len = strlen($message->payload);
+        $message->topic_name = 'test';
+        $message->err = 0;
+        $message->partition = 0;
+
+        return $message;
+    }
+}

bundles/KupShop/KafkaBundle/Tests/dbMessagesTestPayloads.json

@@ -0,0 +1,122 @@
+{
+  "variationsUpdate0" : {
+    "payload" : {
+      "op": "u",
+      "cdc": "dbz",
+      "before": {
+        "in_store" : 312,
+        "updated" : "2025-04-28T10:52:10Z",
+        "ean" : "4321",
+        "delivery_time" : 1,
+        "price" : 128.1492
+      },
+      "after": {
+        "id_product" : 99732,
+        "in_store" : 309,
+        "delivery_time" : 0,
+        "ean" : "1234",
+        "price" : 128.1492,
+        "updated" : "2025-04-29T10:52:10Z"
+      },
+      "source": {
+        "server_id": 1,
+        "sequence": null,
+        "pos": 5247,
+        "gtid": "0-1-187892",
+        "row": 0,
+        "ts_ms": 1745923415000,
+        "db": "kupshop_kupkolo",
+        "table": "products_variations"
+      },
+      "ts_ms": 1745923415872
+    }
+  },
+  "variationsInsert0" : {
+    "payload" : {
+      "op": "c",
+      "cdc": "dbz",
+      "before": null,
+      "after": {
+        "price_buy" : 0,
+        "note" : "",
+        "code" : "545646",
+        "data" : null,
+        "delivery_time" : -2,
+        "title" : null,
+        "in_store_show_max" : 5,
+        "price_common" : null,
+        "ean" : null,
+        "price" : 486.1449,
+        "id" : 1785440,
+        "in_store_min" : null,
+        "height" : null,
+        "figure" : "Y",
+        "weight" : 3,
+        "price_orig" : 0,
+        "date_added" : 0,
+        "id_product" : 103712,
+        "depth" : null,
+        "price_for_discount" : null,
+        "in_store" : 5,
+        "width" : null,
+        "in_store_updated" : null,
+        "updated" : "2025-04-29T10:52:10Z"
+      },
+      "source": {
+        "server_id": 1,
+        "sequence": null,
+        "pos": 8774,
+        "gtid": "0-1-187894",
+        "row": 0,
+        "ts_ms": 1745923628000,
+        "db": "kupshop_zbraneesako",
+        "table": "products_variations"
+      },
+      "ts_ms": 1745923628259
+    }
+  },
+  "variationsDelete0" : {
+    "payload" : {
+      "op": "d",
+      "cdc": "dbz",
+      "before": {
+        "price_buy": 0,
+        "note": "",
+        "code": "545646",
+        "data": null,
+        "delivery_time": -2,
+        "title": "Délka: 30 mm",
+        "in_store_show_max": 5,
+        "price_common": null,
+        "ean": null,
+        "price": 486.1449,
+        "id": 1785440,
+        "in_store_min": null,
+        "height": null,
+        "figure": "Y",
+        "weight": 3,
+        "price_orig": 0,
+        "date_added": 0,
+        "id_product": 103712,
+        "depth": null,
+        "price_for_discount": 69.545500000000004,
+        "in_store": 5,
+        "width": null,
+        "in_store_updated": null,
+        "updated": "2025-04-29T10:52:10Z"
+      },
+      "after": null,
+      "source": {
+        "server_id": 1,
+        "sequence": null,
+        "pos": 83511,
+        "gtid": "0-1-187948",
+        "row": 0,
+        "ts_ms": 1745926153000,
+        "db": "kupshop_zbraneesako",
+        "table": "products_variations"
+      },
+      "ts_ms": 1745926153519
+    }
+  }
+}

bundles/KupShop/KafkaBundle/Util/KafkaAdminUtil.php

@@ -0,0 +1,71 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Util;
+
+use KupShop\KafkaBundle\Connection\KafkaConnection;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+class KafkaAdminUtil
+{
+    use KafkaUtilTrait;
+    private KafkaConnection $connection;
+    public \RdKafka\KafkaConsumer $rdConsumer;
+
+    public function __construct(
+        #[Autowire('%kupshop.kafka.brokers%')] private readonly array $brokers,
+        protected HttpClientInterface $httpClient,
+    ) {
+    }
+
+    public function createConnection($config = [])
+    {
+        $config = array_merge([
+            'transport_name' => 'admin',
+            'metadata.broker.list' => implode(',', $this->brokers),
+            'group.id' => $this->getShopGroupId('admin'),
+        ], $config);
+
+        $this->connection = \KupShop\KafkaBundle\Connection\KafkaConnection::builder($config);
+        $this->rdConsumer = $this->connection->createConsumer();
+    }
+
+    public function getExistingTopics()
+    {
+        $this->createConnection();
+
+        $metadata = $this->rdConsumer->getMetadata(true, null, 1000);
+
+        $topics = [];
+        foreach ($metadata->getTopics() as $topic) {
+            $topics[] = $topic->getTopic();
+        }
+
+        return $topics;
+    }
+
+    public function dbChangesTopicExists()
+    {
+        return in_array($this->getShopTopicName(), $this->getExistingTopics());
+    }
+
+    public function createDbChangesTopic()
+    {
+        if (!isRunningOnCluster()) {
+            return;
+        }
+
+        try {
+            $response = $this->httpClient->request('GET', 'http://rest-proxy.kafka/v3/clusters');
+            $content = $response->toArray();
+
+            $clusterId = $content['data'][0]['cluster_id'];
+            $this->httpClient->request('POST', "http://rest-proxy.kafka/v3/clusters/{$clusterId}/topics", [
+                'json' => ['topic_name' => $this->getShopTopicName(), 'partitions_count' => 1, 'replication_factor' => 1],
+            ]);
+        } catch (\Throwable $exception) {
+        }
+    }
+}

bundles/KupShop/KafkaBundle/Util/KafkaConsumer.php

@@ -0,0 +1,164 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Util;
+
+use KupShop\KafkaBundle\Connection\KafkaConnection;
+use KupShop\KafkaBundle\Connection\KafkaConnectionConfig;
+use KupShop\KafkaBundle\Exception\KafkaException;
+use RdKafka\TopicPartition;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+
+class KafkaConsumer
+{
+    use KafkaUtilTrait;
+    private KafkaConnection $connection;
+    public \RdKafka\KafkaConsumer $rdConsumer;
+
+    public function __construct(
+        #[Autowire('%kupshop.kafka.brokers%')] private readonly array $brokers,
+    ) {
+    }
+
+    public function createConnection(KafkaConnectionConfig $config): void
+    {
+        $config = array_merge([
+            'transport_name' => $config->getGroupId().': ['.implode(',', $config->getTopics()).']',
+            'metadata.broker.list' => implode(',', $this->brokers),
+            'consumer_topics' => $config->getTopics(),
+            'enable.partition.eof' => 'true',
+            'auto.offset.reset' => 'earliest',
+            'group.id' => $this->getShopGroupId($config->getGroupId()),
+        ], $config->getConfig());
+
+        $this->connection = \KupShop\KafkaBundle\Connection\KafkaConnection::builder($config);
+        $this->rdConsumer = $this->connection->createConsumer();
+    }
+
+    public function subscribe()
+    {
+        $this->rdConsumer->subscribe($this->connection->getTopics());
+    }
+
+    /**
+     * @return iterable<\RdKafka\Message>
+     */
+    public function consumeAll(?int $limit = null): iterable
+    {
+        $count = 0;
+
+        while ($message = $this->consumeOne()) {
+            yield $message;
+            $count++;
+            if (is_numeric($limit) && $count >= $limit) {
+                break;
+            }
+        }
+    }
+
+    public function consumeOne(): ?\RdKafka\Message
+    {
+        try {
+            $kafkaMessage = $this->rdConsumer->consume($this->connection->getConsumerConsumeTimeout());
+        } catch (\RdKafka\Exception $exception) {
+            throw new KafkaException($exception->getMessage(), 0, $exception);
+        }
+
+        if (RD_KAFKA_RESP_ERR_NO_ERROR !== $kafkaMessage->err) {
+            switch ($kafkaMessage->err) {
+                case RD_KAFKA_RESP_ERR__PARTITION_EOF: // No more messages
+                case RD_KAFKA_RESP_ERR__TIMED_OUT: // Attempt to connect again
+                    return null;
+                default:
+                    throw new KafkaException($kafkaMessage->errstr(), $kafkaMessage->err);
+            }
+        }
+
+        return $kafkaMessage;
+    }
+
+    public function processDbChangesMessages(KafkaConnectionConfig $config, callable $callback, ?int $limit = null): void
+    {
+        foreach ($this->getDbChangesMessages($config, $limit) as $message) {
+            $callback($message);
+        }
+    }
+
+    public function getDbChangesMessages(KafkaConnectionConfig $config, ?int $limit = null): iterable
+    {
+        return $this->withConsumer($config, function () use ($limit): iterable {
+            return $this->consumeAll($limit);
+        });
+    }
+
+    /**
+     * @template T
+     *
+     * @param callable(KafkaConsumer): T $fn
+     *
+     * @return T
+     */
+    public function withConsumer(KafkaConnectionConfig $config, callable $fn): mixed
+    {
+        return $this->withConnection($config, function () use ($fn) {
+            $this->subscribe();
+
+            return $fn($this);
+        });
+    }
+
+    /**
+     * @template T
+     *
+     * @param callable(KafkaConsumer): T $fn
+     *
+     * @return T
+     */
+    public function withConnection(KafkaConnectionConfig $config, callable $fn): mixed
+    {
+        $this->createConnection($config);
+
+        return $fn($this);
+    }
+
+    public function setConsumerOffset(KafkaConnectionConfig $config, int $offset, bool $allowOnlyHigher = false): bool
+    {
+        return $this->withConnection($config, function () use ($config, $offset, $allowOnlyHigher) {
+            $nextOffset = $offset + 1;
+            if ($allowOnlyHigher && ($commited = $this->rdConsumer->getCommittedOffsets([new \RdKafka\TopicPartition($config->getTopic(), 0)], 1000))) {
+                // commit is not allowed when last consumer commited offset is higher than next
+                if (($commited[0]->getOffset() ?? 0) > $nextOffset) {
+                    return false;
+                }
+            }
+
+            $this->rdConsumer->commit([new \RdKafka\TopicPartition($config->getTopic(), 0, $nextOffset)]);
+
+            return true;
+        });
+    }
+
+    public function setConsumerOffsetByTimestamp(KafkaConnectionConfig $config, int $timestamp): bool
+    {
+        return $this->withConnection($config, function () use ($config, $timestamp) {
+            // Convert timestamp to milliseconds (if necessary)
+            $timestampMs = $timestamp * 1000;
+
+            $topicPartition = [new \RdKafka\TopicPartition($config->getTopic(), 0, $timestampMs)];
+            // Fetch offsets for the specified timestamp
+            $offsets = $this->rdConsumer->offsetsForTimes($topicPartition, 1000);
+
+            // Assign and seek the consumer to the calculated offsets
+            $validOffsets = array_filter($offsets, fn (TopicPartition $offset) => $offset->getOffset() !== -1);
+
+            if (empty($validOffsets)) {
+                return false;
+            }
+
+            $this->rdConsumer->commit($validOffsets);
+
+            return true;
+        });
+    }
+}

bundles/KupShop/KafkaBundle/Util/KafkaUtilTrait.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KupShop\KafkaBundle\Util;
+
+use KupShop\KupShopBundle\Config;
+
+trait KafkaUtilTrait
+{
+    public function getShopTopicName(string $topic): string
+    {
+        $dbName = Config::get()['Connection']['database'];
+
+        if (isLocalDevelopment()) {
+            return $dbName;
+        }
+
+        return $topic.'_'.$dbName;
+    }
+
+    public function getShopGroupId(string $groupId): string
+    {
+        return Config::get()['Connection']['database'].'-'.$groupId;
+    }
+}