first commit
This commit is contained in:
@@ -0,0 +1,212 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace KupShop\SynchronizationBundle\Util\Rabbit;
|
||||
|
||||
use KupShop\KupShopBundle\Util\Logging\SentryLogger;
|
||||
use KupShop\SynchronizationBundle\Exception\RabbitRetryMessageException;
|
||||
use KupShop\SynchronizationBundle\Rabbit\RabbitConsumerConfiguration;
|
||||
use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpStamp;
|
||||
use Symfony\Component\Messenger\Bridge\Amqp\Transport\Connection;
|
||||
|
||||
class RabbitConsumer
|
||||
{
|
||||
public function __construct(private readonly SentryLogger $sentryLogger)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @param callable(\AMQPEnvelope): bool $fn
|
||||
*/
|
||||
public function consume(RabbitConsumerConfiguration $configuration, callable $fn): bool
|
||||
{
|
||||
$connection = $this->createConnection($configuration);
|
||||
|
||||
if (!($messagesCount = $connection->countMessagesInQueues())) {
|
||||
$this->closeConnection($connection);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$moreMessages = true;
|
||||
$processedCount = 0;
|
||||
|
||||
$startTime = microtime(true);
|
||||
|
||||
// zacnu zpracovavat zmeny
|
||||
$this->processMessages($configuration, $connection, function (\AMQPEnvelope $envelope) use ($fn, $connection, $configuration, $startTime, $messagesCount, &$processedCount, &$moreMessages) {
|
||||
// pokud vyprsel limit pro zpracovani zmen, tak to ukoncim
|
||||
if ((microtime(true) - $startTime) >= $configuration->timeout) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$wasAckedEarly = false;
|
||||
// pokud ma na sobe message header s `x-ack-on-start`, tak provedu acknuti message hned na zacatku
|
||||
if ($envelope->hasHeader('x-ack-on-start')) {
|
||||
$connection->ack($envelope, $configuration->queue);
|
||||
$wasAckedEarly = true;
|
||||
}
|
||||
|
||||
// podle celkoveho poctu a zpracovaneho poctu odetekuju, zda ve fronte jsou jeste nejake messages, ktere jsem nezpracoval
|
||||
$moreMessages = ++$processedCount < $messagesCount;
|
||||
|
||||
try {
|
||||
$result = $fn($envelope);
|
||||
} catch (RabbitRetryMessageException) {
|
||||
$this->handleRetryMessage($envelope, $configuration, $connection);
|
||||
|
||||
return $moreMessages;
|
||||
} catch (\Throwable $e) {
|
||||
if (isLocalDevelopment()) {
|
||||
throw $e;
|
||||
}
|
||||
|
||||
$this->sentryLogger->captureException($e, [
|
||||
'extra' => [
|
||||
'body' => $envelope->getBody(),
|
||||
],
|
||||
]);
|
||||
|
||||
$result = false;
|
||||
}
|
||||
|
||||
// pokud mi umrela connectiona, tak zastavim consume, protoze stejnak nefunguje ackovani atd... kdyz jsem uz odpojenej
|
||||
if (!$this->isConnectionAlive($connection)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// pokud byla message uz acknuta, tak vracim vysledek, protoze logika dole uz neni potreba
|
||||
if ($wasAckedEarly) {
|
||||
return $moreMessages;
|
||||
}
|
||||
|
||||
// pokud nastala chyba a nebo jsem na locale, tak message vratim zpatky do fronty
|
||||
if (!$result || isLocalDevelopment()) {
|
||||
$connection->nack($envelope, $configuration->queue, AMQP_REQUEUE);
|
||||
|
||||
return $moreMessages;
|
||||
}
|
||||
|
||||
// vse probehlo uspesne, takze provedu potvrzeni message
|
||||
$connection->ack($envelope, $configuration->queue);
|
||||
|
||||
return $moreMessages;
|
||||
});
|
||||
|
||||
$this->closeConnection($connection);
|
||||
|
||||
return $moreMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Message processing handler.
|
||||
*
|
||||
* Handler is processing all queues that are configured by configuration (priority queues).
|
||||
*/
|
||||
private function processMessages(RabbitConsumerConfiguration $configuration, Connection $connection, callable $fn): void
|
||||
{
|
||||
$processNext = true;
|
||||
|
||||
// multiple queues can be configured when using priorities, so we need to iterate all of them
|
||||
foreach ($configuration->getQueues() as $options) {
|
||||
// process should be stopped because $fn told us
|
||||
if (!$processNext) {
|
||||
break;
|
||||
}
|
||||
|
||||
// no messages in queue, so we can skip it
|
||||
if (!($messagesCount = $connection->queue($options['queueName'])->declareQueue())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$processedCount = 0;
|
||||
|
||||
// start queue consuming
|
||||
$connection->queue($options['queueName'])->consume(function (\AMQPEnvelope $envelope) use ($fn, $messagesCount, &$processedCount, &$processNext) {
|
||||
if (!$fn($envelope)) {
|
||||
$processNext = false;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// no more messages in queue, so return false to stop consume
|
||||
return ++$processedCount < $messagesCount;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private function closeConnection(Connection $connection): void
|
||||
{
|
||||
$amqpConnection = $connection->channel()->getConnection();
|
||||
|
||||
try {
|
||||
$connection->channel()->close();
|
||||
} catch (\Exception) {
|
||||
}
|
||||
|
||||
try {
|
||||
$amqpConnection->disconnect();
|
||||
} catch (\Exception) {
|
||||
}
|
||||
}
|
||||
|
||||
private function handleRetryMessage(\AMQPEnvelope $envelope, RabbitConsumerConfiguration $configuration, Connection $connection): void
|
||||
{
|
||||
// pokud je nastavena retry strategy, tak ji aplikuji
|
||||
if ($configuration->retryStrategy) {
|
||||
// pokud ji chci jeste opakovat, tak ji znovu poslu do delay fronty odkud se potom vrati do
|
||||
// normlani fronty a zkusi se zpracovat znovu
|
||||
if ($configuration->retryStrategy->isRetryable($envelope)) {
|
||||
$connection->publish(
|
||||
$envelope->getBody(),
|
||||
['x-retry-count' => ($envelope->getHeaders()['x-retry-count'] ?? 0) + 1],
|
||||
$configuration->retryStrategy->getWaitingTime($envelope),
|
||||
// retry message should be routed to default queue when returning from delay queue (retry message becomes unprioritized)
|
||||
AmqpStamp::createFromAmqpEnvelope($envelope, retryRoutingKey: $configuration->getDefaultQueue())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// potvrdim message, aby zmizela z fronty
|
||||
$connection->ack($envelope, $configuration->queue);
|
||||
}
|
||||
|
||||
private function createConnection(RabbitConsumerConfiguration $configuration): Connection
|
||||
{
|
||||
$options = [
|
||||
'auto_setup' => false,
|
||||
'queues' => $configuration->getQueuesConfiguration(),
|
||||
];
|
||||
|
||||
if ($configuration->hasPriorityQueuesWithExchange()) {
|
||||
$options['exchange'] = [
|
||||
'name' => $configuration->priorityOptions['exchange'],
|
||||
'type' => 'direct',
|
||||
];
|
||||
}
|
||||
|
||||
$connection = Connection::fromDsn(
|
||||
rtrim($configuration->dsn, '/').'/'.$configuration->queue,
|
||||
$options
|
||||
);
|
||||
|
||||
// priority options are configured, so setup priority queues with exchange
|
||||
if ($configuration->hasPriorityQueuesWithExchange()) {
|
||||
$connection->exchange()->declareExchange();
|
||||
foreach ($configuration->getQueues() as $config) {
|
||||
$connection->queue($config['queueName'])->declareQueue();
|
||||
$connection->queue($config['queueName'])->bind($configuration->priorityOptions['exchange'], $config['priorityName']);
|
||||
}
|
||||
}
|
||||
|
||||
$connection->channel()->setPrefetchCount($configuration->prefetchCount);
|
||||
|
||||
return $connection;
|
||||
}
|
||||
|
||||
private function isConnectionAlive(Connection $connection): bool
|
||||
{
|
||||
return $connection->channel()->isConnected();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace KupShop\SynchronizationBundle\Util;
|
||||
|
||||
use KupShop\SynchronizationBundle\Exception\SynchronizationConfigException;
|
||||
|
||||
class SynchronizationConfiguration
|
||||
{
|
||||
private array $configLoader = [];
|
||||
|
||||
public function set(string $key, callable $loader, int $ttl = 3600): void
|
||||
{
|
||||
$this->configLoader[$key] = [
|
||||
'loader' => $loader,
|
||||
'ttl' => $ttl,
|
||||
];
|
||||
}
|
||||
|
||||
public function has(string $key): bool
|
||||
{
|
||||
return isset($this->configLoader[$key]);
|
||||
}
|
||||
|
||||
public function get(string $key)
|
||||
{
|
||||
if (!$this->has($key)) {
|
||||
throw new SynchronizationConfigException(sprintf('Key "%s" does not exists ibn configuration!', $key));
|
||||
}
|
||||
|
||||
if (!($config = getCache($this->getCacheKey($key)))) {
|
||||
// naloaduju config
|
||||
$config = $this->configLoader[$key]['loader']();
|
||||
// zacahuje config
|
||||
setCache($this->getCacheKey($key), $config, $this->configLoader[$key]['ttl']);
|
||||
}
|
||||
|
||||
return $config;
|
||||
}
|
||||
|
||||
public function clear(string $key): bool
|
||||
{
|
||||
if ($this->has($key) || getCache($this->getCacheKey($key))) {
|
||||
clearCache($this->getCacheKey($key));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function getCacheKey(string $key): string
|
||||
{
|
||||
return 'synchronizationConfig_'.$key;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace KupShop\SynchronizationBundle\Util;
|
||||
|
||||
use KupShop\SynchronizationBundle\Synchronization\Job;
|
||||
use Symfony\Component\DependencyInjection\Attribute\TaggedLocator;
|
||||
use Symfony\Component\DependencyInjection\ServiceLocator;
|
||||
|
||||
class SynchronizationJobProvider
|
||||
{
|
||||
/** @var Job[] */
|
||||
private ?array $jobs = null;
|
||||
|
||||
public function __construct(
|
||||
#[TaggedLocator(tag: 'kupshop.synchronization.register')] protected ServiceLocator $registerLocator,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Job[]
|
||||
*/
|
||||
public function getJobs(): array
|
||||
{
|
||||
if ($this->jobs === null) {
|
||||
$this->jobs = [];
|
||||
|
||||
foreach ($this->registerLocator->getProvidedServices() as $serviceId => $classId) {
|
||||
foreach ($this->registerLocator->get($serviceId)->getJobs() as $job) {
|
||||
$this->jobs[$job->id] = $job;
|
||||
}
|
||||
}
|
||||
|
||||
uasort($this->jobs, fn ($a, $b) => $b->priority <=> $a->priority);
|
||||
}
|
||||
|
||||
return $this->jobs;
|
||||
}
|
||||
|
||||
public function getJob(string $id): ?Job
|
||||
{
|
||||
return $this->getJobs()[$id] ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace KupShop\SynchronizationBundle\Util;
|
||||
|
||||
use KupShop\KupShopBundle\Util\Logging\SentryLogger;
|
||||
use KupShop\SynchronizationBundle\Logger\SynchronizationLogger;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
class SynchronizationLoggerFactory
|
||||
{
|
||||
public function __construct(
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly SentryLogger $sentryLogger,
|
||||
) {
|
||||
}
|
||||
|
||||
public function create(array $activityLogExceptions = [], ?string $logPrefix = null): SynchronizationLogger
|
||||
{
|
||||
$logger = new SynchronizationLogger(
|
||||
$this->logger,
|
||||
$this->sentryLogger
|
||||
);
|
||||
|
||||
return $logger
|
||||
->withLogPrefix($logPrefix)
|
||||
->withActivityLogExceptions($activityLogExceptions);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace KupShop\SynchronizationBundle\Util;
|
||||
|
||||
use Symfony\Component\Console\Output\NullOutput;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Contracts\Service\Attribute\Required;
|
||||
|
||||
class SynchronizationQueueUtil
|
||||
{
|
||||
protected SynchronizationJobProvider $jobProvider;
|
||||
|
||||
public const CACHE_KEY = 'synchronizationQueue';
|
||||
public const CACHE_KEY_JOB_STATUSES = 'jobStatuses';
|
||||
|
||||
public const STATUS_RUNNING = 'running';
|
||||
public const STATUS_SUCCESS = 'success';
|
||||
public const STATUS_FAILED = 'failed';
|
||||
public const CACHE_TTL = 86400;
|
||||
|
||||
public function addToQueue(string $jobId): void
|
||||
{
|
||||
$queue = $this->getQueue();
|
||||
$queue[$jobId] = $jobId;
|
||||
$this->setQueue($queue);
|
||||
}
|
||||
|
||||
public function processOne(?OutputInterface $output = null): bool
|
||||
{
|
||||
$jobId = $this->getJob();
|
||||
if (!$jobId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$job = $this->jobProvider->getJob($jobId);
|
||||
if (!$job) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$this->setJobStatus($jobId, self::STATUS_RUNNING);
|
||||
($job->fn)($output ?: new NullOutput());
|
||||
} catch (\Exception $e) {
|
||||
$this->setJobStatus($jobId, self::STATUS_FAILED, $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
$this->setJobStatus($jobId, self::STATUS_SUCCESS);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function processAll(): void
|
||||
{
|
||||
while (!empty($this->getQueue())) {
|
||||
$this->processOne();
|
||||
}
|
||||
}
|
||||
|
||||
public function isQueued(string $jobId): bool
|
||||
{
|
||||
return in_array($jobId, $this->getQueue());
|
||||
}
|
||||
|
||||
protected function getQueue(): array
|
||||
{
|
||||
return getCache(self::CACHE_KEY) ?: [];
|
||||
}
|
||||
|
||||
protected function setQueue(array $queue): void
|
||||
{
|
||||
setCache(self::CACHE_KEY, $queue);
|
||||
}
|
||||
|
||||
protected function getJob(): ?string
|
||||
{
|
||||
$queue = $this->getQueue();
|
||||
$job = array_shift($queue);
|
||||
$this->setQueue($queue);
|
||||
|
||||
return $job;
|
||||
}
|
||||
|
||||
public function setJobStatus(string $jobId, string $status, string $msg = ''): void
|
||||
{
|
||||
$statuses = $this->getJobStatuses();
|
||||
$currentStatus = $statuses[$jobId] ?? [];
|
||||
|
||||
$currentStatus['status'] = $status;
|
||||
$currentStatus['msg'] = $msg;
|
||||
if ($status === self::STATUS_RUNNING) {
|
||||
$currentStatus['time'] = time();
|
||||
}
|
||||
$statuses[$jobId] = $currentStatus;
|
||||
|
||||
$this->setJobStatuses($statuses);
|
||||
}
|
||||
|
||||
protected function setJobStatuses(array $statuses): void
|
||||
{
|
||||
setCache(self::CACHE_KEY_JOB_STATUSES, $statuses, self::CACHE_TTL);
|
||||
}
|
||||
|
||||
public function getJobStatuses(): array
|
||||
{
|
||||
return getCache(self::CACHE_KEY_JOB_STATUSES) ?: [];
|
||||
}
|
||||
|
||||
public function getJobStatus(string $jobId): ?array
|
||||
{
|
||||
return $this->getJobStatuses()[$jobId] ?? null;
|
||||
}
|
||||
|
||||
#[Required]
|
||||
public function setJobProvider(SynchronizationJobProvider $jobProvider): void
|
||||
{
|
||||
$this->jobProvider = $jobProvider;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user