365 lines
13 KiB
PHP
365 lines
13 KiB
PHP
<?php
|
|
|
|
namespace KupShop\FeedGeneratorBundle\Feed;
|
|
|
|
use Doctrine\Common\Collections\ArrayCollection;
|
|
use KupShop\AdminBundle\Util\ActivityLog;
|
|
use KupShop\FeedGeneratorBundle\Configuration\Configuration;
|
|
use KupShop\FeedGeneratorBundle\Exception\XMLFeedReader\XMLFeedReaderException;
|
|
use KupShop\FeedGeneratorBundle\Exception\XSDParser\XSDParsingException;
|
|
use KupShop\FeedGeneratorBundle\Expressions\ExpressionInterpreterException;
|
|
use KupShop\FeedGeneratorBundle\V8ErrorException;
|
|
use KupShop\FeedGeneratorBundle\V8FeedGenerator;
|
|
use KupShop\FeedsBundle\FeedsBundle;
|
|
use KupShop\FeedsBundle\Wrapper\BaseWrapper;
|
|
use KupShop\KupShopBundle\Config;
|
|
use KupShop\KupShopBundle\Context\CountryContext;
|
|
use KupShop\KupShopBundle\Context\CurrencyContext;
|
|
use KupShop\KupShopBundle\Context\DomainContext;
|
|
use KupShop\KupShopBundle\Context\LanguageContext;
|
|
use KupShop\KupShopBundle\Util\Contexts;
|
|
|
|
trait ConfigurableFeedTrait
|
|
{
|
|
use \DatabaseCommunication;
|
|
|
|
/** @var int */
|
|
protected static $batchSize = 1000;
|
|
|
|
/** @var float|int ttl for cache */
|
|
protected $ttl = 60 * 60 * 4;
|
|
|
|
/** @var V8FeedGenerator */
|
|
protected $feedGenerator;
|
|
|
|
/** @var V8FeedGenerator */
|
|
protected $v8FeedGenerator;
|
|
|
|
/** @var V8FeedGenerator */
|
|
protected $forcedFeedGenerator;
|
|
|
|
/** @var BaseWrapper */
|
|
protected $objectWrapper;
|
|
|
|
/** @var array */
|
|
protected $specs = [];
|
|
|
|
/**
|
|
* @required
|
|
*/
|
|
public function setV8FeedGenerator(?V8FeedGenerator $v8FeedGenerator = null): void
|
|
{
|
|
$this->v8FeedGenerator = $v8FeedGenerator;
|
|
}
|
|
|
|
/** Set forced feedGenerator: do not rely on custom data enableV8 setting */
|
|
public function setFeedGenerator($forcedFeedGenerator = null): void
|
|
{
|
|
$this->forcedFeedGenerator = $forcedFeedGenerator;
|
|
}
|
|
|
|
public static function getType(): string
|
|
{
|
|
return static::$type;
|
|
}
|
|
|
|
public static function getAlias(): string
|
|
{
|
|
return static::$alias;
|
|
}
|
|
|
|
public function getObjectType(): string
|
|
{
|
|
return static::$objectType;
|
|
}
|
|
|
|
public function getTTL(): int
|
|
{
|
|
return $this->ttl;
|
|
}
|
|
|
|
protected function prepareSingleObject($singleObject)
|
|
{
|
|
return $this->objectWrapper->setObject($singleObject);
|
|
}
|
|
|
|
public function render(array $feedRow, ?int $limit = null): void
|
|
{
|
|
$this->initFeedGenerator($feedRow);
|
|
|
|
$this->feedGenerator->setThrowErrors(false);
|
|
if ($feedRow['pretty'] ?? false) {
|
|
$this->feedGenerator->setIndent(true);
|
|
}
|
|
|
|
try {
|
|
$this->renderCommon($feedRow, $limit);
|
|
} catch (ExpressionInterpreterException|V8ErrorException $e) {
|
|
$this->handleV8GeneratorException($e, $feedRow, $limit);
|
|
$this->productList = null;
|
|
|
|
return;
|
|
} catch (XSDParsingException|XMLFeedReaderException $e) {
|
|
$this->handleFeedException($e, $feedRow, $limit);
|
|
}
|
|
|
|
if (!$limit) {
|
|
$this->updateFeedReport($feedRow, $this->feedGenerator->getReport());
|
|
}
|
|
|
|
$this->productList = null;
|
|
}
|
|
|
|
public function renderStrict(array $feedRow, ?int $limit = null)
|
|
{
|
|
$this->initFeedGenerator($feedRow);
|
|
|
|
$this->feedGenerator->setThrowErrors();
|
|
$this->feedGenerator->setIsPreview();
|
|
$this->renderCommon($feedRow, $limit);
|
|
}
|
|
|
|
public function getReport(): array
|
|
{
|
|
return $this->feedGenerator->getReport();
|
|
}
|
|
|
|
public function renderWithGenerator(array $feedRow, ?int $limit = null): \Generator
|
|
{
|
|
$this->initFeedGenerator($feedRow);
|
|
|
|
$this->feedGenerator->setThrowErrors(false);
|
|
if ($feedRow['pretty'] ?? false) {
|
|
$this->feedGenerator->setIndent(true);
|
|
}
|
|
|
|
try {
|
|
return $this->withFeedGeneratorConfigured($feedRow, $limit, function (Configuration $configuration, array $placeholders, \Generator $data) use ($feedRow, $limit): \Generator {
|
|
yield 'start' => str_replace(array_keys($placeholders), array_values($placeholders), $configuration->getBeginning()).PHP_EOL;
|
|
yield from $this->feedGenerator->transformWithGenerator($data);
|
|
yield 'end' => $configuration->getEnd();
|
|
|
|
if (!$limit) {
|
|
$this->updateFeedReport($feedRow, $this->feedGenerator->getReport());
|
|
}
|
|
|
|
$this->productList = null;
|
|
});
|
|
} catch (ExpressionInterpreterException|V8ErrorException $e) {
|
|
$this->handleV8GeneratorException($e, $feedRow, $limit);
|
|
$this->productList = null;
|
|
} catch (XSDParsingException|XMLFeedReaderException $e) {
|
|
$this->handleFeedException($e, $feedRow, $limit);
|
|
}
|
|
}
|
|
|
|
private function renderCommon(array $feedRow, ?int $limit = null)
|
|
{
|
|
$this->withFeedGeneratorConfigured($feedRow, $limit, function (Configuration $configuration, array $placeholders, \Generator $data) {
|
|
echo str_replace(array_keys($placeholders), array_values($placeholders), $configuration->getBeginning()).PHP_EOL;
|
|
$this->feedGenerator->transform($data);
|
|
echo $configuration->getEnd();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param callable(Configuration, array, \Generator): mixed $generatorFn
|
|
*/
|
|
private function withFeedGeneratorConfigured(array $feedRow, ?int $limit, callable $generatorFn): mixed
|
|
{
|
|
$this->objectWrapper->setFeedRow($feedRow);
|
|
$this->feedGenerator->setWrapper($this->objectWrapper); // support multiple custom wrappers
|
|
$this->feedGenerator->setConfiguration($feedRow['template']);
|
|
$configuration = $this->feedGenerator->getConfiguration();
|
|
|
|
$data = $this->getData($feedRow, $limit);
|
|
|
|
return $generatorFn($configuration, $this->getBeginningPlaceholders(), $data);
|
|
}
|
|
|
|
private function getBeginningPlaceholders(): array
|
|
{
|
|
$cfg = Config::get();
|
|
$dbcfg = \Settings::getDefault();
|
|
|
|
return [
|
|
'{timestamp}' => time(),
|
|
'{print_url}' => trim($cfg['Addr']['print'], '/'),
|
|
'{url}' => $cfg['Addr']['full'],
|
|
'{shop_name}' => $dbcfg['shop_firm_name'],
|
|
'{currency}' => Contexts::get(CurrencyContext::class)->getActiveId(),
|
|
'{language}' => Contexts::get(LanguageContext::class)->getActiveId(),
|
|
'{language_locale}' => Contexts::get(LanguageContext::class)->getActive()->getLocale(),
|
|
'{datetime_iso}' => (new \DateTime())->format('Y-m-d\TH:i:s.v'),
|
|
];
|
|
}
|
|
|
|
private function handleV8GeneratorException(\Throwable $e, array $feedRow, ?int $limit = null): void
|
|
{
|
|
if (getAdminUser() && $limit) {
|
|
echo $e->getMessage();
|
|
|
|
return;
|
|
}
|
|
|
|
if (!$limit) {
|
|
$this->updateSQL('feeds',
|
|
[
|
|
'report' => json_encode(['errors' => ['Script error' => $e->getMessage()]]),
|
|
],
|
|
['id' => $feedRow['id']]
|
|
);
|
|
|
|
// log errors related to v8js_generator_export.cc#L32 to sentry in full detail
|
|
if (mb_strpos($e->getMessage(), 'function(wrapped_object) { return (function*()') >= 0
|
|
|| mb_strpos($e->getMessage(), 'XMLFeedReaderException') >= 0
|
|
) {
|
|
\Sentry\captureException($e, \Sentry\EventHint::fromArray(['extra' => [
|
|
'memoryUsage' => memory_get_usage(true),
|
|
'memoryPeak' => memory_get_peak_usage(true),
|
|
'memoryLimit' => ini_get('memory_limit'),
|
|
'feedRow_id' => $feedRow['id'],
|
|
'feedRow_name' => $feedRow['name'],
|
|
'feedRow_format' => $feedRow['data']['format'] ?? '',
|
|
'feedRow_source_url' => $feedRow['data']['feed_source_url'] ?? '',
|
|
'feedRow_description_url' => $feedRow['data']['feed_description_url'] ?? '',
|
|
]]));
|
|
}
|
|
addActivityLog(
|
|
ActivityLog::SEVERITY_ERROR,
|
|
ActivityLog::TYPE_COMMUNICATION,
|
|
sprintf('Feed: Generování feedu se nezdařilo - "%s"', $feedRow['name']),
|
|
[
|
|
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
|
|
...['error' => $e->getMessage()],
|
|
],
|
|
[FeedsBundle::LOG_TAG_FEED]
|
|
);
|
|
}
|
|
}
|
|
|
|
private function handleFeedException(\Throwable $e, array $feedRow, ?int $limit = null): void
|
|
{
|
|
if (!($e instanceof XSDParsingException) && !($e instanceof XMLFeedReaderException)) {
|
|
throw $e;
|
|
}
|
|
|
|
if (getAdminUser() && $limit) {
|
|
echo $e->getMessage();
|
|
|
|
return;
|
|
}
|
|
|
|
throw $e;
|
|
}
|
|
|
|
protected function initFeedGenerator(array $feedRow): void
|
|
{
|
|
if (isset($this->forcedFeedGenerator)) {
|
|
$this->feedGenerator = $this->forcedFeedGenerator;
|
|
} else {
|
|
$this->feedGenerator = $this->v8FeedGenerator;
|
|
}
|
|
|
|
if (method_exists($this->feedGenerator, 'setBuildItemCallback')) {
|
|
$this->feedGenerator->setBuildItemCallback(null); // reset callback for pre-generated expensive feeds
|
|
}
|
|
}
|
|
|
|
public function getData(array $feedRow, ?int $limit = null): \Generator
|
|
{
|
|
$qb = $this->query ?? $this->getQuery();
|
|
$count = (int) (clone $qb)
|
|
->addSelect('COUNT(*)')
|
|
->execute()
|
|
->fetchOne();
|
|
|
|
// use batching only if limit is NOT set
|
|
$iterationLimit = isset($limit) ? 1 : ($count / (float) static::$batchSize);
|
|
for ($i = 0; $i < $iterationLimit; $i++) {
|
|
$qb->setFirstResult($i * static::$batchSize);
|
|
$qb->setMaxResults($limit ?? static::$batchSize);
|
|
$objects = new ArrayCollection();
|
|
foreach ($qb->execute() as $row) {
|
|
$objects->set($row['id'], (object) $row); // cast to (object) to force reference
|
|
}
|
|
$this->objectWrapper->setObjects($objects);
|
|
foreach ($objects as $producer) {
|
|
yield $this->prepareSingleObject($producer);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function updateFeedReport(array $feedRow, array $report): void
|
|
{
|
|
$previousReport = $updateReport = json_decode($feedRow['report'] ?: '', true) ?: [];
|
|
|
|
$feedNameKeys = $this->getReportContextsKeys($feedRow);
|
|
$updateReport = array_merge($updateReport, $report);
|
|
$feedName = implode('_', $feedNameKeys);
|
|
$updateReport[$feedName] = $report;
|
|
|
|
// update feed report
|
|
$this->updateSQL('feeds',
|
|
[
|
|
'report' => json_encode($updateReport),
|
|
],
|
|
['id' => $feedRow['id']]
|
|
);
|
|
|
|
$previousValidCount = $previousReport[$feedName]['validCount'] ?? null;
|
|
|
|
// report large drop of valid items to activity log
|
|
if ($previousValidCount !== null && $report['validCount'] !== null && ($previousValidCount - $report['validCount']) > 500) {
|
|
addActivityLog(
|
|
ActivityLog::SEVERITY_WARNING,
|
|
ActivityLog::TYPE_COMMUNICATION,
|
|
sprintf('Feed: Velký pokles položek ve feedu (%d -> %d) - "%s"', $previousValidCount, $report['validCount'], $feedRow['name']),
|
|
[
|
|
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
|
|
...[
|
|
'previousValidCount' => $previousValidCount,
|
|
'validCount' => $report['validCount'],
|
|
'diff' => $previousValidCount - $report['validCount'],
|
|
'contexts' => $feedNameKeys,
|
|
],
|
|
],
|
|
[FeedsBundle::LOG_TAG_FEED]
|
|
);
|
|
}
|
|
|
|
// report errors in feed to activity log
|
|
if (!empty($report['errors'])) {
|
|
addActivityLog(
|
|
ActivityLog::SEVERITY_WARNING,
|
|
ActivityLog::TYPE_COMMUNICATION,
|
|
sprintf('Feed: Při generování feedu se u některých položek vyskytly chyby (%d) - "%s"', count($report['errors']), $feedRow['name']),
|
|
[
|
|
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
|
|
...[
|
|
'errorsCount' => count($report['errors']),
|
|
'errorsSample' => array_slice($report['errors'], 0, 15),
|
|
],
|
|
],
|
|
[FeedsBundle::LOG_TAG_FEED]
|
|
);
|
|
}
|
|
}
|
|
|
|
protected function getReportContextsKeys(array $feedRow): array
|
|
{
|
|
$domainContext = Contexts::get(DomainContext::class);
|
|
$countryContext = Contexts::get(CountryContext::class);
|
|
$currencyContext = Contexts::get(CurrencyContext::class);
|
|
$languageContext = Contexts::get(LanguageContext::class);
|
|
$feedName['index'] = 'feed';
|
|
$feedName['id'] = $feedRow['id'];
|
|
$feedName['language'] = $languageContext->getActiveId();
|
|
$feedName['currency'] = $currencyContext->getActiveId();
|
|
$feedName['country'] = $countryContext->getActiveId();
|
|
$feedName['domain'] = $domainContext->getActiveId();
|
|
|
|
return $feedName;
|
|
}
|
|
}
|