first commit

This commit is contained in:
2025-08-02 16:30:27 +02:00
commit 23646bfcee
14851 changed files with 1750626 additions and 0 deletions

View File

@@ -0,0 +1,86 @@
<?php
namespace KupShop\FeedGeneratorBundle\Feed;
use KupShop\FeedsBundle\Feed\BaseFeed;
use KupShop\FeedsBundle\FeedProductList;
use KupShop\FeedsBundle\Wrapper\ProductWrapper;
use Query\Operator;
use Query\QueryBuilder;
class ConfigurableFeed extends BaseFeed implements IConfigurableFeed
{
use \DatabaseCommunication;
use ConfigurableFeedTrait;
/** @var string */
public static $type = 'configurable';
/** @var string */
public static $alias = 'Varianty (výchozí)';
public static function isAllowed(): bool
{
return (bool) findModule(\Modules::PRODUCTS);
}
public function __construct(ProductWrapper $productWrapper)
{
$this->objectWrapper = $productWrapper;
}
protected function fetchImages(\ProductList $productList): void
{
$productList->fetchImages(102, null, true);
// additional images are fetched on demand
}
public function getProductList(int $feedID): FeedProductList
{
$productList = parent::getProductList($feedID);
foreach ($this->specs as $spec) {
$productList->andSpec($spec);
}
return $productList;
}
public function filterByObjectID($objectID): void
{
$this->specs[] = function (QueryBuilder $qb) use ($objectID) {
$qb->andWhere('p.id=:product_id')->setParameter('product_id', $objectID);
return '';
};
}
public function getData(array $feedRow, ?int $limit = null): \Generator
{
$templateProductList = $this->productList ?? $this->getProductList($feedRow['id']);
$queryProductIDs = $templateProductList->getQueryBuilder();
$queryProductIDs->select('p.id AS id')
->groupBy('p.id');
if (isset($limit)) {
$queryProductIDs->setMaxResults($limit);
}
foreach (array_chunk($queryProductIDs->execute()->fetchFirstColumn(), static::$batchSize) as $ids) {
$batchProductList = clone $templateProductList;
$batchProductList->andSpec(function (QueryBuilder $qb) use ($ids) {
$qb->andWhere(Operator::inIntArray($ids, 'p.id'));
});
// restrict the number of variations in configurator preview to 10 (configurator UI won't handle tens/hundreds of variations)
if (isset($limit) && $limit === 1) {
$batchProductList->limit(10);
}
$products = $batchProductList->getProducts();
$this->objectWrapper->setProductCollection($products);
foreach ($products as $product) {
yield $this->prepareSingleObject($product);
}
}
}
}

View File

@@ -0,0 +1,364 @@
<?php
namespace KupShop\FeedGeneratorBundle\Feed;
use Doctrine\Common\Collections\ArrayCollection;
use KupShop\AdminBundle\Util\ActivityLog;
use KupShop\FeedGeneratorBundle\Configuration\Configuration;
use KupShop\FeedGeneratorBundle\Exception\XMLFeedReader\XMLFeedReaderException;
use KupShop\FeedGeneratorBundle\Exception\XSDParser\XSDParsingException;
use KupShop\FeedGeneratorBundle\Expressions\ExpressionInterpreterException;
use KupShop\FeedGeneratorBundle\V8ErrorException;
use KupShop\FeedGeneratorBundle\V8FeedGenerator;
use KupShop\FeedsBundle\FeedsBundle;
use KupShop\FeedsBundle\Wrapper\BaseWrapper;
use KupShop\KupShopBundle\Config;
use KupShop\KupShopBundle\Context\CountryContext;
use KupShop\KupShopBundle\Context\CurrencyContext;
use KupShop\KupShopBundle\Context\DomainContext;
use KupShop\KupShopBundle\Context\LanguageContext;
use KupShop\KupShopBundle\Util\Contexts;
trait ConfigurableFeedTrait
{
use \DatabaseCommunication;
/** @var int */
protected static $batchSize = 1000;
/** @var float|int ttl for cache */
protected $ttl = 60 * 60 * 4;
/** @var V8FeedGenerator */
protected $feedGenerator;
/** @var V8FeedGenerator */
protected $v8FeedGenerator;
/** @var V8FeedGenerator */
protected $forcedFeedGenerator;
/** @var BaseWrapper */
protected $objectWrapper;
/** @var array */
protected $specs = [];
/**
* @required
*/
public function setV8FeedGenerator(?V8FeedGenerator $v8FeedGenerator = null): void
{
$this->v8FeedGenerator = $v8FeedGenerator;
}
/** Set forced feedGenerator: do not rely on custom data enableV8 setting */
public function setFeedGenerator($forcedFeedGenerator = null): void
{
$this->forcedFeedGenerator = $forcedFeedGenerator;
}
public static function getType(): string
{
return static::$type;
}
public static function getAlias(): string
{
return static::$alias;
}
public function getObjectType(): string
{
return static::$objectType;
}
public function getTTL(): int
{
return $this->ttl;
}
protected function prepareSingleObject($singleObject)
{
return $this->objectWrapper->setObject($singleObject);
}
public function render(array $feedRow, ?int $limit = null): void
{
$this->initFeedGenerator($feedRow);
$this->feedGenerator->setThrowErrors(false);
if ($feedRow['pretty'] ?? false) {
$this->feedGenerator->setIndent(true);
}
try {
$this->renderCommon($feedRow, $limit);
} catch (ExpressionInterpreterException|V8ErrorException $e) {
$this->handleV8GeneratorException($e, $feedRow, $limit);
$this->productList = null;
return;
} catch (XSDParsingException|XMLFeedReaderException $e) {
$this->handleFeedException($e, $feedRow, $limit);
}
if (!$limit) {
$this->updateFeedReport($feedRow, $this->feedGenerator->getReport());
}
$this->productList = null;
}
public function renderStrict(array $feedRow, ?int $limit = null)
{
$this->initFeedGenerator($feedRow);
$this->feedGenerator->setThrowErrors();
$this->feedGenerator->setIsPreview();
$this->renderCommon($feedRow, $limit);
}
public function getReport(): array
{
return $this->feedGenerator->getReport();
}
public function renderWithGenerator(array $feedRow, ?int $limit = null): \Generator
{
$this->initFeedGenerator($feedRow);
$this->feedGenerator->setThrowErrors(false);
if ($feedRow['pretty'] ?? false) {
$this->feedGenerator->setIndent(true);
}
try {
return $this->withFeedGeneratorConfigured($feedRow, $limit, function (Configuration $configuration, array $placeholders, \Generator $data) use ($feedRow, $limit): \Generator {
yield 'start' => str_replace(array_keys($placeholders), array_values($placeholders), $configuration->getBeginning()).PHP_EOL;
yield from $this->feedGenerator->transformWithGenerator($data);
yield 'end' => $configuration->getEnd();
if (!$limit) {
$this->updateFeedReport($feedRow, $this->feedGenerator->getReport());
}
$this->productList = null;
});
} catch (ExpressionInterpreterException|V8ErrorException $e) {
$this->handleV8GeneratorException($e, $feedRow, $limit);
$this->productList = null;
} catch (XSDParsingException|XMLFeedReaderException $e) {
$this->handleFeedException($e, $feedRow, $limit);
}
}
private function renderCommon(array $feedRow, ?int $limit = null)
{
$this->withFeedGeneratorConfigured($feedRow, $limit, function (Configuration $configuration, array $placeholders, \Generator $data) {
echo str_replace(array_keys($placeholders), array_values($placeholders), $configuration->getBeginning()).PHP_EOL;
$this->feedGenerator->transform($data);
echo $configuration->getEnd();
});
}
/**
* @param callable(Configuration, array, \Generator): mixed $generatorFn
*/
private function withFeedGeneratorConfigured(array $feedRow, ?int $limit, callable $generatorFn): mixed
{
$this->objectWrapper->setFeedRow($feedRow);
$this->feedGenerator->setWrapper($this->objectWrapper); // support multiple custom wrappers
$this->feedGenerator->setConfiguration($feedRow['template']);
$configuration = $this->feedGenerator->getConfiguration();
$data = $this->getData($feedRow, $limit);
return $generatorFn($configuration, $this->getBeginningPlaceholders(), $data);
}
private function getBeginningPlaceholders(): array
{
$cfg = Config::get();
$dbcfg = \Settings::getDefault();
return [
'{timestamp}' => time(),
'{print_url}' => trim($cfg['Addr']['print'], '/'),
'{url}' => $cfg['Addr']['full'],
'{shop_name}' => $dbcfg['shop_firm_name'],
'{currency}' => Contexts::get(CurrencyContext::class)->getActiveId(),
'{language}' => Contexts::get(LanguageContext::class)->getActiveId(),
'{language_locale}' => Contexts::get(LanguageContext::class)->getActive()->getLocale(),
'{datetime_iso}' => (new \DateTime())->format('Y-m-d\TH:i:s.v'),
];
}
private function handleV8GeneratorException(\Throwable $e, array $feedRow, ?int $limit = null): void
{
if (getAdminUser() && $limit) {
echo $e->getMessage();
return;
}
if (!$limit) {
$this->updateSQL('feeds',
[
'report' => json_encode(['errors' => ['Script error' => $e->getMessage()]]),
],
['id' => $feedRow['id']]
);
// log errors related to v8js_generator_export.cc#L32 to sentry in full detail
if (mb_strpos($e->getMessage(), 'function(wrapped_object) { return (function*()') >= 0
|| mb_strpos($e->getMessage(), 'XMLFeedReaderException') >= 0
) {
\Sentry\captureException($e, \Sentry\EventHint::fromArray(['extra' => [
'memoryUsage' => memory_get_usage(true),
'memoryPeak' => memory_get_peak_usage(true),
'memoryLimit' => ini_get('memory_limit'),
'feedRow_id' => $feedRow['id'],
'feedRow_name' => $feedRow['name'],
'feedRow_format' => $feedRow['data']['format'] ?? '',
'feedRow_source_url' => $feedRow['data']['feed_source_url'] ?? '',
'feedRow_description_url' => $feedRow['data']['feed_description_url'] ?? '',
]]));
}
addActivityLog(
ActivityLog::SEVERITY_ERROR,
ActivityLog::TYPE_COMMUNICATION,
sprintf('Feed: Generování feedu se nezdařilo - "%s"', $feedRow['name']),
[
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
...['error' => $e->getMessage()],
],
[FeedsBundle::LOG_TAG_FEED]
);
}
}
private function handleFeedException(\Throwable $e, array $feedRow, ?int $limit = null): void
{
if (!($e instanceof XSDParsingException) && !($e instanceof XMLFeedReaderException)) {
throw $e;
}
if (getAdminUser() && $limit) {
echo $e->getMessage();
return;
}
throw $e;
}
protected function initFeedGenerator(array $feedRow): void
{
if (isset($this->forcedFeedGenerator)) {
$this->feedGenerator = $this->forcedFeedGenerator;
} else {
$this->feedGenerator = $this->v8FeedGenerator;
}
if (method_exists($this->feedGenerator, 'setBuildItemCallback')) {
$this->feedGenerator->setBuildItemCallback(null); // reset callback for pre-generated expensive feeds
}
}
public function getData(array $feedRow, ?int $limit = null): \Generator
{
$qb = $this->query ?? $this->getQuery();
$count = (int) (clone $qb)
->addSelect('COUNT(*)')
->execute()
->fetchOne();
// use batching only if limit is NOT set
$iterationLimit = isset($limit) ? 1 : ($count / (float) static::$batchSize);
for ($i = 0; $i < $iterationLimit; $i++) {
$qb->setFirstResult($i * static::$batchSize);
$qb->setMaxResults($limit ?? static::$batchSize);
$objects = new ArrayCollection();
foreach ($qb->execute() as $row) {
$objects->set($row['id'], (object) $row); // cast to (object) to force reference
}
$this->objectWrapper->setObjects($objects);
foreach ($objects as $producer) {
yield $this->prepareSingleObject($producer);
}
}
}
private function updateFeedReport(array $feedRow, array $report): void
{
$previousReport = $updateReport = json_decode($feedRow['report'] ?: '', true) ?: [];
$feedNameKeys = $this->getReportContextsKeys($feedRow);
$updateReport = array_merge($updateReport, $report);
$feedName = implode('_', $feedNameKeys);
$updateReport[$feedName] = $report;
// update feed report
$this->updateSQL('feeds',
[
'report' => json_encode($updateReport),
],
['id' => $feedRow['id']]
);
$previousValidCount = $previousReport[$feedName]['validCount'] ?? null;
// report large drop of valid items to activity log
if ($previousValidCount !== null && $report['validCount'] !== null && ($previousValidCount - $report['validCount']) > 500) {
addActivityLog(
ActivityLog::SEVERITY_WARNING,
ActivityLog::TYPE_COMMUNICATION,
sprintf('Feed: Velký pokles položek ve feedu (%d -> %d) - "%s"', $previousValidCount, $report['validCount'], $feedRow['name']),
[
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
...[
'previousValidCount' => $previousValidCount,
'validCount' => $report['validCount'],
'diff' => $previousValidCount - $report['validCount'],
'contexts' => $feedNameKeys,
],
],
[FeedsBundle::LOG_TAG_FEED]
);
}
// report errors in feed to activity log
if (!empty($report['errors'])) {
addActivityLog(
ActivityLog::SEVERITY_WARNING,
ActivityLog::TYPE_COMMUNICATION,
sprintf('Feed: Při generování feedu se u některých položek vyskytly chyby (%d) - "%s"', count($report['errors']), $feedRow['name']),
[
...ActivityLog::addObjectData([$feedRow['id'] => $feedRow['name']], 'feeds'),
...[
'errorsCount' => count($report['errors']),
'errorsSample' => array_slice($report['errors'], 0, 15),
],
],
[FeedsBundle::LOG_TAG_FEED]
);
}
}
protected function getReportContextsKeys(array $feedRow): array
{
$domainContext = Contexts::get(DomainContext::class);
$countryContext = Contexts::get(CountryContext::class);
$currencyContext = Contexts::get(CurrencyContext::class);
$languageContext = Contexts::get(LanguageContext::class);
$feedName['index'] = 'feed';
$feedName['id'] = $feedRow['id'];
$feedName['language'] = $languageContext->getActiveId();
$feedName['currency'] = $currencyContext->getActiveId();
$feedName['country'] = $countryContext->getActiveId();
$feedName['domain'] = $domainContext->getActiveId();
return $feedName;
}
}

View File

@@ -0,0 +1,170 @@
<?php
namespace KupShop\FeedGeneratorBundle\Feed;
use KupShop\FeedGeneratorBundle\Exception\XMLFeedReader\UnreadableXMLException;
use KupShop\FeedGeneratorBundle\Exception\XMLFeedReader\XMLFeedReaderException;
use KupShop\FeedGeneratorBundle\Exception\XSDParser\XSDParsingException;
use KupShop\FeedGeneratorBundle\Utils\ExternalFeedCache;
use KupShop\FeedGeneratorBundle\V8FeedGenerator;
use KupShop\FeedGeneratorBundle\Wrapper\ExternalFeedObjectWrapper;
use KupShop\FeedGeneratorBundle\XMLFeedReader;
use KupShop\FeedGeneratorBundle\XSD\Parser;
use KupShop\FeedsBundle\FeedProductList;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class ExternalFeed implements IExternalFeed
{
use ConfigurableFeedTrait;
public const TYPE_DESCRIPTION = 'feed_description_url';
public const TYPE_SOURCE = 'feed_source_url';
protected static $type = 'external';
protected static $objectType = 'external';
protected static $alias = 'Externí';
private HttpClientInterface $http;
private V8FeedGenerator $generator;
private Parser $parser;
private XMLFeedReader $feedReader;
private ?string $searchValue = null;
/** @var FeedProductList */
protected $productList; // only declared to avoid redefining whole render() method
public static function isAllowed(): bool
{
return true;
}
public function __construct(HttpClientInterface $httpClient, V8FeedGenerator $generator, ExternalFeedObjectWrapper $objectWrapper)
{
$this->http = $httpClient;
$this->generator = $generator;
$this->objectWrapper = $objectWrapper;
$this->parser = new Parser();
$this->feedReader = new XMLFeedReader();
$this->setFeedGenerator($generator);
}
private function initParser(array $feedRow): void
{
$cache = new ExternalFeedCache($feedRow);
$this->parser->loadFile($cache->getDescriptionUrl());
}
/**
* @throws UnreadableXMLException
*/
private function initReader(array $feedRow, ?array $basePath = null): void
{
$cache = new ExternalFeedCache($feedRow, $basePath);
$this->feedReader->openFile($cache->getSourceUrl());
}
/**
* Called from template, shows in data tab in feed configuration.
*/
public function fetchDataDescription(array $feedRow): array
{
if (is_string($feedRow['data'])) {
$feedRow['data'] = json_decode($feedRow['data'], true);
}
$this->initParser($feedRow);
$data = $this->parser->getFeedRootElement()->serialize();
$data['children'] = $data['children'] ?? [];
$data['children'][] = ['type' => 'xs:string', 'name' => '__search', 'minOccurs' => '0'];
usort($data['children'], function ($a, $b) {
return strcmp($a['name'], $b['name']);
});
return $data;
}
private function doSearch(string $searchQuery): ?array
{
try {
return $this->feedReader->findElementBySearchAttribute($searchQuery);
} catch (XMLFeedReaderException $e) {
}
return null;
}
/**
* @throws XSDParsingException|XMLFeedReaderException
*/
public function getData(array $feedRow, ?int $limit = null): \Generator
{
if (is_string($feedRow['data'])) {
$feedRow['data'] = json_decode($feedRow['data'], true);
}
$this->initParser($feedRow);
$feedElementPath = $this->parser->getFeedRootElementPath();
$types = $this->parser->getTypeConstraints();
$this->initReader($feedRow, $feedElementPath);
$this->feedReader->setBasePath($feedElementPath);
$this->feedReader->setTypeCorrectionMap($types);
if ($this->searchValue !== null) {
$cache = new ExternalFeedCache($feedRow, $feedElementPath);
$res = $cache->getCacheItem($this->searchValue);
if ($res !== null) {
return yield $this->prepareSingleObject($res);
}
$found = $this->doSearch($this->searchValue);
if ($found === null) {
return new \EmptyIterator();
} else {
$res = $found;
$cache->setCacheItem($this->searchValue, $res);
}
return yield $this->prepareSingleObject($res);
}
$counter = 0;
foreach ($this->feedReader->read() as $feedItem) {
if ($limit !== null && $counter >= $limit) {
break;
}
yield $this->prepareSingleObject($feedItem);
$counter++;
}
}
public function getDataNoThrow(array $feedRow, ?int $limit = null): \Generator
{
try {
$data = $this->getData($feedRow, $limit);
foreach ($data as $elem) {
yield $elem;
}
} catch (\Throwable $ignore) {
return new \EmptyIterator();
}
}
public function fetchDataDescriptionNoThrow(array $feedRow): array
{
try {
return $this->fetchDataDescription($feedRow);
} catch (\Throwable $ignore) {
return [];
}
}
public function filterByObjectID($objectID): void
{
$this->searchValue = trim($objectID);
}
}

View File

@@ -0,0 +1,16 @@
<?php
namespace KupShop\FeedGeneratorBundle\Feed;
interface IConfigurableFeed
{
public function getData(array $feedRow, ?int $limit = null): \Generator;
public function renderStrict(array $feedRow, ?int $limit = null);
public function renderWithGenerator(array $feedRow, ?int $limit = null): \Generator;
public function filterByObjectID($objectID): void;
public function getReport(): array;
}

View File

@@ -0,0 +1,15 @@
<?php
namespace KupShop\FeedGeneratorBundle\Feed;
use KupShop\FeedsBundle\Feed\IFeed;
interface IExternalFeed extends IFeed, IConfigurableFeed
{
/**
* @param array $feedRow Feed from database
*
* @return array Description parsed from XML schema
*/
public function fetchDataDescription(array $feedRow): array;
}

View File

@@ -0,0 +1,29 @@
<?php
declare(strict_types=1);
namespace KupShop\FeedGeneratorBundle\Feed;
use KupShop\FeedsBundle\FeedProductList;
use Query\QueryBuilder;
class ProductConfigurableFeed extends ConfigurableFeed
{
public static $type = 'productConfigurable';
public static $alias = 'Produkty';
public function getProductList(int $feedID): FeedProductList
{
$productList = parent::getProductList($feedID);
$productList->setVariationsAsResult(false);
$productList->andSpec(function (QueryBuilder $qb) {
$qb->addSelect('MIN(COALESCE(pv.price, p.price)) AS price');
$qb->addSelect('SUM(COALESCE(pv.in_store, p.in_store, 0)) AS in_store');
// $qb->groupBy('p.id'); // Don't group by here, it would break ProductList::getProductsCount()
return '';
});
return $productList;
}
}