From bbdf2e9977cc7ad6324a6c090daa60135117dcf4 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Sun, 21 Jun 2026 22:00:24 +0200 Subject: [PATCH 01/13] feat(flow-php/etl): configurable storage for aggregating functions GroupBy aggregation can move state out of the PHP heap instead of holding every result in memory. A new AggregationStorage abstraction offers three backends: memory (default, unchanged behavior), filesystem spill+merge, and PSR-16 key-value (APCu/Redis). - MergeableAggregatingFunction + merge() on all built-in aggregators - AggregationStorage: Memory / External (filesystem spill+merge) / Kv (PSR-16) - KV storage fails loudly on eviction / write failure instead of returning a wrong aggregate - config: AggregationStorageStrategy + FLOW_AGGREGATION_MAX_MEMORY wired into ConfigBuilder - APCu enabled in the nix dev shell (for the PSR-16/APCu storage test) --- .nix/php/lib/php.ini.dist | 5 +- .nix/pkgs/flow-php/package.nix | 1 + src/core/etl/src/Flow/ETL/Config.php | 2 + .../Config/Aggregation/AggregationConfig.php | 21 ++ .../Aggregation/AggregationConfigBuilder.php | 79 +++++++ .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 35 ++++ .../etl/src/Flow/ETL/Function/Average.php | 12 +- .../etl/src/Flow/ETL/Function/Collect.php | 12 +- .../src/Flow/ETL/Function/CollectUnique.php | 16 +- src/core/etl/src/Flow/ETL/Function/Count.php | 11 +- src/core/etl/src/Flow/ETL/Function/First.php | 11 +- src/core/etl/src/Flow/ETL/Function/Last.php | 13 +- src/core/etl/src/Flow/ETL/Function/Max.php | 21 +- .../Function/MergeableAggregatingFunction.php | 10 + src/core/etl/src/Flow/ETL/Function/Min.php | 21 +- .../src/Flow/ETL/Function/StringAggregate.php | 13 +- src/core/etl/src/Flow/ETL/Function/Sum.php | 12 +- src/core/etl/src/Flow/ETL/GroupBy.php | 58 ++++-- .../GroupBy/AggregationStorageStrategy.php | 17 ++ .../GroupBy/Storage/AggregationStorage.php | 29 +++ .../FilesystemGroupBucketsCache.php | 58 ++++++ .../Storage/ExternalAggregationStorage.php | 121 +++++++++++ .../Flow/ETL/GroupBy/Storage/GroupBucket.php | 15 ++ .../ETL/GroupBy/Storage/GroupBucketsCache.php | 23 +++ .../Flow/ETL/GroupBy/Storage/GroupEntry.php | 19 ++ .../GroupBy/Storage/KvAggregationStorage.php | 109 ++++++++++ .../Storage/MemoryAggregationStorage.php | 49 +++++ .../Flow/ETL/Processor/GroupByProcessor.php | 32 +++ .../DataFrame/GroupByStorageTest.php | 131 ++++++++++++ .../ExternalAggregationStorageTest.php | 67 ++++++ .../AggregationConfigBuilderTest.php | 71 +++++++ .../Function/AggregatingFunctionMergeTest.php | 194 ++++++++++++++++++ .../Storage/KvAggregationStorageTest.php | 93 +++++++++ 33 files changed, 1348 insertions(+), 33 deletions(-) create mode 100644 src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregationStorageStrategy.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/AggregationStorage.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php diff --git a/.nix/php/lib/php.ini.dist b/.nix/php/lib/php.ini.dist index 488665798b..ba9900777c 100644 --- a/.nix/php/lib/php.ini.dist +++ b/.nix/php/lib/php.ini.dist @@ -9,4 +9,7 @@ file_uploads = On max_file_uploads = 20 short_open_tag = off opcache.enable=1 -opcache.enable_cli=0 \ No newline at end of file +opcache.enable_cli=0 +apc.enabled=1 +apc.enable_cli=1 +apc.shm_size=128M \ No newline at end of file diff --git a/.nix/pkgs/flow-php/package.nix b/.nix/pkgs/flow-php/package.nix index 1d6bbd7406..384e33db14 100644 --- a/.nix/pkgs/flow-php/package.nix +++ b/.nix/pkgs/flow-php/package.nix @@ -22,6 +22,7 @@ let with all; enabled ++ [ + apcu bcmath dom mbstring diff --git a/src/core/etl/src/Flow/ETL/Config.php b/src/core/etl/src/Flow/ETL/Config.php index 00ebf0a0c6..4cf96bd43b 100644 --- a/src/core/etl/src/Flow/ETL/Config.php +++ b/src/core/etl/src/Flow/ETL/Config.php @@ -4,6 +4,7 @@ namespace Flow\ETL; +use Flow\ETL\Config\Aggregation\AggregationConfig; use Flow\ETL\Config\Cache\CacheConfig; use Flow\ETL\Config\ConfigBuilder; use Flow\ETL\Config\Sort\SortConfig; @@ -36,6 +37,7 @@ public function __construct( public SortConfig $sort, private ?Analyze $analyze, public TelemetryConfig $telemetry, + public AggregationConfig $aggregation, ) {} public static function builder(): ConfigBuilder diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php new file mode 100644 index 0000000000..f10975f2d0 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php @@ -0,0 +1,21 @@ +memoryLimit === null) { + $aggregationMemory = getenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); + + if (is_string($aggregationMemory)) { + $this->memoryLimit = Unit::fromString($aggregationMemory); + } else { + $memoryLimit = ini_get('memory_limit'); + + if ($memoryLimit === false || $memoryLimit === '-1') { + $this->memoryLimit = Unit::fromBytes(PHP_INT_MAX); + } else { + $this->memoryLimit = Unit::fromString( + $memoryLimit, + )->percentage(self::DEFAULT_AGGREGATION_MEMORY_PERCENTAGE); + } + } + } + + return new AggregationConfig($this->strategy, $this->memoryLimit, $this->storage, $this->filesystemProtocol); + } + + public function filesystemProtocol(string $protocol): self + { + $this->filesystemProtocol = $protocol; + + return $this; + } + + public function memoryLimit(Unit $memoryLimit): self + { + $this->memoryLimit = $memoryLimit; + + return $this; + } + + public function storage(AggregationStorage $storage): self + { + $this->storage = $storage; + + return $this; + } + + public function strategy(AggregationStorageStrategy $strategy): self + { + $this->strategy = $strategy; + + return $this; + } +} diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index 02606bbd83..80152a7237 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -8,12 +8,15 @@ use Flow\ETL\Analyze; use Flow\ETL\Cache; use Flow\ETL\Config; +use Flow\ETL\Config\Aggregation\AggregationConfigBuilder; use Flow\ETL\Config\Cache\CacheConfigBuilder; use Flow\ETL\Config\Sort\SortConfigBuilder; use Flow\ETL\Config\Telemetry\TelemetryConfig; use Flow\ETL\Config\Telemetry\TelemetryOptions; use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Filesystem\FilesystemStreams; +use Flow\ETL\GroupBy\AggregationStorageStrategy; +use Flow\ETL\GroupBy\Storage\AggregationStorage; use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Pipeline\Optimizer; use Flow\ETL\Pipeline\Optimizer\BatchSizeOptimization; @@ -34,6 +37,8 @@ final class ConfigBuilder { + public readonly AggregationConfigBuilder $aggregation; + public readonly CacheConfigBuilder $cache; public readonly SortConfigBuilder $sort; @@ -69,6 +74,7 @@ public function __construct() $this->putInputIntoRows = false; $this->optimizer = null; $this->clock = null; + $this->aggregation = new AggregationConfigBuilder(); $this->cache = new CacheConfigBuilder(); $this->sort = new SortConfigBuilder(); $this->randomValueGenerator = new NativePHPRandomValueGenerator(); @@ -79,6 +85,34 @@ public function __construct() : PackageVersion::get('flow-php/etl'); } + public function aggregationFilesystem(string $protocol): self + { + $this->aggregation->filesystemProtocol($protocol); + + return $this; + } + + public function aggregationMemoryLimit(Unit $unit): self + { + $this->aggregation->memoryLimit($unit); + + return $this; + } + + public function aggregationStorage(AggregationStorageStrategy $strategy): self + { + $this->aggregation->strategy($strategy); + + return $this; + } + + public function aggregationStore(AggregationStorage $storage): self + { + $this->aggregation->storage($storage); + + return $this; + } + public function analyze(Analyze $analyze): self { $this->analyze = $analyze; @@ -111,6 +145,7 @@ public function build(EntryFactory $entryFactory = new EntryFactory()): Config $this->sort->build(), $this->analyze, $this->telemetryConfig ?? TelemetryConfig::default($this->getClock()), + $this->aggregation->build(), ); } diff --git a/src/core/etl/src/Flow/ETL/Function/Average.php b/src/core/etl/src/Flow/ETL/Function/Average.php index d404adea22..1676783917 100644 --- a/src/core/etl/src/Flow/ETL/Function/Average.php +++ b/src/core/etl/src/Flow/ETL/Function/Average.php @@ -21,7 +21,7 @@ use function is_int; use function is_numeric; -final class Average implements AggregatingFunction, WindowFunction +final class Average implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction { private int $count; @@ -82,6 +82,16 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return (new Calculator())->divide($sum, $count, $this->scale, $this->rounding); } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + $this->sum += $other->sum; + $this->count += $other->count; + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/Collect.php b/src/core/etl/src/Flow/ETL/Function/Collect.php index 12209f5710..78d4ad51be 100644 --- a/src/core/etl/src/Flow/ETL/Function/Collect.php +++ b/src/core/etl/src/Flow/ETL/Function/Collect.php @@ -11,10 +11,11 @@ use Flow\ETL\Row\EntryFactory; use Flow\ETL\Row\Reference; +use function array_merge; use function current; use function Flow\ETL\DSL\to_entry; -final class Collect implements AggregatingFunction +final class Collect implements AggregatingFunction, MergeableAggregatingFunction { /** * @var array @@ -41,6 +42,15 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + $this->collection = array_merge($this->collection, $other->collection); + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php index 3f87b0ff09..fd86c2deaf 100644 --- a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php +++ b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php @@ -15,7 +15,7 @@ use function Flow\ETL\DSL\to_entry; use function in_array; -final class CollectUnique implements AggregatingFunction +final class CollectUnique implements AggregatingFunction, MergeableAggregatingFunction { /** * @var array @@ -49,6 +49,20 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + /** @var mixed $value */ + foreach ($other->collection as $value) { + if (!in_array($value, $this->collection, true)) { + $this->collection[] = $value; + } + } + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Count.php b/src/core/etl/src/Flow/ETL/Function/Count.php index 52e585ef3c..3087659836 100644 --- a/src/core/etl/src/Flow/ETL/Function/Count.php +++ b/src/core/etl/src/Flow/ETL/Function/Count.php @@ -16,7 +16,7 @@ use function Flow\ETL\DSL\int_entry; -final class Count implements AggregatingFunction, WindowFunction +final class Count implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction { private int $count; @@ -80,6 +80,15 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $count; } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + $this->count += $other->count; + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/First.php b/src/core/etl/src/Flow/ETL/Function/First.php index 36126f0176..9e7c39b750 100644 --- a/src/core/etl/src/Flow/ETL/Function/First.php +++ b/src/core/etl/src/Flow/ETL/Function/First.php @@ -13,7 +13,7 @@ use function Flow\ETL\DSL\string_entry; -final class First implements AggregatingFunction +final class First implements AggregatingFunction, MergeableAggregatingFunction { /** * @var null|Entry @@ -37,6 +37,15 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + $this->first ??= $other->first; + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Last.php b/src/core/etl/src/Flow/ETL/Function/Last.php index d5c1e76f76..8df306d96d 100644 --- a/src/core/etl/src/Flow/ETL/Function/Last.php +++ b/src/core/etl/src/Flow/ETL/Function/Last.php @@ -13,7 +13,7 @@ use function Flow\ETL\DSL\string_entry; -final class Last implements AggregatingFunction +final class Last implements AggregatingFunction, MergeableAggregatingFunction { /** * @var null|Entry @@ -35,6 +35,17 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + if ($other->last !== null) { + $this->last = $other->last; + } + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Max.php b/src/core/etl/src/Flow/ETL/Function/Max.php index 6794783b8c..9f5eda78a7 100644 --- a/src/core/etl/src/Flow/ETL/Function/Max.php +++ b/src/core/etl/src/Flow/ETL/Function/Max.php @@ -18,7 +18,7 @@ use function is_numeric; use function max; -final class Max implements AggregatingFunction +final class Max implements AggregatingFunction, MergeableAggregatingFunction { private float|DateTimeInterface|null $max; @@ -52,6 +52,25 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + if ($other->max === null) { + return; + } + + if ($this->max === null) { + $this->max = $other->max; + + return; + } + + $this->max = max($this->max, $other->max); + } + /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php b/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php new file mode 100644 index 0000000000..4065cd7cf7 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php @@ -0,0 +1,10 @@ +min === null) { + return; + } + + if ($this->min === null) { + $this->min = $other->min; + + return; + } + + $this->min = min($this->min, $other->min); + } + /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php index 13f1e468a3..c68fa25544 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Row; use Flow\ETL\Row\Entry; @@ -11,6 +12,7 @@ use Flow\ETL\Row\Reference; use Flow\ETL\Row\SortOrder; +use function array_merge; use function count; use function Flow\ETL\DSL\str_entry; use function implode; @@ -18,7 +20,7 @@ use function rsort; use function sort; -final class StringAggregate implements AggregatingFunction +final class StringAggregate implements AggregatingFunction, MergeableAggregatingFunction { /** * @var array @@ -40,6 +42,15 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + $this->values = array_merge($this->values, $other->values); + } + /** * @return Row\Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Sum.php b/src/core/etl/src/Flow/ETL/Function/Sum.php index 77415475c3..e117a9f302 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sum.php +++ b/src/core/etl/src/Flow/ETL/Function/Sum.php @@ -19,7 +19,7 @@ use function Flow\ETL\DSL\int_entry; use function is_numeric; -final class Sum implements AggregatingFunction, WindowFunction +final class Sum implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction { private float|int $sum; @@ -68,6 +68,16 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $sum; } + public function merge(MergeableAggregatingFunction $other): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); + } + + // @mago-ignore analysis:possibly-invalid-argument + $this->sum = (new Calculator())->add($this->sum, $other->sum); + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index d606797cd4..3329121704 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -9,6 +9,9 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Function\AggregatingFunction; +use Flow\ETL\Function\MergeableAggregatingFunction; +use Flow\ETL\GroupBy\Storage\AggregationStorage; +use Flow\ETL\GroupBy\Storage\MemoryAggregationStorage; use Flow\ETL\Hash\NativePHPHash; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; @@ -36,11 +39,6 @@ final class GroupBy */ private array $aggregations; - /** - * @var array, aggregators: array}> - */ - private array $groupedTable; - private ?Reference $pivot; /** @@ -55,14 +53,16 @@ final class GroupBy private readonly References $refs; + private AggregationStorage $storage; + public function __construct(string|Reference ...$entries) { $this->refs = References::init(...array_unique($entries)); $this->aggregations = []; - $this->groupedTable = []; $this->pivotedTable = []; $this->pivotColumns = []; $this->pivot = null; + $this->storage = new MemoryAggregationStorage(); } public function aggregate(AggregatingFunction ...$aggregator): void @@ -80,6 +80,17 @@ public function aggregate(AggregatingFunction ...$aggregator): void $this->aggregations = $aggregator; } + public function allAggregationsMergeable(): bool + { + foreach ($this->aggregations as $aggregation) { + if (!$aggregation instanceof MergeableAggregatingFunction) { + return false; + } + } + + return true; + } + public function group(Rows $rows, FlowContext $context): void { $pivot = $this->pivot; @@ -146,26 +157,22 @@ public function group(Rows $rows, FlowContext $context): void $valuesHash = $this->hash($values); - if (!array_key_exists($valuesHash, $this->groupedTable)) { - $aggregators = []; - - foreach ($this->aggregations as $aggregator) { - $aggregators[] = clone $aggregator; - } - - $this->groupedTable[$valuesHash] = [ - 'values' => $values, - 'aggregators' => $aggregators, - ]; - } + $entry = $this->storage->entry($valuesHash, $values, $this->aggregations); - foreach ($this->groupedTable[$valuesHash]['aggregators'] as $aggregator) { + foreach ($entry->aggregators as $aggregator) { $aggregator->aggregate($row, $context); } + + $this->storage->save($valuesHash, $entry); } } } + public function isPivot(): bool + { + return $this->pivot !== null; + } + public function pivot(Reference $ref): void { $this->pivot = $ref; @@ -199,15 +206,17 @@ public function result(FlowContext $context): Rows return array_to_rows($rows, $context->entryFactory()); } - foreach ($this->groupedTable as $group) { + $this->storage->flush(); + + foreach ($this->storage->all() as $group) { $entries = []; /** @var mixed $value */ - foreach ($group['values'] ?? [] as $entry => $value) { + foreach ($group->values as $entry => $value) { $entries[] = $context->entryFactory()->create($entry, $value); } - foreach ($group['aggregators'] as $aggregator) { + foreach ($group->aggregators as $aggregator) { $entries[] = $aggregator->result($context->entryFactory()); } @@ -219,6 +228,11 @@ public function result(FlowContext $context): Rows return new Rows(...$rows); } + public function useStorage(AggregationStorage $storage): void + { + $this->storage = $storage; + } + /** * @param array $values */ diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregationStorageStrategy.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregationStorageStrategy.php new file mode 100644 index 0000000000..18b30633c3 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/AggregationStorageStrategy.php @@ -0,0 +1,17 @@ + + */ + public function all(): Generator; + + /** + * @param array $values + * @param array $prototypes + */ + public function entry(string $hash, array $values, array $prototypes): GroupEntry; + + public function flush(): void; + + public function save(string $hash, GroupEntry $entry): void; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php new file mode 100644 index 0000000000..425ed8633f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php @@ -0,0 +1,58 @@ +cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); + } + + public function get(string $bucketId): array + { + $path = $this->keyPath($bucketId); + + if (!$this->filesystem->status($path)) { + return []; + } + + $stream = $this->filesystem->readFrom($path); + $content = $stream->content(); + $stream->close(); + + return $this->serializer->unserialize($content, [GroupBucket::class])->entries; + } + + public function remove(string $bucketId): void + { + $this->filesystem->rm($this->keyPath($bucketId)->parentDirectory()); + } + + public function set(string $bucketId, array $entries): void + { + $stream = $this->filesystem->writeTo($this->keyPath($bucketId)); + $stream->append($this->serializer->serialize(new GroupBucket($entries))); + $stream->close(); + } + + private function keyPath(string $key): Path + { + return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php new file mode 100644 index 0000000000..7885259db3 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php @@ -0,0 +1,121 @@ + + */ + private array $hot = []; + + /** + * @var list + */ + private array $spilledBuckets = []; + + public function __construct( + private readonly GroupBucketsCache $cache, + private readonly Unit $memoryLimit, + ) { + $this->consumption = new Consumption(false); + } + + /** + * @return Generator + */ + public function all(): Generator + { + if ($this->spilledBuckets === []) { + foreach ($this->hot as $entry) { + yield $entry; + } + + return; + } + + /** @var array $merged */ + $merged = []; + + foreach ($this->spilledBuckets as $bucketId) { + foreach ($this->cache->get($bucketId) as $hash => $entry) { + if (!array_key_exists($hash, $merged)) { + $merged[$hash] = $entry; + + continue; + } + + foreach ($merged[$hash]->aggregators as $index => $aggregator) { + $partial = $entry->aggregators[$index]; + + if ( + $aggregator instanceof MergeableAggregatingFunction + && $partial instanceof MergeableAggregatingFunction + ) { + $aggregator->merge($partial); + } + } + } + + $this->cache->remove($bucketId); + } + + foreach ($merged as $entry) { + yield $entry; + } + } + + public function entry(string $hash, array $values, array $prototypes): GroupEntry + { + if (!array_key_exists($hash, $this->hot)) { + $aggregators = []; + + foreach ($prototypes as $prototype) { + $aggregators[] = clone $prototype; + } + + $this->hot[$hash] = new GroupEntry($values, $aggregators); + } + + return $this->hot[$hash]; + } + + public function flush(): void + { + if ($this->spilledBuckets !== [] && $this->hot !== []) { + $this->spill(); + } + } + + public function save(string $hash, GroupEntry $entry): void + { + if ($this->consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { + $this->spill(); + } + } + + private function spill(): void + { + $bucketId = bin2hex(random_bytes(16)); + $this->cache->set($bucketId, $this->hot); + $this->spilledBuckets[] = $bucketId; + $this->hot = []; + $this->consumption = new Consumption(false); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php new file mode 100644 index 0000000000..38860554e8 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php @@ -0,0 +1,15 @@ + $entries + */ + public function __construct( + public array $entries, + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php new file mode 100644 index 0000000000..025e523968 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php @@ -0,0 +1,23 @@ + + */ + public function get(string $bucketId): array; + + public function remove(string $bucketId): void; + + /** + * @param array $entries + */ + public function set(string $bucketId, array $entries): void; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php new file mode 100644 index 0000000000..40f90004cf --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php @@ -0,0 +1,19 @@ + $values + * @param array $aggregators + */ + public function __construct( + public array $values, + public array $aggregators, + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php new file mode 100644 index 0000000000..d15fa2c2f2 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php @@ -0,0 +1,109 @@ + + */ + private array $hashes = []; + + public function __construct( + private readonly CacheInterface $cache, + private readonly Serializer $serializer = new NativePHPSerializer(), + private readonly string $namespace = 'flow_group_by_', + ) {} + + /** + * @return Generator + */ + public function all(): Generator + { + foreach (array_keys($this->hashes) as $hash) { + $entry = $this->read($hash); + + if ($entry === null) { + throw $this->evicted($hash); + } + + yield $entry; + + $this->cache->delete($this->namespace . $hash); + } + } + + public function entry(string $hash, array $values, array $prototypes): GroupEntry + { + $existing = $this->read($hash); + + if ($existing !== null) { + return $existing; + } + + if (isset($this->hashes[$hash])) { + throw $this->evicted($hash); + } + + $aggregators = []; + + foreach ($prototypes as $prototype) { + $aggregators[] = clone $prototype; + } + + $this->hashes[$hash] = true; + + return new GroupEntry($values, $aggregators); + } + + public function flush(): void {} + + public function save(string $hash, GroupEntry $entry): void + { + $this->hashes[$hash] = true; + + if (!$this->cache->set($this->namespace . $hash, $this->serializer->serialize($entry))) { + throw new RuntimeException( + 'Failed to write aggregation group to the KV store (store full?). Increase its capacity ' + . '(e.g. apc.shm_size, Redis maxmemory) or use AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL.', + ); + } + } + + private function evicted(string $hash): RuntimeException + { + return new RuntimeException( + 'Aggregation group "' + . $hash + . '" was evicted from the KV store mid-aggregation; the result ' + . 'would be incorrect. Increase the store capacity (e.g. apc.shm_size, Redis maxmemory) or use ' + . 'AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL.', + ); + } + + private function read(string $hash): ?GroupEntry + { + // @mago-ignore analysis:mixed-assignment + $serialized = $this->cache->get($this->namespace . $hash); + + if (!is_string($serialized)) { + return null; + } + + return $this->serializer->unserialize($serialized, [GroupEntry::class]); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php new file mode 100644 index 0000000000..d87dc6f2af --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php @@ -0,0 +1,49 @@ + + */ + private array $groups = []; + + /** + * @return Generator + */ + public function all(): Generator + { + foreach ($this->groups as $group) { + yield $group; + } + } + + public function entry(string $hash, array $values, array $prototypes): GroupEntry + { + if (!array_key_exists($hash, $this->groups)) { + $aggregators = []; + + foreach ($prototypes as $prototype) { + $aggregators[] = clone $prototype; + } + + $this->groups[$hash] = new GroupEntry($values, $aggregators); + } + + return $this->groups[$hash]; + } + + public function flush(): void {} + + public function save(string $hash, GroupEntry $entry): void {} +} diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index 46f0050ad6..bc0ed3433b 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -6,6 +6,10 @@ use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; +use Flow\ETL\GroupBy\Storage\AggregationStorage; +use Flow\ETL\GroupBy\Storage\BucketsCache\FilesystemGroupBucketsCache; +use Flow\ETL\GroupBy\Storage\ExternalAggregationStorage; +use Flow\ETL\GroupBy\Storage\MemoryAggregationStorage; use Flow\ETL\Processor; use Generator; @@ -22,10 +26,38 @@ public function __construct( public function process(Generator $rows, FlowContext $context): Generator { + $this->groupBy->useStorage($this->createStorage($context)); + foreach ($rows as $batch) { $this->groupBy->group($batch, $context); } yield $this->groupBy->result($context); } + + private function createStorage(FlowContext $context): AggregationStorage + { + $config = $context->config->aggregation; + + if ($config->storage !== null) { + return $config->storage; + } + + if ( + $config->strategy->useMemory() + || $this->groupBy->isPivot() + || !$this->groupBy->allAggregationsMergeable() + ) { + return new MemoryAggregationStorage(); + } + + return new ExternalAggregationStorage( + new FilesystemGroupBucketsCache( + $context->filesystem($config->filesystemProtocol), + $context->config->serializer(), + $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), + ), + $config->memoryLimit, + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php new file mode 100644 index 0000000000..3a19f8f7a1 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php @@ -0,0 +1,131 @@ + + */ + private static ?array $orders = null; + + public function test_apcu_kv_storage_produces_identical_results(): void + { + if (!ApcuAdapter::isSupported()) { + self::markTestSkipped( + 'APCu (with apc.enable_cli=1) is required to evaluate APCu as an aggregation storage backend.', + ); + } + + $cache = new Psr16Cache(new ApcuAdapter('flow_group_by_test')); + $cache->clear(); + + $expected = $this->aggregateBySeller(config_builder()); + $actual = $this->aggregateBySeller(config_builder()->aggregationStore(new KvAggregationStorage($cache))); + + self::assertEqualsCanonicalizing($expected, $actual); + } + + public function test_high_cardinality_group_by_spills_and_stays_correct(): void + { + $memory = $this->aggregateByEmail(config_builder()); + + $external = $this->aggregateByEmail( + config_builder() + ->aggregationStorage(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) + ->aggregationMemoryLimit(Unit::fromKb(256)), + ); + + self::assertEqualsCanonicalizing($memory, $external); + } + + public function test_kv_storage_produces_identical_results(): void + { + $expected = $this->aggregateBySeller(config_builder()); + + $actual = $this->aggregateBySeller(config_builder()->aggregationStore(new KvAggregationStorage(new Psr16Cache( + new ArrayAdapter(), + )))); + + self::assertEqualsCanonicalizing($expected, $actual); + } + + public function test_seller_aggregation_is_identical_between_memory_and_external_spill_storage(): void + { + $memory = $this->aggregateBySeller(config_builder()); + + $external = $this->aggregateBySeller( + config_builder() + ->aggregationStorage(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) + ->aggregationMemoryLimit(Unit::fromKb(256)), + ); + + self::assertEqualsCanonicalizing($memory, $external); + self::assertCount(5, $memory); + } + + private function aggregateByEmail(ConfigBuilder $config): array + { + return df($config)->read(from_array($this->orders()))->groupBy('email')->aggregate(count())->fetch()->toArray(); + } + + private function aggregateBySeller(ConfigBuilder $config): array + { + return df($config) + ->read(from_array($this->orders())) + ->groupBy('seller_id') + ->aggregate(count(), sum('discount'), collect('customer')) + ->fetch() + ->toArray(); + } + + /** + * @return list + */ + private function orders(): array + { + if (self::$orders !== null) { + return self::$orders; + } + + $orders = []; + + foreach ((new FakeRandomOrdersExtractor(self::ORDERS))->rawData() as $order) { + /** @var mixed $discount */ + $discount = $order['discount']; + + $orders[] = [ + 'seller_id' => (string) $order['seller_id'], + 'email' => (string) $order['email'], + 'customer' => (string) $order['customer'], + 'discount' => is_numeric($discount) ? (float) $discount : null, + ]; + } + + self::$orders = $orders; + + return $orders; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php new file mode 100644 index 0000000000..aea7423f04 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php @@ -0,0 +1,67 @@ +cacheDir->suffix('/external-aggregation-storage/'); + $this->fs()->rm($cacheDir); + + $context = flow_context(); + $storage = new ExternalAggregationStorage( + new FilesystemGroupBucketsCache($this->fs(), $this->serializer(), $cacheDir), + Unit::fromBytes(0), + ); + + $input = [ + ['type' => 'a', 'v' => 1], + ['type' => 'b', 'v' => 2], + ['type' => 'a', 'v' => 3], + ['type' => 'a', 'v' => 4], + ['type' => 'b', 'v' => 5], + ]; + + foreach ($input as $record) { + $entry = $storage->entry($record['type'], ['type' => $record['type']], [sum(ref('v'))]); + + foreach ($entry->aggregators as $aggregator) { + $aggregator->aggregate(row(str_entry('type', $record['type']), int_entry('v', $record['v'])), $context); + } + + $storage->save($record['type'], $entry); + } + + self::assertNotNull($this->fs()->status($cacheDir)); + + $results = []; + + foreach ($storage->all() as $group) { + /** @var int $sum */ + $sum = $group->aggregators[0]->result($context->entryFactory())->value(); + $results[(string) $group->values['type']] = $sum; + } + + ksort($results); + + self::assertSame(['a' => 8, 'b' => 7], $results); + + $this->fs()->rm($cacheDir); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php new file mode 100644 index 0000000000..112e62b03e --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php @@ -0,0 +1,71 @@ +memoryLimit(Unit::fromMb(16)) + ->build(); + + self::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); + } + + public function test_default_strategy_is_memory(): void + { + $config = (new AggregationConfigBuilder())->build(); + + self::assertSame(AggregationStorageStrategy::MEMORY, $config->strategy); + self::assertNull($config->storage); + self::assertGreaterThan(0, $config->memoryLimit->inBytes()); + } + + public function test_injecting_a_custom_storage(): void + { + $storage = new KvAggregationStorage(new Psr16Cache(new ArrayAdapter())); + + $config = (new AggregationConfigBuilder()) + ->storage($storage) + ->build(); + + self::assertSame($storage, $config->storage); + } + + public function test_memory_limit_is_read_from_environment_variable(): void + { + putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV . '=10M'); + + try { + $config = (new AggregationConfigBuilder())->build(); + + self::assertSame(Unit::fromMb(10)->inBytes(), $config->memoryLimit->inBytes()); + } finally { + putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); + } + } + + public function test_selecting_the_external_strategy(): void + { + $config = (new AggregationConfigBuilder()) + ->strategy(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) + ->build(); + + self::assertSame(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL, $config->strategy); + self::assertFalse($config->strategy->useMemory()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php new file mode 100644 index 0000000000..8f6d059be3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php @@ -0,0 +1,194 @@ +aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = average(ref('v')); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertEquals(2, $left->result($context->entryFactory())->value()); + } + + public function test_collect_merge_appends_values(): void + { + $context = flow_context(); + + $left = collect(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = collect(ref('v')); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + } + + public function test_collect_unique_merge_unions_values(): void + { + $context = flow_context(); + + $left = collect_unique(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = collect_unique(ref('v')); + $right->aggregate(row(int_entry('v', 2)), $context); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + } + + public function test_count_merge_adds_counts(): void + { + $context = flow_context(); + + $left = count(); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = count(); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame(3, $left->result($context->entryFactory())->value()); + } + + public function test_first_merge_keeps_earlier_value(): void + { + $context = flow_context(); + + $left = first(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + + $right = first(ref('v')); + $right->aggregate(row(int_entry('v', 9)), $context); + + $left->merge($right); + + self::assertSame(1, $left->result($context->entryFactory())->value()); + } + + public function test_last_merge_keeps_later_value(): void + { + $context = flow_context(); + + $left = last(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + + $right = last(ref('v')); + $right->aggregate(row(int_entry('v', 9)), $context); + + $left->merge($right); + + self::assertSame(9, $left->result($context->entryFactory())->value()); + } + + public function test_max_merge_keeps_greatest(): void + { + $context = flow_context(); + + $left = max(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + $left->aggregate(row(int_entry('v', 3)), $context); + + $right = max(ref('v')); + $right->aggregate(row(int_entry('v', 9)), $context); + + $left->merge($right); + + self::assertSame(9, $left->result($context->entryFactory())->value()); + } + + public function test_merge_with_different_function_type_throws(): void + { + $this->expectException(InvalidArgumentException::class); + + sum(ref('v'))->merge(count()); + } + + public function test_min_merge_keeps_smallest(): void + { + $context = flow_context(); + + $left = min(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + $left->aggregate(row(int_entry('v', 3)), $context); + + $right = min(ref('v')); + $right->aggregate(row(int_entry('v', 1)), $context); + + $left->merge($right); + + self::assertSame(1, $left->result($context->entryFactory())->value()); + } + + public function test_string_aggregate_merge_concatenates_values(): void + { + $context = flow_context(); + + $left = new StringAggregate(ref('v'), ',', SortOrder::ASC); + $left->aggregate(row(str_entry('v', 'b')), $context); + $left->aggregate(row(str_entry('v', 'a')), $context); + + $right = new StringAggregate(ref('v'), ',', SortOrder::ASC); + $right->aggregate(row(str_entry('v', 'c')), $context); + + $left->merge($right); + + self::assertSame('a,b,c', $left->result($context->entryFactory())->value()); + } + + public function test_sum_merge_adds_partial_sums(): void + { + $context = flow_context(); + + $left = sum(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = sum(ref('v')); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame(6, $left->result($context->entryFactory())->value()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php new file mode 100644 index 0000000000..4621ff2f9f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php @@ -0,0 +1,93 @@ +entry($group, ['type' => $group], [sum(ref('v'))]); + $entry->aggregators[0]->aggregate(row(str_entry('type', $group), int_entry('v', 1)), $context); + $storage->save($group, $entry); + } + + $cache->delete('flow_group_by_a'); + + $this->expectException(RuntimeException::class); + + iterator_to_array($storage->all()); + } + + public function test_detects_eviction_on_subsequent_access(): void + { + $cache = new Psr16Cache(new ArrayAdapter()); + $storage = new KvAggregationStorage($cache); + $context = flow_context(); + + $entry = $storage->entry('a', ['type' => 'a'], [sum(ref('v'))]); + $entry->aggregators[0]->aggregate(row(str_entry('type', 'a'), int_entry('v', 1)), $context); + $storage->save('a', $entry); + + $cache->delete('flow_group_by_a'); + + $this->expectException(RuntimeException::class); + + $storage->entry('a', ['type' => 'a'], [sum(ref('v'))]); + } + + public function test_round_trip_produces_correct_aggregates(): void + { + $cache = new Psr16Cache(new ArrayAdapter()); + $storage = new KvAggregationStorage($cache); + $context = flow_context(); + + $input = [ + ['type' => 'a', 'v' => 1], + ['type' => 'b', 'v' => 2], + ['type' => 'a', 'v' => 3], + ]; + + foreach ($input as $record) { + $entry = $storage->entry($record['type'], ['type' => $record['type']], [sum(ref('v'))]); + $entry->aggregators[0]->aggregate( + row(str_entry('type', $record['type']), int_entry('v', $record['v'])), + $context, + ); + $storage->save($record['type'], $entry); + } + + $results = []; + + foreach ($storage->all() as $group) { + /** @var int $value */ + $value = $group->aggregators[0]->result($context->entryFactory())->value(); + $results[(string) $group->values['type']] = $value; + } + + ksort($results); + + self::assertSame(['a' => 4, 'b' => 2], $results); + } +} From 95dee7f03ca43acb78441237d0f7b95d4b20c3fa Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Mon, 6 Jul 2026 22:10:29 +0200 Subject: [PATCH 02/13] refactor(flow-php/etl): sort-based aggregation instead of pluggable storage Reworks the aggregation-storage approach following review. External aggregation is now sort-then-fold: sort rows by the group-by columns (reusing ExternalSort, which spills and merges in bounded memory) and fold adjacent equal keys. This makes the whole abstraction simpler and drops the concepts the review found confusing. - remove the AggregationStorage layer (Memory/External/Kv storage, GroupEntry, GroupBucket(sCache)) and the AggregationStorageStrategy enum - remove MergeableAggregatingFunction and merge() from aggregators - sort-fold never splits a group, so no merge contract is needed - add AggregatingAlgorithm with MemoryAggregation (default, hash map) and ExternalAggregation (sort + fold), selected via AggregationAlgorithms (mirrors SortAlgorithms) - GroupBy is now a spec (references/aggregations/hash/keyValues); GroupByProcessor routes pivot and global aggregation to memory - config: aggregationAlgorithm() replaces aggregationStorage()/aggregationStore(); output is streamed in batches instead of one Rows - external tags rows with an input ordinal and sorts by (group keys, ordinal), so first()/last()/collect() return the same result as the in-memory path even when a group spills across sort runs --- .../Config/Aggregation/AggregationConfig.php | 10 +- .../Aggregation/AggregationConfigBuilder.php | 62 +--- .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 25 +- .../etl/src/Flow/ETL/Function/Average.php | 12 +- .../etl/src/Flow/ETL/Function/Collect.php | 12 +- .../src/Flow/ETL/Function/CollectUnique.php | 16 +- src/core/etl/src/Flow/ETL/Function/Count.php | 11 +- src/core/etl/src/Flow/ETL/Function/First.php | 11 +- src/core/etl/src/Flow/ETL/Function/Last.php | 13 +- src/core/etl/src/Flow/ETL/Function/Max.php | 21 +- .../Function/MergeableAggregatingFunction.php | 10 - src/core/etl/src/Flow/ETL/Function/Min.php | 21 +- .../src/Flow/ETL/Function/StringAggregate.php | 13 +- src/core/etl/src/Flow/ETL/Function/Sum.php | 12 +- src/core/etl/src/Flow/ETL/GroupBy.php | 308 ++++++++++-------- .../Flow/ETL/GroupBy/AggregatingAlgorithm.php | 25 ++ .../ETL/GroupBy/AggregationAlgorithms.php | 25 ++ .../GroupBy/AggregationStorageStrategy.php | 17 - .../Flow/ETL/GroupBy/ExternalAggregation.php | 110 +++++++ .../Flow/ETL/GroupBy/MemoryAggregation.php | 62 ++++ .../GroupBy/Storage/AggregationStorage.php | 29 -- .../FilesystemGroupBucketsCache.php | 58 ---- .../Storage/ExternalAggregationStorage.php | 121 ------- .../Flow/ETL/GroupBy/Storage/GroupBucket.php | 15 - .../ETL/GroupBy/Storage/GroupBucketsCache.php | 23 -- .../Flow/ETL/GroupBy/Storage/GroupEntry.php | 19 -- .../GroupBy/Storage/KvAggregationStorage.php | 109 ------- .../Storage/MemoryAggregationStorage.php | 49 --- .../Flow/ETL/Processor/GroupByProcessor.php | 50 +-- ...ageTest.php => GroupByAggregationTest.php} | 73 ++--- .../ExternalAggregationStorageTest.php | 67 ---- .../AggregationConfigBuilderTest.php | 60 +--- .../Function/AggregatingFunctionMergeTest.php | 194 ----------- .../Storage/KvAggregationStorageTest.php | 93 ------ .../tests/Flow/ETL/Tests/Unit/GroupByTest.php | 90 ++--- .../Unit/Processor/GroupByProcessorTest.php | 9 +- 36 files changed, 533 insertions(+), 1322 deletions(-) delete mode 100644 src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregationStorageStrategy.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/ExternalAggregation.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/AggregationStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/{GroupByStorageTest.php => GroupByAggregationTest.php} (55%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php index f10975f2d0..1109d60a1c 100644 --- a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php @@ -4,18 +4,12 @@ namespace Flow\ETL\Config\Aggregation; -use Flow\ETL\Dataset\Memory\Unit; -use Flow\ETL\GroupBy\AggregationStorageStrategy; -use Flow\ETL\GroupBy\Storage\AggregationStorage; +use Flow\ETL\GroupBy\AggregationAlgorithms; final readonly class AggregationConfig { - public const string AGGREGATION_MAX_MEMORY_ENV = 'FLOW_AGGREGATION_MAX_MEMORY'; - public function __construct( - public AggregationStorageStrategy $strategy, - public Unit $memoryLimit, - public ?AggregationStorage $storage = null, + public AggregationAlgorithms $algorithm, public string $filesystemProtocol = 'file', ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php index c5f7076a4b..35277349c6 100644 --- a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php @@ -4,75 +4,29 @@ namespace Flow\ETL\Config\Aggregation; -use Flow\ETL\Dataset\Memory\Unit; -use Flow\ETL\GroupBy\AggregationStorageStrategy; -use Flow\ETL\GroupBy\Storage\AggregationStorage; - -use function getenv; -use function ini_get; -use function is_string; - -use const PHP_INT_MAX; +use Flow\ETL\GroupBy\AggregationAlgorithms; final class AggregationConfigBuilder { - public const int DEFAULT_AGGREGATION_MEMORY_PERCENTAGE = 70; + private AggregationAlgorithms $algorithm = AggregationAlgorithms::MEMORY_AGGREGATION; private string $filesystemProtocol = 'file'; - private ?Unit $memoryLimit = null; - - private ?AggregationStorage $storage = null; - - private AggregationStorageStrategy $strategy = AggregationStorageStrategy::MEMORY; - - public function build(): AggregationConfig - { - if ($this->memoryLimit === null) { - $aggregationMemory = getenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); - - if (is_string($aggregationMemory)) { - $this->memoryLimit = Unit::fromString($aggregationMemory); - } else { - $memoryLimit = ini_get('memory_limit'); - - if ($memoryLimit === false || $memoryLimit === '-1') { - $this->memoryLimit = Unit::fromBytes(PHP_INT_MAX); - } else { - $this->memoryLimit = Unit::fromString( - $memoryLimit, - )->percentage(self::DEFAULT_AGGREGATION_MEMORY_PERCENTAGE); - } - } - } - - return new AggregationConfig($this->strategy, $this->memoryLimit, $this->storage, $this->filesystemProtocol); - } - - public function filesystemProtocol(string $protocol): self - { - $this->filesystemProtocol = $protocol; - - return $this; - } - - public function memoryLimit(Unit $memoryLimit): self + public function algorithm(AggregationAlgorithms $algorithm): self { - $this->memoryLimit = $memoryLimit; + $this->algorithm = $algorithm; return $this; } - public function storage(AggregationStorage $storage): self + public function build(): AggregationConfig { - $this->storage = $storage; - - return $this; + return new AggregationConfig($this->algorithm, $this->filesystemProtocol); } - public function strategy(AggregationStorageStrategy $strategy): self + public function filesystemProtocol(string $protocol): self { - $this->strategy = $strategy; + $this->filesystemProtocol = $protocol; return $this; } diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index 80152a7237..f98012c89b 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -15,8 +15,7 @@ use Flow\ETL\Config\Telemetry\TelemetryOptions; use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Filesystem\FilesystemStreams; -use Flow\ETL\GroupBy\AggregationStorageStrategy; -use Flow\ETL\GroupBy\Storage\AggregationStorage; +use Flow\ETL\GroupBy\AggregationAlgorithms; use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Pipeline\Optimizer; use Flow\ETL\Pipeline\Optimizer\BatchSizeOptimization; @@ -85,30 +84,16 @@ public function __construct() : PackageVersion::get('flow-php/etl'); } - public function aggregationFilesystem(string $protocol): self - { - $this->aggregation->filesystemProtocol($protocol); - - return $this; - } - - public function aggregationMemoryLimit(Unit $unit): self + public function aggregationAlgorithm(AggregationAlgorithms $algorithm): self { - $this->aggregation->memoryLimit($unit); + $this->aggregation->algorithm($algorithm); return $this; } - public function aggregationStorage(AggregationStorageStrategy $strategy): self - { - $this->aggregation->strategy($strategy); - - return $this; - } - - public function aggregationStore(AggregationStorage $storage): self + public function aggregationFilesystem(string $protocol): self { - $this->aggregation->storage($storage); + $this->aggregation->filesystemProtocol($protocol); return $this; } diff --git a/src/core/etl/src/Flow/ETL/Function/Average.php b/src/core/etl/src/Flow/ETL/Function/Average.php index 1676783917..d404adea22 100644 --- a/src/core/etl/src/Flow/ETL/Function/Average.php +++ b/src/core/etl/src/Flow/ETL/Function/Average.php @@ -21,7 +21,7 @@ use function is_int; use function is_numeric; -final class Average implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction +final class Average implements AggregatingFunction, WindowFunction { private int $count; @@ -82,16 +82,6 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return (new Calculator())->divide($sum, $count, $this->scale, $this->rounding); } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - $this->sum += $other->sum; - $this->count += $other->count; - } - public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/Collect.php b/src/core/etl/src/Flow/ETL/Function/Collect.php index 78d4ad51be..12209f5710 100644 --- a/src/core/etl/src/Flow/ETL/Function/Collect.php +++ b/src/core/etl/src/Flow/ETL/Function/Collect.php @@ -11,11 +11,10 @@ use Flow\ETL\Row\EntryFactory; use Flow\ETL\Row\Reference; -use function array_merge; use function current; use function Flow\ETL\DSL\to_entry; -final class Collect implements AggregatingFunction, MergeableAggregatingFunction +final class Collect implements AggregatingFunction { /** * @var array @@ -42,15 +41,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - $this->collection = array_merge($this->collection, $other->collection); - } - /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php index fd86c2deaf..3f87b0ff09 100644 --- a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php +++ b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php @@ -15,7 +15,7 @@ use function Flow\ETL\DSL\to_entry; use function in_array; -final class CollectUnique implements AggregatingFunction, MergeableAggregatingFunction +final class CollectUnique implements AggregatingFunction { /** * @var array @@ -49,20 +49,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - /** @var mixed $value */ - foreach ($other->collection as $value) { - if (!in_array($value, $this->collection, true)) { - $this->collection[] = $value; - } - } - } - /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Count.php b/src/core/etl/src/Flow/ETL/Function/Count.php index 3087659836..52e585ef3c 100644 --- a/src/core/etl/src/Flow/ETL/Function/Count.php +++ b/src/core/etl/src/Flow/ETL/Function/Count.php @@ -16,7 +16,7 @@ use function Flow\ETL\DSL\int_entry; -final class Count implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction +final class Count implements AggregatingFunction, WindowFunction { private int $count; @@ -80,15 +80,6 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $count; } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - $this->count += $other->count; - } - public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/First.php b/src/core/etl/src/Flow/ETL/Function/First.php index 9e7c39b750..36126f0176 100644 --- a/src/core/etl/src/Flow/ETL/Function/First.php +++ b/src/core/etl/src/Flow/ETL/Function/First.php @@ -13,7 +13,7 @@ use function Flow\ETL\DSL\string_entry; -final class First implements AggregatingFunction, MergeableAggregatingFunction +final class First implements AggregatingFunction { /** * @var null|Entry @@ -37,15 +37,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - $this->first ??= $other->first; - } - /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Last.php b/src/core/etl/src/Flow/ETL/Function/Last.php index 8df306d96d..d5c1e76f76 100644 --- a/src/core/etl/src/Flow/ETL/Function/Last.php +++ b/src/core/etl/src/Flow/ETL/Function/Last.php @@ -13,7 +13,7 @@ use function Flow\ETL\DSL\string_entry; -final class Last implements AggregatingFunction, MergeableAggregatingFunction +final class Last implements AggregatingFunction { /** * @var null|Entry @@ -35,17 +35,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - if ($other->last !== null) { - $this->last = $other->last; - } - } - /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Max.php b/src/core/etl/src/Flow/ETL/Function/Max.php index 9f5eda78a7..6794783b8c 100644 --- a/src/core/etl/src/Flow/ETL/Function/Max.php +++ b/src/core/etl/src/Flow/ETL/Function/Max.php @@ -18,7 +18,7 @@ use function is_numeric; use function max; -final class Max implements AggregatingFunction, MergeableAggregatingFunction +final class Max implements AggregatingFunction { private float|DateTimeInterface|null $max; @@ -52,25 +52,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - if ($other->max === null) { - return; - } - - if ($this->max === null) { - $this->max = $other->max; - - return; - } - - $this->max = max($this->max, $other->max); - } - /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php b/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php deleted file mode 100644 index 4065cd7cf7..0000000000 --- a/src/core/etl/src/Flow/ETL/Function/MergeableAggregatingFunction.php +++ /dev/null @@ -1,10 +0,0 @@ -min === null) { - return; - } - - if ($this->min === null) { - $this->min = $other->min; - - return; - } - - $this->min = min($this->min, $other->min); - } - /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php index c68fa25544..13f1e468a3 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Function; -use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Row; use Flow\ETL\Row\Entry; @@ -12,7 +11,6 @@ use Flow\ETL\Row\Reference; use Flow\ETL\Row\SortOrder; -use function array_merge; use function count; use function Flow\ETL\DSL\str_entry; use function implode; @@ -20,7 +18,7 @@ use function rsort; use function sort; -final class StringAggregate implements AggregatingFunction, MergeableAggregatingFunction +final class StringAggregate implements AggregatingFunction { /** * @var array @@ -42,15 +40,6 @@ public function aggregate(Row $row, FlowContext $context): void } } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - $this->values = array_merge($this->values, $other->values); - } - /** * @return Row\Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Sum.php b/src/core/etl/src/Flow/ETL/Function/Sum.php index e117a9f302..77415475c3 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sum.php +++ b/src/core/etl/src/Flow/ETL/Function/Sum.php @@ -19,7 +19,7 @@ use function Flow\ETL\DSL\int_entry; use function is_numeric; -final class Sum implements AggregatingFunction, MergeableAggregatingFunction, WindowFunction +final class Sum implements AggregatingFunction, WindowFunction { private float|int $sum; @@ -68,16 +68,6 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $sum; } - public function merge(MergeableAggregatingFunction $other): void - { - if (!$other instanceof self) { - throw new InvalidArgumentException('Cannot merge ' . self::class . ' with ' . $other::class); - } - - // @mago-ignore analysis:possibly-invalid-argument - $this->sum = (new Calculator())->add($this->sum, $other->sum); - } - public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 3329121704..0ce6f8ca6f 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -9,16 +9,16 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Function\AggregatingFunction; -use Flow\ETL\Function\MergeableAggregatingFunction; -use Flow\ETL\GroupBy\Storage\AggregationStorage; -use Flow\ETL\GroupBy\Storage\MemoryAggregationStorage; use Flow\ETL\Hash\NativePHPHash; +use Flow\ETL\Row\EntryFactory; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; +use Generator; use Stringable; use function array_filter; use function array_key_exists; +use function array_map; use function array_unique; use function array_values; use function count; @@ -32,37 +32,29 @@ use function is_scalar; use function serialize; +/** + * Specification of a group-by: the key columns and the aggregations to run. + * + * The actual accumulation of non-pivot groups is done by the aggregating algorithms + * (see {@see \Flow\ETL\GroupBy\AggregatingAlgorithm}); this class only exposes the spec + * and small helpers they need. Pivot is a memory-only cross-tab and is executed here. + */ final class GroupBy { + private const int RESULT_BATCH_SIZE = 1000; + /** * @var array */ - private array $aggregations; + private array $aggregations = []; - private ?Reference $pivot; - - /** - * @var array|bool|float|int|object|string> - */ - private array $pivotColumns; - - /** - * @var array|bool|float|int|object|string>> - */ - private array $pivotedTable; + private ?Reference $pivot = null; private readonly References $refs; - private AggregationStorage $storage; - public function __construct(string|Reference ...$entries) { $this->refs = References::init(...array_unique($entries)); - $this->aggregations = []; - $this->pivotedTable = []; - $this->pivotColumns = []; - $this->pivot = null; - $this->storage = new MemoryAggregationStorage(); } public function aggregate(AggregatingFunction ...$aggregator): void @@ -80,33 +72,142 @@ public function aggregate(AggregatingFunction ...$aggregator): void $this->aggregations = $aggregator; } - public function allAggregationsMergeable(): bool + /** + * Build the output row for one finalized group. + * + * @param array $keyValues + * @param array $aggregators + */ + public function aggregatedRow(array $keyValues, array $aggregators, EntryFactory $entryFactory): Row + { + $entries = []; + + /** @var mixed $value */ + foreach ($keyValues as $name => $value) { + $entries[] = $entryFactory->create($name, $value); + } + + foreach ($aggregators as $aggregator) { + $entries[] = $aggregator->result($entryFactory); + } + + return Row::create(...$entries); + } + + /** + * @return array + */ + public function aggregations(): array + { + return $this->aggregations; + } + + /** + * A stable string key for the given group-by values (used to bucket equal groups). + * + * @param array $values + */ + public function hash(array $values): string + { + /** @var array $stringValues */ + $stringValues = []; + + /** @var mixed $value */ + foreach ($values as $value) { + if ($value === null) { + $stringValues[] = 'null'; + } elseif (is_scalar($value)) { + $stringValues[] = (string) $value; + } else { + if ($value instanceof Stringable) { + $stringValues[] = $value->__toString(); + } elseif ($value instanceof DateTimeInterface) { + $stringValues[] = $value->format(DateTimeImmutable::ATOM); + } elseif (is_array($value)) { + $stringValues[] = $this->hash($value); + } else { + $stringValues[] = serialize($value); + } + } + } + + return NativePHPHash::xxh128(implode('', $stringValues)); + } + + public function isPivot(): bool { - foreach ($this->aggregations as $aggregation) { - if (!$aggregation instanceof MergeableAggregatingFunction) { - return false; + return $this->pivot !== null; + } + + /** + * The group-by key values for a single row (missing columns become null). + * + * @return array + */ + public function keyValues(Row $row): array + { + $values = []; + + foreach ($this->refs as $ref) { + try { + $values[$ref->name()] = $row->valueOf($ref); + } catch (InvalidArgumentException) { + $values[$ref->name()] = null; } } - return true; + return $values; + } + + /** + * A fresh set of aggregator instances for a new group. + * + * @return array + */ + public function newAggregators(): array + { + return array_map( + static fn(AggregatingFunction $aggregation): AggregatingFunction => clone $aggregation, + $this->aggregations, + ); + } + + public function pivot(Reference $ref): void + { + $this->pivot = $ref; } - public function group(Rows $rows, FlowContext $context): void + /** + * Execute the pivot cross-tab. Pivot is memory-only (it must know every distinct pivot + * value to build the columns), so it always aggregates in memory and streams the result. + * + * @param Generator $rows + * + * @return Generator + */ + public function pivotResult(Generator $rows, FlowContext $context): Generator { $pivot = $this->pivot; - if ($pivot !== null) { - foreach ($rows as $row) { + if ($pivot === null) { + throw new RuntimeException('pivotResult() called without a pivot reference'); + } + + /** @var array|bool|float|int|object|string> $pivotColumns */ + $pivotColumns = []; + /** @var array|bool|float|int|object|string>> $pivotedTable */ + $pivotedTable = []; + + foreach ($rows as $batch) { + foreach ($batch as $row) { try { - $this->pivotColumns[] = $row->valueOf($pivot); + $pivotColumns[] = $row->valueOf($pivot); } catch (InvalidArgumentException) { - $this->pivotColumns[] = null; + $pivotColumns[] = null; } } - $this->pivotColumns = array_values(array_filter(array_unique($this->pivotColumns))); - - foreach ($rows as $row) { + foreach ($batch as $row) { $values = []; foreach ($this->refs as $ref) { @@ -114,15 +215,14 @@ public function group(Rows $rows, FlowContext $context): void } $indexValue = $this->hash($values); - $pivotValue = $row->valueOf($pivot); - if (!array_key_exists($indexValue, $this->pivotedTable)) { - $this->pivotedTable[$indexValue] = []; + if (!array_key_exists($indexValue, $pivotedTable)) { + $pivotedTable[$indexValue] = []; } foreach ($this->refs as $ref) { - $this->pivotedTable[$indexValue][$ref->name()] = $row->valueOf($ref); + $pivotedTable[$indexValue][$ref->name()] = $row->valueOf($ref); } if ($pivotValue === null) { @@ -131,135 +231,55 @@ public function group(Rows $rows, FlowContext $context): void $pivotValue = type_union(type_string(), type_integer())->assert($pivotValue); - if (!array_key_exists($pivotValue, $this->pivotedTable[$indexValue])) { + if (!array_key_exists($pivotValue, $pivotedTable[$indexValue])) { // @mago-ignore analysis:invalid-property-assignment-value,possibly-invalid-clone - $this->pivotedTable[$indexValue][$pivotValue] = clone current($this->aggregations); + $pivotedTable[$indexValue][$pivotValue] = clone current($this->aggregations); } - $aggregator = $this->pivotedTable[$indexValue][$pivotValue]; + $aggregator = $pivotedTable[$indexValue][$pivotValue]; if ($aggregator instanceof AggregatingFunction) { $aggregator->aggregate($row, $context); } } - } else { - foreach ($rows as $row) { - /** @var array $values */ - $values = []; - - foreach ($this->refs as $ref) { - try { - $values[$ref->name()] = $row->valueOf($ref); - } catch (InvalidArgumentException) { - $values[$ref->name()] = null; - } - } - - $valuesHash = $this->hash($values); - - $entry = $this->storage->entry($valuesHash, $values, $this->aggregations); - - foreach ($entry->aggregators as $aggregator) { - $aggregator->aggregate($row, $context); - } - - $this->storage->save($valuesHash, $entry); - } } - } - public function isPivot(): bool - { - return $this->pivot !== null; - } + $pivotColumns = array_values(array_filter(array_unique($pivotColumns))); - public function pivot(Reference $ref): void - { - $this->pivot = $ref; - } - - public function result(FlowContext $context): Rows - { - $rows = []; - - if ($this->pivot) { - foreach ($this->pivotedTable as $index => $columns) { - $row = [$this->refs->first()->name() => $index]; + $buffer = []; - foreach ($columns as $rowIndex => $values) { - $row[$rowIndex] = $values instanceof AggregatingFunction - ? $values->result($context->entryFactory())->value() - : $values; - } - - foreach ($this->pivotColumns as $column) { - $column = type_union(type_string(), type_integer())->assert($column); - - if (!array_key_exists($column, $row)) { - $row[$column] = null; - } - } + foreach ($pivotedTable as $index => $columns) { + $row = [$this->refs->first()->name() => $index]; - $rows[] = $row; + foreach ($columns as $rowIndex => $value) { + $row[$rowIndex] = $value instanceof AggregatingFunction + ? $value->result($context->entryFactory())->value() + : $value; } - return array_to_rows($rows, $context->entryFactory()); - } - - $this->storage->flush(); - - foreach ($this->storage->all() as $group) { - $entries = []; + foreach ($pivotColumns as $column) { + $column = type_union(type_string(), type_integer())->assert($column); - /** @var mixed $value */ - foreach ($group->values as $entry => $value) { - $entries[] = $context->entryFactory()->create($entry, $value); + if (!array_key_exists($column, $row)) { + $row[$column] = null; + } } - foreach ($group->aggregators as $aggregator) { - $entries[] = $aggregator->result($context->entryFactory()); - } + $buffer[] = $row; - if (count($entries)) { - $rows[] = Row::create(...$entries); + if (count($buffer) >= self::RESULT_BATCH_SIZE) { + yield array_to_rows($buffer, $context->entryFactory()); + $buffer = []; } } - return new Rows(...$rows); - } - - public function useStorage(AggregationStorage $storage): void - { - $this->storage = $storage; + if ($buffer !== []) { + yield array_to_rows($buffer, $context->entryFactory()); + } } - /** - * @param array $values - */ - private function hash(array $values): string + public function references(): References { - /** @var array $stringValues */ - $stringValues = []; - - /** @var mixed $value */ - foreach ($values as $value) { - if ($value === null) { - $stringValues[] = 'null'; - } elseif (is_scalar($value)) { - $stringValues[] = (string) $value; - } else { - if ($value instanceof Stringable) { - $stringValues[] = $value->__toString(); - } elseif ($value instanceof DateTimeInterface) { - $stringValues[] = $value->format(DateTimeImmutable::ATOM); - } elseif (is_array($value)) { - $stringValues[] = $this->hash($value); - } else { - $stringValues[] = serialize($value); - } - } - } - - return NativePHPHash::xxh128(implode('', $stringValues)); + return $this->refs; } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php new file mode 100644 index 0000000000..8e4f54e301 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php @@ -0,0 +1,25 @@ + $rows + * + * @return Generator + */ + public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php new file mode 100644 index 0000000000..5ef1902306 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php @@ -0,0 +1,25 @@ +references()->all(); + $sortRefs[] = ref(self::INPUT_ORDINAL); + + $sorted = $this->sort->sortGenerator($this->withInputOrdinal($rows), $context, References::init(...$sortRefs)); + + $currentHash = null; + /** @var array $currentKey */ + $currentKey = []; + /** @var array $aggregators */ + $aggregators = []; + $buffer = []; + + foreach ($sorted as $batch) { + foreach ($batch as $row) { + $keyValues = $groupBy->keyValues($row); + $hash = $groupBy->hash($keyValues); + + if ($hash !== $currentHash) { + if ($currentHash !== null) { + $buffer[] = $groupBy->aggregatedRow($currentKey, $aggregators, $context->entryFactory()); + + if (count($buffer) >= self::BATCH_SIZE) { + yield new Rows(...$buffer); + $buffer = []; + } + } + + $currentHash = $hash; + $currentKey = $keyValues; + $aggregators = $groupBy->newAggregators(); + } + + foreach ($aggregators as $aggregator) { + $aggregator->aggregate($row, $context); + } + } + } + + if ($currentHash !== null) { + $buffer[] = $groupBy->aggregatedRow($currentKey, $aggregators, $context->entryFactory()); + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + } + } + + /** + * @param Generator $rows + * + * @return Generator + */ + private function withInputOrdinal(Generator $rows): Generator + { + $ordinal = 0; + + foreach ($rows as $batch) { + $tagged = []; + + foreach ($batch as $row) { + $tagged[] = $row->add(int_entry(self::INPUT_ORDINAL, $ordinal)); + $ordinal++; + } + + yield new Rows(...$tagged); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php new file mode 100644 index 0000000000..daac717d81 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php @@ -0,0 +1,62 @@ +, aggregators: array}> $groups */ + $groups = []; + + foreach ($rows as $batch) { + foreach ($batch as $row) { + $keyValues = $groupBy->keyValues($row); + $hash = $groupBy->hash($keyValues); + + if (!array_key_exists($hash, $groups)) { + $groups[$hash] = ['keyValues' => $keyValues, 'aggregators' => $groupBy->newAggregators()]; + } + + foreach ($groups[$hash]['aggregators'] as $aggregator) { + $aggregator->aggregate($row, $context); + } + } + } + + $buffer = []; + + foreach ($groups as $group) { + $buffer[] = $groupBy->aggregatedRow($group['keyValues'], $group['aggregators'], $context->entryFactory()); + + if (count($buffer) >= self::BATCH_SIZE) { + yield new Rows(...$buffer); + $buffer = []; + } + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/AggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/AggregationStorage.php deleted file mode 100644 index 2121450c3c..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/AggregationStorage.php +++ /dev/null @@ -1,29 +0,0 @@ - - */ - public function all(): Generator; - - /** - * @param array $values - * @param array $prototypes - */ - public function entry(string $hash, array $values, array $prototypes): GroupEntry; - - public function flush(): void; - - public function save(string $hash, GroupEntry $entry): void; -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php deleted file mode 100644 index 425ed8633f..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/BucketsCache/FilesystemGroupBucketsCache.php +++ /dev/null @@ -1,58 +0,0 @@ -cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); - } - - public function get(string $bucketId): array - { - $path = $this->keyPath($bucketId); - - if (!$this->filesystem->status($path)) { - return []; - } - - $stream = $this->filesystem->readFrom($path); - $content = $stream->content(); - $stream->close(); - - return $this->serializer->unserialize($content, [GroupBucket::class])->entries; - } - - public function remove(string $bucketId): void - { - $this->filesystem->rm($this->keyPath($bucketId)->parentDirectory()); - } - - public function set(string $bucketId, array $entries): void - { - $stream = $this->filesystem->writeTo($this->keyPath($bucketId)); - $stream->append($this->serializer->serialize(new GroupBucket($entries))); - $stream->close(); - } - - private function keyPath(string $key): Path - { - return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php deleted file mode 100644 index 7885259db3..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/ExternalAggregationStorage.php +++ /dev/null @@ -1,121 +0,0 @@ - - */ - private array $hot = []; - - /** - * @var list - */ - private array $spilledBuckets = []; - - public function __construct( - private readonly GroupBucketsCache $cache, - private readonly Unit $memoryLimit, - ) { - $this->consumption = new Consumption(false); - } - - /** - * @return Generator - */ - public function all(): Generator - { - if ($this->spilledBuckets === []) { - foreach ($this->hot as $entry) { - yield $entry; - } - - return; - } - - /** @var array $merged */ - $merged = []; - - foreach ($this->spilledBuckets as $bucketId) { - foreach ($this->cache->get($bucketId) as $hash => $entry) { - if (!array_key_exists($hash, $merged)) { - $merged[$hash] = $entry; - - continue; - } - - foreach ($merged[$hash]->aggregators as $index => $aggregator) { - $partial = $entry->aggregators[$index]; - - if ( - $aggregator instanceof MergeableAggregatingFunction - && $partial instanceof MergeableAggregatingFunction - ) { - $aggregator->merge($partial); - } - } - } - - $this->cache->remove($bucketId); - } - - foreach ($merged as $entry) { - yield $entry; - } - } - - public function entry(string $hash, array $values, array $prototypes): GroupEntry - { - if (!array_key_exists($hash, $this->hot)) { - $aggregators = []; - - foreach ($prototypes as $prototype) { - $aggregators[] = clone $prototype; - } - - $this->hot[$hash] = new GroupEntry($values, $aggregators); - } - - return $this->hot[$hash]; - } - - public function flush(): void - { - if ($this->spilledBuckets !== [] && $this->hot !== []) { - $this->spill(); - } - } - - public function save(string $hash, GroupEntry $entry): void - { - if ($this->consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { - $this->spill(); - } - } - - private function spill(): void - { - $bucketId = bin2hex(random_bytes(16)); - $this->cache->set($bucketId, $this->hot); - $this->spilledBuckets[] = $bucketId; - $this->hot = []; - $this->consumption = new Consumption(false); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php deleted file mode 100644 index 38860554e8..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucket.php +++ /dev/null @@ -1,15 +0,0 @@ - $entries - */ - public function __construct( - public array $entries, - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php deleted file mode 100644 index 025e523968..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupBucketsCache.php +++ /dev/null @@ -1,23 +0,0 @@ - - */ - public function get(string $bucketId): array; - - public function remove(string $bucketId): void; - - /** - * @param array $entries - */ - public function set(string $bucketId, array $entries): void; -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php deleted file mode 100644 index 40f90004cf..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/GroupEntry.php +++ /dev/null @@ -1,19 +0,0 @@ - $values - * @param array $aggregators - */ - public function __construct( - public array $values, - public array $aggregators, - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php deleted file mode 100644 index d15fa2c2f2..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/KvAggregationStorage.php +++ /dev/null @@ -1,109 +0,0 @@ - - */ - private array $hashes = []; - - public function __construct( - private readonly CacheInterface $cache, - private readonly Serializer $serializer = new NativePHPSerializer(), - private readonly string $namespace = 'flow_group_by_', - ) {} - - /** - * @return Generator - */ - public function all(): Generator - { - foreach (array_keys($this->hashes) as $hash) { - $entry = $this->read($hash); - - if ($entry === null) { - throw $this->evicted($hash); - } - - yield $entry; - - $this->cache->delete($this->namespace . $hash); - } - } - - public function entry(string $hash, array $values, array $prototypes): GroupEntry - { - $existing = $this->read($hash); - - if ($existing !== null) { - return $existing; - } - - if (isset($this->hashes[$hash])) { - throw $this->evicted($hash); - } - - $aggregators = []; - - foreach ($prototypes as $prototype) { - $aggregators[] = clone $prototype; - } - - $this->hashes[$hash] = true; - - return new GroupEntry($values, $aggregators); - } - - public function flush(): void {} - - public function save(string $hash, GroupEntry $entry): void - { - $this->hashes[$hash] = true; - - if (!$this->cache->set($this->namespace . $hash, $this->serializer->serialize($entry))) { - throw new RuntimeException( - 'Failed to write aggregation group to the KV store (store full?). Increase its capacity ' - . '(e.g. apc.shm_size, Redis maxmemory) or use AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL.', - ); - } - } - - private function evicted(string $hash): RuntimeException - { - return new RuntimeException( - 'Aggregation group "' - . $hash - . '" was evicted from the KV store mid-aggregation; the result ' - . 'would be incorrect. Increase the store capacity (e.g. apc.shm_size, Redis maxmemory) or use ' - . 'AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL.', - ); - } - - private function read(string $hash): ?GroupEntry - { - // @mago-ignore analysis:mixed-assignment - $serialized = $this->cache->get($this->namespace . $hash); - - if (!is_string($serialized)) { - return null; - } - - return $this->serializer->unserialize($serialized, [GroupEntry::class]); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php b/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php deleted file mode 100644 index d87dc6f2af..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/Storage/MemoryAggregationStorage.php +++ /dev/null @@ -1,49 +0,0 @@ - - */ - private array $groups = []; - - /** - * @return Generator - */ - public function all(): Generator - { - foreach ($this->groups as $group) { - yield $group; - } - } - - public function entry(string $hash, array $values, array $prototypes): GroupEntry - { - if (!array_key_exists($hash, $this->groups)) { - $aggregators = []; - - foreach ($prototypes as $prototype) { - $aggregators[] = clone $prototype; - } - - $this->groups[$hash] = new GroupEntry($values, $aggregators); - } - - return $this->groups[$hash]; - } - - public function flush(): void {} - - public function save(string $hash, GroupEntry $entry): void {} -} diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index bc0ed3433b..53e01179cb 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -6,16 +6,21 @@ use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; -use Flow\ETL\GroupBy\Storage\AggregationStorage; -use Flow\ETL\GroupBy\Storage\BucketsCache\FilesystemGroupBucketsCache; -use Flow\ETL\GroupBy\Storage\ExternalAggregationStorage; -use Flow\ETL\GroupBy\Storage\MemoryAggregationStorage; +use Flow\ETL\GroupBy\AggregatingAlgorithm; +use Flow\ETL\GroupBy\ExternalAggregation; +use Flow\ETL\GroupBy\MemoryAggregation; use Flow\ETL\Processor; +use Flow\ETL\Sort\ExternalSort; +use Flow\ETL\Sort\ExternalSort\BucketsCache\FilesystemBucketsCache; use Generator; /** * Groups all rows and applies aggregation functions. * + * Picks the aggregation algorithm from configuration (mirroring SortingProcessor): in-memory + * by default, or an external sort-based fold for bounded memory. Pivot and global aggregation + * (no group-by columns) always run in memory. + * * @internal */ final readonly class GroupByProcessor implements Processor @@ -26,38 +31,33 @@ public function __construct( public function process(Generator $rows, FlowContext $context): Generator { - $this->groupBy->useStorage($this->createStorage($context)); + if ($this->groupBy->isPivot()) { + yield from $this->groupBy->pivotResult($rows, $context); - foreach ($rows as $batch) { - $this->groupBy->group($batch, $context); + return; } - yield $this->groupBy->result($context); + yield from $this->algorithm($context)->aggregate($rows, $context, $this->groupBy); } - private function createStorage(FlowContext $context): AggregationStorage + private function algorithm(FlowContext $context): AggregatingAlgorithm { $config = $context->config->aggregation; - if ($config->storage !== null) { - return $config->storage; - } - - if ( - $config->strategy->useMemory() - || $this->groupBy->isPivot() - || !$this->groupBy->allAggregationsMergeable() - ) { - return new MemoryAggregationStorage(); + if ($config->algorithm->useMemory() || $this->groupBy->references()->count() === 0) { + return new MemoryAggregation(); } - return new ExternalAggregationStorage( - new FilesystemGroupBucketsCache( - $context->filesystem($config->filesystemProtocol), - $context->config->serializer(), - $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), + return new ExternalAggregation( + new ExternalSort( + new FilesystemBucketsCache( + $context->filesystem($config->filesystemProtocol), + $context->config->serializer(), + 100, + $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), + ), + $context->config->cache->externalSortBucketsCount, ), - $config->memoryLimit, ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php similarity index 55% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 3a19f8f7a1..0dfbbbec41 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByStorageTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -5,24 +5,22 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use Flow\ETL\Config\ConfigBuilder; -use Flow\ETL\Dataset\Memory\Unit; -use Flow\ETL\GroupBy\AggregationStorageStrategy; -use Flow\ETL\GroupBy\Storage\KvAggregationStorage; +use Flow\ETL\GroupBy\AggregationAlgorithms; use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; -use Symfony\Component\Cache\Adapter\ApcuAdapter; -use Symfony\Component\Cache\Adapter\ArrayAdapter; -use Symfony\Component\Cache\Psr16Cache; use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\df; +use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\from_array; +use function Flow\ETL\DSL\last; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\sum; use function is_numeric; -final class GroupByStorageTest extends FlowIntegrationTestCase +final class GroupByAggregationTest extends FlowIntegrationTestCase { private const int ORDERS = 20_000; @@ -31,56 +29,51 @@ final class GroupByStorageTest extends FlowIntegrationTestCase */ private static ?array $orders = null; - public function test_apcu_kv_storage_produces_identical_results(): void + public function test_first_and_last_preserve_input_order_under_external_spill(): void { - if (!ApcuAdapter::isSupported()) { - self::markTestSkipped( - 'APCu (with apc.enable_cli=1) is required to evaluate APCu as an aggregation storage backend.', - ); + // 2000 rows interleaved into two groups, each spanning multiple spilled sort runs; + // v is the input position so first/last are unambiguous in input order. + $data = []; + + for ($i = 0; $i < 2000; $i++) { + $data[] = ['g' => ($i % 2) === 0 ? 'a' : 'b', 'v' => $i]; } - $cache = new Psr16Cache(new ApcuAdapter('flow_group_by_test')); - $cache->clear(); + $run = static fn(ConfigBuilder $config): array => df($config) + ->read(from_array($data)) + ->groupBy('g') + ->aggregate(first(ref('v')), last(ref('v'))) + ->sortBy(ref('g')) + ->fetch() + ->toArray(); - $expected = $this->aggregateBySeller(config_builder()); - $actual = $this->aggregateBySeller(config_builder()->aggregationStore(new KvAggregationStorage($cache))); + $memory = $run(config_builder()); + $external = $run(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); - self::assertEqualsCanonicalizing($expected, $actual); + self::assertSame($memory, $external); + self::assertSame( + [ + ['g' => 'a', 'v_first' => 0, 'v_last' => 1998], + ['g' => 'b', 'v_first' => 1, 'v_last' => 1999], + ], + $external, + ); } - public function test_high_cardinality_group_by_spills_and_stays_correct(): void + public function test_high_cardinality_group_by_is_identical_between_memory_and_external(): void { $memory = $this->aggregateByEmail(config_builder()); - $external = $this->aggregateByEmail( - config_builder() - ->aggregationStorage(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) - ->aggregationMemoryLimit(Unit::fromKb(256)), - ); + $external = $this->aggregateByEmail(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); self::assertEqualsCanonicalizing($memory, $external); } - public function test_kv_storage_produces_identical_results(): void - { - $expected = $this->aggregateBySeller(config_builder()); - - $actual = $this->aggregateBySeller(config_builder()->aggregationStore(new KvAggregationStorage(new Psr16Cache( - new ArrayAdapter(), - )))); - - self::assertEqualsCanonicalizing($expected, $actual); - } - - public function test_seller_aggregation_is_identical_between_memory_and_external_spill_storage(): void + public function test_seller_aggregation_is_identical_between_memory_and_external(): void { $memory = $this->aggregateBySeller(config_builder()); - $external = $this->aggregateBySeller( - config_builder() - ->aggregationStorage(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) - ->aggregationMemoryLimit(Unit::fromKb(256)), - ); + $external = $this->aggregateBySeller(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); self::assertEqualsCanonicalizing($memory, $external); self::assertCount(5, $memory); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php deleted file mode 100644 index aea7423f04..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/Storage/ExternalAggregationStorageTest.php +++ /dev/null @@ -1,67 +0,0 @@ -cacheDir->suffix('/external-aggregation-storage/'); - $this->fs()->rm($cacheDir); - - $context = flow_context(); - $storage = new ExternalAggregationStorage( - new FilesystemGroupBucketsCache($this->fs(), $this->serializer(), $cacheDir), - Unit::fromBytes(0), - ); - - $input = [ - ['type' => 'a', 'v' => 1], - ['type' => 'b', 'v' => 2], - ['type' => 'a', 'v' => 3], - ['type' => 'a', 'v' => 4], - ['type' => 'b', 'v' => 5], - ]; - - foreach ($input as $record) { - $entry = $storage->entry($record['type'], ['type' => $record['type']], [sum(ref('v'))]); - - foreach ($entry->aggregators as $aggregator) { - $aggregator->aggregate(row(str_entry('type', $record['type']), int_entry('v', $record['v'])), $context); - } - - $storage->save($record['type'], $entry); - } - - self::assertNotNull($this->fs()->status($cacheDir)); - - $results = []; - - foreach ($storage->all() as $group) { - /** @var int $sum */ - $sum = $group->aggregators[0]->result($context->entryFactory())->value(); - $results[(string) $group->values['type']] = $sum; - } - - ksort($results); - - self::assertSame(['a' => 8, 'b' => 7], $results); - - $this->fs()->rm($cacheDir); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php index 112e62b03e..2763f800e0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php @@ -4,68 +4,30 @@ namespace Flow\ETL\Tests\Unit\Config\Aggregation; -use Flow\ETL\Config\Aggregation\AggregationConfig; use Flow\ETL\Config\Aggregation\AggregationConfigBuilder; -use Flow\ETL\Dataset\Memory\Unit; -use Flow\ETL\GroupBy\AggregationStorageStrategy; -use Flow\ETL\GroupBy\Storage\KvAggregationStorage; +use Flow\ETL\GroupBy\AggregationAlgorithms; use Flow\ETL\Tests\FlowTestCase; -use Symfony\Component\Cache\Adapter\ArrayAdapter; -use Symfony\Component\Cache\Psr16Cache; - -use function putenv; final class AggregationConfigBuilderTest extends FlowTestCase { - public function test_custom_memory_limit_overrides_default(): void - { - $config = (new AggregationConfigBuilder()) - ->memoryLimit(Unit::fromMb(16)) - ->build(); - - self::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); - } - - public function test_default_strategy_is_memory(): void + public function test_default_algorithm_is_memory(): void { $config = (new AggregationConfigBuilder())->build(); - self::assertSame(AggregationStorageStrategy::MEMORY, $config->strategy); - self::assertNull($config->storage); - self::assertGreaterThan(0, $config->memoryLimit->inBytes()); - } - - public function test_injecting_a_custom_storage(): void - { - $storage = new KvAggregationStorage(new Psr16Cache(new ArrayAdapter())); - - $config = (new AggregationConfigBuilder()) - ->storage($storage) - ->build(); - - self::assertSame($storage, $config->storage); - } - - public function test_memory_limit_is_read_from_environment_variable(): void - { - putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV . '=10M'); - - try { - $config = (new AggregationConfigBuilder())->build(); - - self::assertSame(Unit::fromMb(10)->inBytes(), $config->memoryLimit->inBytes()); - } finally { - putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); - } + self::assertSame(AggregationAlgorithms::MEMORY_AGGREGATION, $config->algorithm); + self::assertTrue($config->algorithm->useMemory()); + self::assertSame('file', $config->filesystemProtocol); } - public function test_selecting_the_external_strategy(): void + public function test_selecting_the_external_algorithm(): void { $config = (new AggregationConfigBuilder()) - ->strategy(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL) + ->algorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION) + ->filesystemProtocol('memory') ->build(); - self::assertSame(AggregationStorageStrategy::MEMORY_FALLBACK_EXTERNAL, $config->strategy); - self::assertFalse($config->strategy->useMemory()); + self::assertSame(AggregationAlgorithms::EXTERNAL_AGGREGATION, $config->algorithm); + self::assertFalse($config->algorithm->useMemory()); + self::assertSame('memory', $config->filesystemProtocol); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php deleted file mode 100644 index 8f6d059be3..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php +++ /dev/null @@ -1,194 +0,0 @@ -aggregate(row(int_entry('v', 1)), $context); - $left->aggregate(row(int_entry('v', 2)), $context); - - $right = average(ref('v')); - $right->aggregate(row(int_entry('v', 3)), $context); - - $left->merge($right); - - self::assertEquals(2, $left->result($context->entryFactory())->value()); - } - - public function test_collect_merge_appends_values(): void - { - $context = flow_context(); - - $left = collect(ref('v')); - $left->aggregate(row(int_entry('v', 1)), $context); - $left->aggregate(row(int_entry('v', 2)), $context); - - $right = collect(ref('v')); - $right->aggregate(row(int_entry('v', 3)), $context); - - $left->merge($right); - - self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); - } - - public function test_collect_unique_merge_unions_values(): void - { - $context = flow_context(); - - $left = collect_unique(ref('v')); - $left->aggregate(row(int_entry('v', 1)), $context); - $left->aggregate(row(int_entry('v', 2)), $context); - - $right = collect_unique(ref('v')); - $right->aggregate(row(int_entry('v', 2)), $context); - $right->aggregate(row(int_entry('v', 3)), $context); - - $left->merge($right); - - self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); - } - - public function test_count_merge_adds_counts(): void - { - $context = flow_context(); - - $left = count(); - $left->aggregate(row(int_entry('v', 1)), $context); - $left->aggregate(row(int_entry('v', 2)), $context); - - $right = count(); - $right->aggregate(row(int_entry('v', 3)), $context); - - $left->merge($right); - - self::assertSame(3, $left->result($context->entryFactory())->value()); - } - - public function test_first_merge_keeps_earlier_value(): void - { - $context = flow_context(); - - $left = first(ref('v')); - $left->aggregate(row(int_entry('v', 1)), $context); - - $right = first(ref('v')); - $right->aggregate(row(int_entry('v', 9)), $context); - - $left->merge($right); - - self::assertSame(1, $left->result($context->entryFactory())->value()); - } - - public function test_last_merge_keeps_later_value(): void - { - $context = flow_context(); - - $left = last(ref('v')); - $left->aggregate(row(int_entry('v', 1)), $context); - - $right = last(ref('v')); - $right->aggregate(row(int_entry('v', 9)), $context); - - $left->merge($right); - - self::assertSame(9, $left->result($context->entryFactory())->value()); - } - - public function test_max_merge_keeps_greatest(): void - { - $context = flow_context(); - - $left = max(ref('v')); - $left->aggregate(row(int_entry('v', 5)), $context); - $left->aggregate(row(int_entry('v', 3)), $context); - - $right = max(ref('v')); - $right->aggregate(row(int_entry('v', 9)), $context); - - $left->merge($right); - - self::assertSame(9, $left->result($context->entryFactory())->value()); - } - - public function test_merge_with_different_function_type_throws(): void - { - $this->expectException(InvalidArgumentException::class); - - sum(ref('v'))->merge(count()); - } - - public function test_min_merge_keeps_smallest(): void - { - $context = flow_context(); - - $left = min(ref('v')); - $left->aggregate(row(int_entry('v', 5)), $context); - $left->aggregate(row(int_entry('v', 3)), $context); - - $right = min(ref('v')); - $right->aggregate(row(int_entry('v', 1)), $context); - - $left->merge($right); - - self::assertSame(1, $left->result($context->entryFactory())->value()); - } - - public function test_string_aggregate_merge_concatenates_values(): void - { - $context = flow_context(); - - $left = new StringAggregate(ref('v'), ',', SortOrder::ASC); - $left->aggregate(row(str_entry('v', 'b')), $context); - $left->aggregate(row(str_entry('v', 'a')), $context); - - $right = new StringAggregate(ref('v'), ',', SortOrder::ASC); - $right->aggregate(row(str_entry('v', 'c')), $context); - - $left->merge($right); - - self::assertSame('a,b,c', $left->result($context->entryFactory())->value()); - } - - public function test_sum_merge_adds_partial_sums(): void - { - $context = flow_context(); - - $left = sum(ref('v')); - $left->aggregate(row(int_entry('v', 1)), $context); - $left->aggregate(row(int_entry('v', 2)), $context); - - $right = sum(ref('v')); - $right->aggregate(row(int_entry('v', 3)), $context); - - $left->merge($right); - - self::assertSame(6, $left->result($context->entryFactory())->value()); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php deleted file mode 100644 index 4621ff2f9f..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/Storage/KvAggregationStorageTest.php +++ /dev/null @@ -1,93 +0,0 @@ -entry($group, ['type' => $group], [sum(ref('v'))]); - $entry->aggregators[0]->aggregate(row(str_entry('type', $group), int_entry('v', 1)), $context); - $storage->save($group, $entry); - } - - $cache->delete('flow_group_by_a'); - - $this->expectException(RuntimeException::class); - - iterator_to_array($storage->all()); - } - - public function test_detects_eviction_on_subsequent_access(): void - { - $cache = new Psr16Cache(new ArrayAdapter()); - $storage = new KvAggregationStorage($cache); - $context = flow_context(); - - $entry = $storage->entry('a', ['type' => 'a'], [sum(ref('v'))]); - $entry->aggregators[0]->aggregate(row(str_entry('type', 'a'), int_entry('v', 1)), $context); - $storage->save('a', $entry); - - $cache->delete('flow_group_by_a'); - - $this->expectException(RuntimeException::class); - - $storage->entry('a', ['type' => 'a'], [sum(ref('v'))]); - } - - public function test_round_trip_produces_correct_aggregates(): void - { - $cache = new Psr16Cache(new ArrayAdapter()); - $storage = new KvAggregationStorage($cache); - $context = flow_context(); - - $input = [ - ['type' => 'a', 'v' => 1], - ['type' => 'b', 'v' => 2], - ['type' => 'a', 'v' => 3], - ]; - - foreach ($input as $record) { - $entry = $storage->entry($record['type'], ['type' => $record['type']], [sum(ref('v'))]); - $entry->aggregators[0]->aggregate( - row(str_entry('type', $record['type']), int_entry('v', $record['v'])), - $context, - ); - $storage->save($record['type'], $entry); - } - - $results = []; - - foreach ($storage->all() as $group) { - /** @var int $value */ - $value = $group->aggregators[0]->result($context->entryFactory())->value(); - $results[(string) $group->values['type']] = $value; - } - - ksort($results); - - self::assertSame(['a' => 4, 'b' => 2], $results); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php index 2d97fc5e58..89926efe6f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php @@ -6,6 +6,8 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\GroupBy; +use Flow\ETL\Processor\GroupByProcessor; +use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; @@ -24,32 +26,20 @@ public function test_group_by_missing_entry(): void { $groupBy = new GroupBy('type'); - $groupBy->group( - rows(row(str_entry('type', 'a')), row(str_entry('not-type', 'b')), row(str_entry('type', 'c'))), - flow_context(), - ); - static::assertEquals( rows(row(str_entry('type', 'a')), row(null_entry('type')), row(str_entry('type', 'c'))), - $groupBy->result(flow_context(config())), + $this->aggregate($groupBy, rows( + row(str_entry('type', 'a')), + row(str_entry('not-type', 'b')), + row(str_entry('type', 'c')), + )), ); } public function test_group_by_with_aggregation(): void { $group = new GroupBy('type'); - $group->aggregate(sum(ref('id'))); - $group->group( - rows( - row(int_entry('id', 1), str_entry('type', 'a')), - row(int_entry('id', 2), str_entry('type', 'b')), - row(int_entry('id', 3), str_entry('type', 'c')), - row(int_entry('id', 4), str_entry('type', 'a')), - row(int_entry('id', 5), str_entry('type', 'd')), - ), - flow_context(), - ); static::assertEquals( rows( @@ -58,7 +48,13 @@ public function test_group_by_with_aggregation(): void row(int_entry('id_sum', 3), str_entry('type', 'c')), row(int_entry('id_sum', 5), str_entry('type', 'd')), ), - $group->result(flow_context(config())), + $this->aggregate($group, rows( + row(int_entry('id', 1), str_entry('type', 'a')), + row(int_entry('id', 2), str_entry('type', 'b')), + row(int_entry('id', 3), str_entry('type', 'c')), + row(int_entry('id', 4), str_entry('type', 'a')), + row(int_entry('id', 5), str_entry('type', 'd')), + )), ); } @@ -72,27 +68,10 @@ public function test_group_by_with_empty_aggregations(): void public function test_group_by_with_pivoting(): void { - $rows = rows( - row(str_entry('product', 'Banana'), int_entry('amount', 1000), str_entry('country', 'USA')), - row(str_entry('product', 'Carrots'), int_entry('amount', 1500), str_entry('country', 'USA')), - row(str_entry('product', 'Beans'), int_entry('amount', 1600), str_entry('country', 'USA')), - row(str_entry('product', 'Orange'), int_entry('amount', 2000), str_entry('country', 'USA')), - row(str_entry('product', 'Orange'), int_entry('amount', 2000), str_entry('country', 'USA')), - row(str_entry('product', 'Banana'), int_entry('amount', 400), str_entry('country', 'China')), - row(str_entry('product', 'Carrots'), int_entry('amount', 1200), str_entry('country', 'China')), - row(str_entry('product', 'Beans'), int_entry('amount', 1500), str_entry('country', 'China')), - row(str_entry('product', 'Orange'), int_entry('amount', 4000), str_entry('country', 'China')), - row(str_entry('product', 'Banana'), int_entry('amount', 2000), str_entry('country', 'Canada')), - row(str_entry('product', 'Carrots'), int_entry('amount', 2000), str_entry('country', 'Canada')), - row(str_entry('product', 'Beans'), int_entry('amount', 2000), str_entry('country', 'Mexico')), - ); - $group = new GroupBy(ref('product')); $group->aggregate(sum(ref('amount'))); $group->pivot(ref('country')); - $group->group($rows, flow_context()); - static::assertEquals( rows( row( @@ -124,29 +103,52 @@ public function test_group_by_with_pivoting(): void int_entry('USA', 4000), ), ), - $group->result(flow_context(config()))->sortBy(ref('product')), + $this->aggregate($group, rows( + row(str_entry('product', 'Banana'), int_entry('amount', 1000), str_entry('country', 'USA')), + row(str_entry('product', 'Carrots'), int_entry('amount', 1500), str_entry('country', 'USA')), + row(str_entry('product', 'Beans'), int_entry('amount', 1600), str_entry('country', 'USA')), + row(str_entry('product', 'Orange'), int_entry('amount', 2000), str_entry('country', 'USA')), + row(str_entry('product', 'Orange'), int_entry('amount', 2000), str_entry('country', 'USA')), + row(str_entry('product', 'Banana'), int_entry('amount', 400), str_entry('country', 'China')), + row(str_entry('product', 'Carrots'), int_entry('amount', 1200), str_entry('country', 'China')), + row(str_entry('product', 'Beans'), int_entry('amount', 1500), str_entry('country', 'China')), + row(str_entry('product', 'Orange'), int_entry('amount', 4000), str_entry('country', 'China')), + row(str_entry('product', 'Banana'), int_entry('amount', 2000), str_entry('country', 'Canada')), + row(str_entry('product', 'Carrots'), int_entry('amount', 2000), str_entry('country', 'Canada')), + row(str_entry('product', 'Beans'), int_entry('amount', 2000), str_entry('country', 'Mexico')), + ))->sortBy(ref('product')), ); } public function test_group_by_with_pivoting_with_null_pivot_column(): void { - $rows = rows( - row(str_entry('product', 'Banana'), str_entry('country', 'USA'), int_entry('amount', 1000)), - row(str_entry('product', 'Apple'), str_entry('country', null), int_entry('amount', 400)), - ); - $group = new GroupBy(ref('product')); $group->aggregate(sum(ref('amount'))); $group->pivot(ref('country')); - $group->group($rows, flow_context()); - static::assertEquals( rows( row(str_entry('product', 'Apple'), null_entry('USA')), row(str_entry('product', 'Banana'), int_entry('USA', 1000)), ), - $group->result(flow_context(config()))->sortBy(ref('product')), + $this->aggregate($group, rows( + row(str_entry('product', 'Banana'), str_entry('country', 'USA'), int_entry('amount', 1000)), + row(str_entry('product', 'Apple'), str_entry('country', null), int_entry('amount', 400)), + ))->sortBy(ref('product')), ); } + + private function aggregate(GroupBy $groupBy, Rows $input): Rows + { + $result = new Rows(); + + $output = (new GroupByProcessor($groupBy))->process((static fn() => yield $input)(), flow_context(config())); + + /** @var Rows $batch */ + foreach ($output as $batch) { + $result = $result->merge($batch); + } + + return $result; + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php index fa42188bb7..0659e97f71 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php @@ -89,7 +89,12 @@ public function test_handles_empty_input(): void /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); - static::assertCount(1, $result); - static::assertCount(0, $result[0]); + $total = 0; + + foreach ($result as $batch) { + $total += $batch->count(); + } + + static::assertSame(0, $total); } } From e47c88008ed7a8ad61ea4087b7d91d5c29658036 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 08:15:24 +0200 Subject: [PATCH 03/13] refactor(flow-php/etl): adaptive hash aggregation with spill and merge Non-pivot aggregation now runs through hash aggregation with adaptive spilling: groups accumulate in an in-memory hash map, and once the configured memory limit is exceeded the map is spilled to a hash-sorted bucket and cleared. If nothing spills the map is the result; otherwise a streaming k-way merge folds partial aggregates by group key, so spilled state is O(groups), not O(rows). - merge() moves onto the base AggregatingFunction contract (no separate opt-in interface); every built-in aggregator implements it and self-validates by reference and result-affecting arguments - HashAggregation + GroupState / BucketGroup / GroupsMinHeap / GroupBucketsCache (FilesystemGroupBucketsCache), mirroring the external-sort building blocks - GroupBy becomes a spec (references/aggregations/hash/keyValues/newAggregators); pivot stays memory-only - config: aggregationMemoryLimit(Unit) (default unbounded, opt-in spilling) replaces the algorithm enum; output streamed in batches - fix multi-column group-by key hash collision (length-prefix the values) --- .../Config/Aggregation/AggregationConfig.php | 6 +- .../Aggregation/AggregationConfigBuilder.php | 31 ++- .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 9 +- .../Flow/ETL/Function/AggregatingFunction.php | 2 + .../etl/src/Flow/ETL/Function/Average.php | 17 ++ .../etl/src/Flow/ETL/Function/Collect.php | 10 + .../src/Flow/ETL/Function/CollectUnique.php | 14 ++ src/core/etl/src/Flow/ETL/Function/Count.php | 13 ++ src/core/etl/src/Flow/ETL/Function/First.php | 9 + src/core/etl/src/Flow/ETL/Function/Last.php | 11 ++ src/core/etl/src/Flow/ETL/Function/Max.php | 19 ++ src/core/etl/src/Flow/ETL/Function/Min.php | 19 ++ .../src/Flow/ETL/Function/StringAggregate.php | 18 ++ src/core/etl/src/Flow/ETL/Function/Sum.php | 10 + src/core/etl/src/Flow/ETL/GroupBy.php | 28 +-- .../Flow/ETL/GroupBy/AggregatingAlgorithm.php | 25 --- .../ETL/GroupBy/AggregationAlgorithms.php | 25 --- .../etl/src/Flow/ETL/GroupBy/BucketGroup.php | 17 ++ .../FilesystemGroupBucketsCache.php | 96 +++++++++ .../Flow/ETL/GroupBy/ExternalAggregation.php | 110 ----------- .../Flow/ETL/GroupBy/GroupBucketsCache.php | 25 +++ .../etl/src/Flow/ETL/GroupBy/GroupState.php | 22 +++ .../src/Flow/ETL/GroupBy/GroupsMinHeap.php | 36 ++++ .../src/Flow/ETL/GroupBy/HashAggregation.php | 187 ++++++++++++++++++ .../Flow/ETL/GroupBy/MemoryAggregation.php | 62 ------ .../Flow/ETL/Processor/GroupByProcessor.php | 38 ++-- .../DataFrame/GroupByAggregationTest.php | 65 ++++-- .../AggregationConfigBuilderTest.php | 16 +- .../Function/AggregatingFunctionMergeTest.php | 113 +++++++++++ 29 files changed, 740 insertions(+), 313 deletions(-) delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketGroup.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/ExternalAggregation.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupState.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php index 1109d60a1c..cfcb9bd5ea 100644 --- a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php @@ -4,12 +4,14 @@ namespace Flow\ETL\Config\Aggregation; -use Flow\ETL\GroupBy\AggregationAlgorithms; +use Flow\ETL\Dataset\Memory\Unit; final readonly class AggregationConfig { + public const string AGGREGATION_MAX_MEMORY_ENV = 'FLOW_AGGREGATION_MAX_MEMORY'; + public function __construct( - public AggregationAlgorithms $algorithm, + public Unit $memoryLimit, public string $filesystemProtocol = 'file', ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php index 35277349c6..56446c2a6b 100644 --- a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php @@ -4,24 +4,28 @@ namespace Flow\ETL\Config\Aggregation; -use Flow\ETL\GroupBy\AggregationAlgorithms; +use Flow\ETL\Dataset\Memory\Unit; + +use function getenv; +use function is_string; + +use const PHP_INT_MAX; final class AggregationConfigBuilder { - private AggregationAlgorithms $algorithm = AggregationAlgorithms::MEMORY_AGGREGATION; - private string $filesystemProtocol = 'file'; - public function algorithm(AggregationAlgorithms $algorithm): self - { - $this->algorithm = $algorithm; - - return $this; - } + private ?Unit $memoryLimit = null; public function build(): AggregationConfig { - return new AggregationConfig($this->algorithm, $this->filesystemProtocol); + if ($this->memoryLimit === null) { + $envLimit = getenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); + + $this->memoryLimit = is_string($envLimit) ? Unit::fromString($envLimit) : Unit::fromBytes(PHP_INT_MAX); + } + + return new AggregationConfig($this->memoryLimit, $this->filesystemProtocol); } public function filesystemProtocol(string $protocol): self @@ -30,4 +34,11 @@ public function filesystemProtocol(string $protocol): self return $this; } + + public function memoryLimit(Unit $memoryLimit): self + { + $this->memoryLimit = $memoryLimit; + + return $this; + } } diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index f98012c89b..609add4da6 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -15,7 +15,6 @@ use Flow\ETL\Config\Telemetry\TelemetryOptions; use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Filesystem\FilesystemStreams; -use Flow\ETL\GroupBy\AggregationAlgorithms; use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Pipeline\Optimizer; use Flow\ETL\Pipeline\Optimizer\BatchSizeOptimization; @@ -84,16 +83,16 @@ public function __construct() : PackageVersion::get('flow-php/etl'); } - public function aggregationAlgorithm(AggregationAlgorithms $algorithm): self + public function aggregationFilesystem(string $protocol): self { - $this->aggregation->algorithm($algorithm); + $this->aggregation->filesystemProtocol($protocol); return $this; } - public function aggregationFilesystem(string $protocol): self + public function aggregationMemoryLimit(Unit $unit): self { - $this->aggregation->filesystemProtocol($protocol); + $this->aggregation->memoryLimit($unit); return $this; } diff --git a/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php b/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php index 925e32da6b..b65743a9c3 100644 --- a/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php +++ b/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php @@ -13,6 +13,8 @@ interface AggregatingFunction { public function aggregate(Row $row, FlowContext $context): void; + public function merge(self $other): void; + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Average.php b/src/core/etl/src/Flow/ETL/Function/Average.php index d404adea22..00fe1a8775 100644 --- a/src/core/etl/src/Flow/ETL/Function/Average.php +++ b/src/core/etl/src/Flow/ETL/Function/Average.php @@ -82,6 +82,23 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return (new Calculator())->divide($sum, $count, $this->scale, $this->rounding); } + public function merge(AggregatingFunction $other): void + { + if ( + !$other instanceof self + || !$this->ref->is($other->ref) + || $this->scale !== $other->scale + || $this->rounding !== $other->rounding + ) { + throw new InvalidArgumentException( + 'Cannot merge ' . self::class . ' aggregating a different reference, scale or rounding', + ); + } + + $this->sum += $other->sum; + $this->count += $other->count; + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/Collect.php b/src/core/etl/src/Flow/ETL/Function/Collect.php index 12209f5710..93614b95e4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Collect.php +++ b/src/core/etl/src/Flow/ETL/Function/Collect.php @@ -11,6 +11,7 @@ use Flow\ETL\Row\EntryFactory; use Flow\ETL\Row\Reference; +use function array_merge; use function current; use function Flow\ETL\DSL\to_entry; @@ -41,6 +42,15 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + $this->collection = array_merge($this->collection, $other->collection); + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php index 3f87b0ff09..e4c9173c94 100644 --- a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php +++ b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php @@ -49,6 +49,20 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + /** @var mixed $value */ + foreach ($other->collection as $value) { + if (!in_array($value, $this->collection, true)) { + $this->collection[] = $value; + } + } + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Count.php b/src/core/etl/src/Flow/ETL/Function/Count.php index 52e585ef3c..ddd9459dfc 100644 --- a/src/core/etl/src/Flow/ETL/Function/Count.php +++ b/src/core/etl/src/Flow/ETL/Function/Count.php @@ -80,6 +80,19 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $count; } + public function merge(AggregatingFunction $other): void + { + if ( + !$other instanceof self + || ($this->ref === null) !== ($other->ref === null) + || ($this->ref !== null && $other->ref !== null && !$this->ref->is($other->ref)) + ) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + $this->count += $other->count; + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/Function/First.php b/src/core/etl/src/Flow/ETL/Function/First.php index 36126f0176..7be217b5a6 100644 --- a/src/core/etl/src/Flow/ETL/Function/First.php +++ b/src/core/etl/src/Flow/ETL/Function/First.php @@ -37,6 +37,15 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + $this->first ??= $other->first; + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Last.php b/src/core/etl/src/Flow/ETL/Function/Last.php index d5c1e76f76..6dd8f36b8d 100644 --- a/src/core/etl/src/Flow/ETL/Function/Last.php +++ b/src/core/etl/src/Flow/ETL/Function/Last.php @@ -35,6 +35,17 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + if ($other->last !== null) { + $this->last = $other->last; + } + } + /** * @return Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Max.php b/src/core/etl/src/Flow/ETL/Function/Max.php index 6794783b8c..36b88e69f4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Max.php +++ b/src/core/etl/src/Flow/ETL/Function/Max.php @@ -52,6 +52,25 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + if ($other->max === null) { + return; + } + + if ($this->max === null) { + $this->max = $other->max; + + return; + } + + $this->max = max($this->max, $other->max); + } + /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Min.php b/src/core/etl/src/Flow/ETL/Function/Min.php index 4960439cae..c3d5578fd5 100644 --- a/src/core/etl/src/Flow/ETL/Function/Min.php +++ b/src/core/etl/src/Flow/ETL/Function/Min.php @@ -52,6 +52,25 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + if ($other->min === null) { + return; + } + + if ($this->min === null) { + $this->min = $other->min; + + return; + } + + $this->min = min($this->min, $other->min); + } + /** * @return Entry|Entry|Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php index 13f1e468a3..f190a7a203 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Row; use Flow\ETL\Row\Entry; @@ -11,6 +12,7 @@ use Flow\ETL\Row\Reference; use Flow\ETL\Row\SortOrder; +use function array_merge; use function count; use function Flow\ETL\DSL\str_entry; use function implode; @@ -40,6 +42,22 @@ public function aggregate(Row $row, FlowContext $context): void } } + public function merge(AggregatingFunction $other): void + { + if ( + !$other instanceof self + || !$this->ref->is($other->ref) + || $this->separator !== $other->separator + || $this->sort !== $other->sort + ) { + throw new InvalidArgumentException( + 'Cannot merge ' . self::class . ' aggregating a different reference, separator or sort', + ); + } + + $this->values = array_merge($this->values, $other->values); + } + /** * @return Row\Entry */ diff --git a/src/core/etl/src/Flow/ETL/Function/Sum.php b/src/core/etl/src/Flow/ETL/Function/Sum.php index 77415475c3..4289b5263a 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sum.php +++ b/src/core/etl/src/Flow/ETL/Function/Sum.php @@ -68,6 +68,16 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed return $sum; } + public function merge(AggregatingFunction $other): void + { + if (!$other instanceof self || !$this->ref->is($other->ref)) { + throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); + } + + // @mago-ignore analysis:possibly-invalid-argument + $this->sum = (new Calculator())->add($this->sum, $other->sum); + } + public function over(Window $window): WindowFunction { $this->window = $window; diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 0ce6f8ca6f..4594280273 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -27,18 +27,11 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_union; -use function implode; use function is_array; use function is_scalar; use function serialize; +use function strlen; -/** - * Specification of a group-by: the key columns and the aggregations to run. - * - * The actual accumulation of non-pivot groups is done by the aggregating algorithms - * (see {@see \Flow\ETL\GroupBy\AggregatingAlgorithm}); this class only exposes the spec - * and small helpers they need. Pivot is a memory-only cross-tab and is executed here. - */ final class GroupBy { private const int RESULT_BATCH_SIZE = 1000; @@ -73,8 +66,6 @@ public function aggregate(AggregatingFunction ...$aggregator): void } /** - * Build the output row for one finalized group. - * * @param array $keyValues * @param array $aggregators */ @@ -103,8 +94,6 @@ public function aggregations(): array } /** - * A stable string key for the given group-by values (used to bucket equal groups). - * * @param array $values */ public function hash(array $values): string @@ -131,7 +120,13 @@ public function hash(array $values): string } } - return NativePHPHash::xxh128(implode('', $stringValues)); + $key = ''; + + foreach ($stringValues as $stringValue) { + $key .= strlen($stringValue) . ':' . $stringValue; + } + + return NativePHPHash::xxh128($key); } public function isPivot(): bool @@ -140,8 +135,6 @@ public function isPivot(): bool } /** - * The group-by key values for a single row (missing columns become null). - * * @return array */ public function keyValues(Row $row): array @@ -160,8 +153,6 @@ public function keyValues(Row $row): array } /** - * A fresh set of aggregator instances for a new group. - * * @return array */ public function newAggregators(): array @@ -178,9 +169,6 @@ public function pivot(Reference $ref): void } /** - * Execute the pivot cross-tab. Pivot is memory-only (it must know every distinct pivot - * value to build the columns), so it always aggregates in memory and streams the result. - * * @param Generator $rows * * @return Generator diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php deleted file mode 100644 index 8e4f54e301..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/AggregatingAlgorithm.php +++ /dev/null @@ -1,25 +0,0 @@ - $rows - * - * @return Generator - */ - public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator; -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php deleted file mode 100644 index 5ef1902306..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/AggregationAlgorithms.php +++ /dev/null @@ -1,25 +0,0 @@ - $chunkSize + */ + public function __construct( + private Filesystem $filesystem, + private Serializer $serializer = new NativePHPSerializer(), + private int $chunkSize = 100, + ?Path $cacheDir = null, + ) { + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->chunkSize < 1) { + throw new InvalidArgumentException('Chunk size must be greater than 0'); + } + + $this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); + } + + public function get(string $bucketId): Generator + { + $path = $this->keyPath($bucketId); + + if (!$this->filesystem->status($path)) { + return; + } + + $stream = $this->filesystem->readFrom($path); + + foreach ($stream->readLines() as $line) { + $separator = strpos($line, ' '); + $hash = substr($line, 0, $separator); + $serialized = substr($line, $separator + 1); + + yield [$hash, $this->serializer->unserialize($serialized, [GroupState::class])]; + } + + $stream->close(); + } + + public function remove(string $bucketId): void + { + $this->filesystem->rm($this->keyPath($bucketId)->parentDirectory()); + } + + public function set(string $bucketId, iterable $groups): void + { + $stream = $this->filesystem->writeTo($this->keyPath($bucketId)); + + $buffer = ''; + $counter = 0; + + foreach ($groups as [$hash, $state]) { + $buffer .= $hash . ' ' . $this->serializer->serialize($state) . "\n"; + $counter++; + + if ($counter >= $this->chunkSize) { + $stream->append($buffer); + $buffer = ''; + $counter = 0; + } + } + + if ($counter > 0) { + $stream->append($buffer); + } + + $stream->close(); + } + + private function keyPath(string $key): Path + { + return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/ExternalAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/ExternalAggregation.php deleted file mode 100644 index c8324cf22f..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/ExternalAggregation.php +++ /dev/null @@ -1,110 +0,0 @@ -references()->all(); - $sortRefs[] = ref(self::INPUT_ORDINAL); - - $sorted = $this->sort->sortGenerator($this->withInputOrdinal($rows), $context, References::init(...$sortRefs)); - - $currentHash = null; - /** @var array $currentKey */ - $currentKey = []; - /** @var array $aggregators */ - $aggregators = []; - $buffer = []; - - foreach ($sorted as $batch) { - foreach ($batch as $row) { - $keyValues = $groupBy->keyValues($row); - $hash = $groupBy->hash($keyValues); - - if ($hash !== $currentHash) { - if ($currentHash !== null) { - $buffer[] = $groupBy->aggregatedRow($currentKey, $aggregators, $context->entryFactory()); - - if (count($buffer) >= self::BATCH_SIZE) { - yield new Rows(...$buffer); - $buffer = []; - } - } - - $currentHash = $hash; - $currentKey = $keyValues; - $aggregators = $groupBy->newAggregators(); - } - - foreach ($aggregators as $aggregator) { - $aggregator->aggregate($row, $context); - } - } - } - - if ($currentHash !== null) { - $buffer[] = $groupBy->aggregatedRow($currentKey, $aggregators, $context->entryFactory()); - } - - if ($buffer !== []) { - yield new Rows(...$buffer); - } - } - - /** - * @param Generator $rows - * - * @return Generator - */ - private function withInputOrdinal(Generator $rows): Generator - { - $ordinal = 0; - - foreach ($rows as $batch) { - $tagged = []; - - foreach ($batch as $row) { - $tagged[] = $row->add(int_entry(self::INPUT_ORDINAL, $ordinal)); - $ordinal++; - } - - yield new Rows(...$tagged); - } - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php new file mode 100644 index 0000000000..1c1fd5352b --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php @@ -0,0 +1,25 @@ + + */ + public function get(string $bucketId): Generator; + + public function remove(string $bucketId): void; + + /** + * @param iterable $groups + */ + public function set(string $bucketId, iterable $groups): void; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupState.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupState.php new file mode 100644 index 0000000000..ded39a6a25 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupState.php @@ -0,0 +1,22 @@ + $keyValues + * @param array $aggregators + */ + public function __construct( + public array $keyValues, + public array $aggregators, + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php new file mode 100644 index 0000000000..8bd8180101 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php @@ -0,0 +1,36 @@ + + */ +final class GroupsMinHeap extends SplMinHeap +{ + public function insert(mixed $value): true + { + if (!$value instanceof BucketGroup) { + throw new InvalidArgumentException('Value inserted into GroupsMinHeap must be a ' . BucketGroup::class); + } + + parent::insert($value); + + return true; + } + + /** + * @param BucketGroup $value1 + * @param BucketGroup $value2 + */ + protected function compare($value1, $value2): int + { + return [$value2->hash, $value2->bucketIndex] <=> [$value1->hash, $value1->bucketIndex]; + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php new file mode 100644 index 0000000000..cd2bb712c1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php @@ -0,0 +1,187 @@ + $rows + * + * @return Generator + */ + public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator + { + $consumption = new Consumption(false); + /** @var array $hot */ + $hot = []; + /** @var list $buckets */ + $buckets = []; + + foreach ($rows as $batch) { + foreach ($batch as $row) { + $keyValues = $groupBy->keyValues($row); + $hash = $groupBy->hash($keyValues); + + if (!array_key_exists($hash, $hot)) { + $hot[$hash] = new GroupState($keyValues, $groupBy->newAggregators()); + } + + foreach ($hot[$hash]->aggregators as $aggregator) { + $aggregator->aggregate($row, $context); + } + } + + if ($consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { + $buckets[] = $this->spill($hot); + $hot = []; + $consumption = new Consumption(false); + } + } + + $groups = $buckets === [] ? $this->iterate($hot) : $this->merge($buckets, $hot); + + yield from $this->emit($groups, $groupBy, $context); + } + + /** + * @param Generator $groups + * + * @return Generator + */ + private function emit(Generator $groups, GroupBy $groupBy, FlowContext $context): Generator + { + $buffer = []; + + foreach ($groups as $state) { + $buffer[] = $groupBy->aggregatedRow($state->keyValues, $state->aggregators, $context->entryFactory()); + + if (count($buffer) >= self::BATCH_SIZE) { + yield new Rows(...$buffer); + $buffer = []; + } + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + } + } + + /** + * @param array $hot + * + * @return Generator + */ + private function iterate(array $hot): Generator + { + foreach ($hot as $state) { + yield $state; + } + } + + /** + * @param list $buckets + * @param array $hot + * + * @return Generator + */ + private function merge(array $buckets, array $hot): Generator + { + ksort($hot); + + /** @var array> $cursors */ + $cursors = []; + + foreach ($buckets as $index => $bucketId) { + $cursors[$index] = $this->cache->get($bucketId); + } + + $cursors[count($buckets)] = (static function () use ($hot): Generator { + foreach ($hot as $hash => $state) { + yield [$hash, $state]; + } + })(); + + $heap = new GroupsMinHeap(); + + foreach ($cursors as $index => $cursor) { + $this->advance($cursor, $index, $heap); + } + + while (!$heap->isEmpty()) { + $current = $heap->extract(); + $merged = $current->state; + $this->advance($cursors[$current->bucketIndex], $current->bucketIndex, $heap); + + while (!$heap->isEmpty() && $heap->top()->hash === $current->hash) { + $next = $heap->extract(); + + foreach ($merged->aggregators as $index => $aggregator) { + $aggregator->merge($next->state->aggregators[$index]); + } + + $this->advance($cursors[$next->bucketIndex], $next->bucketIndex, $heap); + } + + yield $merged; + } + + foreach ($buckets as $bucketId) { + $this->cache->remove($bucketId); + } + } + + /** + * @param Generator $cursor + */ + private function advance(Generator $cursor, int $bucketIndex, GroupsMinHeap $heap): void + { + if ($cursor->valid()) { + [$hash, $state] = $cursor->current(); + $heap->insert(new BucketGroup($hash, $bucketIndex, $state)); + $cursor->next(); + } + } + + /** + * @param array $hot + */ + private function spill(array $hot): string + { + ksort($hot); + + $bucketId = bin2hex(random_bytes(16)); + + $this->cache->set($bucketId, (static function () use ($hot): Generator { + foreach ($hot as $hash => $state) { + yield [$hash, $state]; + } + })()); + + return $bucketId; + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php deleted file mode 100644 index daac717d81..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/MemoryAggregation.php +++ /dev/null @@ -1,62 +0,0 @@ -, aggregators: array}> $groups */ - $groups = []; - - foreach ($rows as $batch) { - foreach ($batch as $row) { - $keyValues = $groupBy->keyValues($row); - $hash = $groupBy->hash($keyValues); - - if (!array_key_exists($hash, $groups)) { - $groups[$hash] = ['keyValues' => $keyValues, 'aggregators' => $groupBy->newAggregators()]; - } - - foreach ($groups[$hash]['aggregators'] as $aggregator) { - $aggregator->aggregate($row, $context); - } - } - } - - $buffer = []; - - foreach ($groups as $group) { - $buffer[] = $groupBy->aggregatedRow($group['keyValues'], $group['aggregators'], $context->entryFactory()); - - if (count($buffer) >= self::BATCH_SIZE) { - yield new Rows(...$buffer); - $buffer = []; - } - } - - if ($buffer !== []) { - yield new Rows(...$buffer); - } - } -} diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index 53e01179cb..ccc4eb59e5 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -6,21 +6,14 @@ use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; -use Flow\ETL\GroupBy\AggregatingAlgorithm; -use Flow\ETL\GroupBy\ExternalAggregation; -use Flow\ETL\GroupBy\MemoryAggregation; +use Flow\ETL\GroupBy\BucketsCache\FilesystemGroupBucketsCache; +use Flow\ETL\GroupBy\HashAggregation; use Flow\ETL\Processor; -use Flow\ETL\Sort\ExternalSort; -use Flow\ETL\Sort\ExternalSort\BucketsCache\FilesystemBucketsCache; use Generator; /** * Groups all rows and applies aggregation functions. * - * Picks the aggregation algorithm from configuration (mirroring SortingProcessor): in-memory - * by default, or an external sort-based fold for bounded memory. Pivot and global aggregation - * (no group-by columns) always run in memory. - * * @internal */ final readonly class GroupByProcessor implements Processor @@ -37,27 +30,18 @@ public function process(Generator $rows, FlowContext $context): Generator return; } - yield from $this->algorithm($context)->aggregate($rows, $context, $this->groupBy); - } - - private function algorithm(FlowContext $context): AggregatingAlgorithm - { $config = $context->config->aggregation; - if ($config->algorithm->useMemory() || $this->groupBy->references()->count() === 0) { - return new MemoryAggregation(); - } - - return new ExternalAggregation( - new ExternalSort( - new FilesystemBucketsCache( - $context->filesystem($config->filesystemProtocol), - $context->config->serializer(), - 100, - $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), - ), - $context->config->cache->externalSortBucketsCount, + $aggregation = new HashAggregation( + $config->memoryLimit, + new FilesystemGroupBucketsCache( + $context->filesystem($config->filesystemProtocol), + $context->config->serializer(), + 100, + $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), ), ); + + yield from $aggregation->aggregate($rows, $context, $this->groupBy); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 0dfbbbec41..857c7edd72 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -5,7 +5,7 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use Flow\ETL\Config\ConfigBuilder; -use Flow\ETL\GroupBy\AggregationAlgorithms; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -29,17 +29,15 @@ final class GroupByAggregationTest extends FlowIntegrationTestCase */ private static ?array $orders = null; - public function test_first_and_last_preserve_input_order_under_external_spill(): void + public function test_first_and_last_preserve_input_order_across_spilled_buckets(): void { - // 2000 rows interleaved into two groups, each spanning multiple spilled sort runs; - // v is the input position so first/last are unambiguous in input order. $data = []; - for ($i = 0; $i < 2000; $i++) { - $data[] = ['g' => ($i % 2) === 0 ? 'a' : 'b', 'v' => $i]; + for ($i = 0; $i < 400; $i++) { + $data[] = ['g' => $i % 2 === 0 ? 'a' : 'b', 'v' => $i]; } - $run = static fn(ConfigBuilder $config): array => df($config) + $run = static fn (ConfigBuilder $config): array => df($config) ->read(from_array($data)) ->groupBy('g') ->aggregate(first(ref('v')), last(ref('v'))) @@ -48,42 +46,71 @@ public function test_first_and_last_preserve_input_order_under_external_spill(): ->toArray(); $memory = $run(config_builder()); - $external = $run(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); + $spilled = $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1))); - self::assertSame($memory, $external); + self::assertSame($memory, $spilled); self::assertSame( [ - ['g' => 'a', 'v_first' => 0, 'v_last' => 1998], - ['g' => 'b', 'v_first' => 1, 'v_last' => 1999], + ['g' => 'a', 'v_first' => 0, 'v_last' => 398], + ['g' => 'b', 'v_first' => 1, 'v_last' => 399], ], - $external, + $spilled, ); } - public function test_high_cardinality_group_by_is_identical_between_memory_and_external(): void + public function test_high_cardinality_group_by_is_identical_when_spilling(): void { $memory = $this->aggregateByEmail(config_builder()); + $spilled = $this->aggregateByEmail(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); - $external = $this->aggregateByEmail(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); + self::assertEqualsCanonicalizing($memory, $spilled); + } + + public function test_multi_column_keys_do_not_collide(): void + { + $data = [ + ['x' => 'a', 'y' => 'bc'], + ['x' => 'ab', 'y' => 'c'], + ['x' => 'a', 'y' => 'bc'], + ]; + + $run = static fn (ConfigBuilder $config): array => df($config) + ->read(from_array($data)) + ->groupBy('x', 'y') + ->aggregate(count()) + ->sortBy(ref('x')) + ->fetch() + ->toArray(); - self::assertEqualsCanonicalizing($memory, $external); + $expected = [ + ['x' => 'a', 'y' => 'bc', '_count' => 2], + ['x' => 'ab', 'y' => 'c', '_count' => 1], + ]; + + self::assertSame($expected, $run(config_builder())); + self::assertSame($expected, $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1)))); } - public function test_seller_aggregation_is_identical_between_memory_and_external(): void + public function test_seller_aggregation_is_identical_when_spilling(): void { $memory = $this->aggregateBySeller(config_builder()); + $spilled = $this->aggregateBySeller(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); - $external = $this->aggregateBySeller(config_builder()->aggregationAlgorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION)); - - self::assertEqualsCanonicalizing($memory, $external); + self::assertEqualsCanonicalizing($memory, $spilled); self::assertCount(5, $memory); } + /** + * @return array> + */ private function aggregateByEmail(ConfigBuilder $config): array { return df($config)->read(from_array($this->orders()))->groupBy('email')->aggregate(count())->fetch()->toArray(); } + /** + * @return array> + */ private function aggregateBySeller(ConfigBuilder $config): array { return df($config) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php index 2763f800e0..7a5bf424c5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php @@ -5,29 +5,29 @@ namespace Flow\ETL\Tests\Unit\Config\Aggregation; use Flow\ETL\Config\Aggregation\AggregationConfigBuilder; -use Flow\ETL\GroupBy\AggregationAlgorithms; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\FlowTestCase; +use const PHP_INT_MAX; + final class AggregationConfigBuilderTest extends FlowTestCase { - public function test_default_algorithm_is_memory(): void + public function test_default_memory_limit_is_unbounded(): void { $config = (new AggregationConfigBuilder())->build(); - self::assertSame(AggregationAlgorithms::MEMORY_AGGREGATION, $config->algorithm); - self::assertTrue($config->algorithm->useMemory()); + self::assertSame(PHP_INT_MAX, $config->memoryLimit->inBytes()); self::assertSame('file', $config->filesystemProtocol); } - public function test_selecting_the_external_algorithm(): void + public function test_setting_a_memory_limit_and_filesystem(): void { $config = (new AggregationConfigBuilder()) - ->algorithm(AggregationAlgorithms::EXTERNAL_AGGREGATION) + ->memoryLimit(Unit::fromMb(16)) ->filesystemProtocol('memory') ->build(); - self::assertSame(AggregationAlgorithms::EXTERNAL_AGGREGATION, $config->algorithm); - self::assertFalse($config->algorithm->useMemory()); + self::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); self::assertSame('memory', $config->filesystemProtocol); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php new file mode 100644 index 0000000000..a5e8b43c11 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php @@ -0,0 +1,113 @@ +aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = average(ref('v')); + $right->aggregate(row(int_entry('v', 6)), $context); + + $left->merge($right); + + self::assertSame(3, $left->result($context->entryFactory())->value()); + } + + public function test_collect_merges_in_order(): void + { + $context = flow_context(); + + $left = collect(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + + $right = collect(ref('v')); + $right->aggregate(row(int_entry('v', 2)), $context); + + $left->merge($right); + + self::assertSame([1, 2], $left->result($context->entryFactory())->value()); + } + + public function test_first_keeps_the_earlier_partial(): void + { + $context = flow_context(); + + $left = first(ref('v')); + $left->aggregate(row(int_entry('v', 10)), $context); + + $right = first(ref('v')); + $right->aggregate(row(int_entry('v', 20)), $context); + + $left->merge($right); + + self::assertSame(10, $left->result($context->entryFactory())->value()); + } + + public function test_last_keeps_the_later_partial(): void + { + $context = flow_context(); + + $left = last(ref('v')); + $left->aggregate(row(int_entry('v', 10)), $context); + + $right = last(ref('v')); + $right->aggregate(row(int_entry('v', 20)), $context); + + $left->merge($right); + + self::assertSame(20, $left->result($context->entryFactory())->value()); + } + + public function test_merging_a_different_function_type_throws(): void + { + $this->expectException(InvalidArgumentException::class); + + sum(ref('v'))->merge(count()); + } + + public function test_merging_the_same_function_on_a_different_reference_throws(): void + { + $this->expectException(InvalidArgumentException::class); + + sum(ref('a'))->merge(sum(ref('b'))); + } + + public function test_sum_merges_partial_sums(): void + { + $context = flow_context(); + + $left = sum(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = sum(ref('v')); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame(6, $left->result($context->entryFactory())->value()); + } +} From 2d2734728c74c2dbc1c69f5024ce4d3f2d17bcab Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 08:43:10 +0200 Subject: [PATCH 04/13] test(flow-php/etl): fix static analysis and cover aggregation merge - guard strpos()===false and drop unused mago pragmas flagged by CI static analysis - annotate the bucket-cursor generators as Generator and narrow current() in the k-way merge - cover merge() on all 10 aggregators (success, null branches, self-validation), the min-heap ordering/guard, the bucket cache round-trip, and the memory-limit env var --- src/core/etl/src/Flow/ETL/GroupBy.php | 1 - .../FilesystemGroupBucketsCache.php | 14 +- .../Flow/ETL/GroupBy/GroupBucketsCache.php | 2 +- .../src/Flow/ETL/GroupBy/HashAggregation.php | 39 ++-- .../FilesystemGroupBucketsCacheTest.php | 50 +++++ .../AggregationConfigBuilderTest.php | 16 ++ .../Function/AggregatingFunctionMergeTest.php | 202 +++++++++++++++++- .../Tests/Unit/GroupBy/GroupsMinHeapTest.php | 38 ++++ 8 files changed, 337 insertions(+), 25 deletions(-) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 4594280273..7c816a3a1d 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -220,7 +220,6 @@ public function pivotResult(Generator $rows, FlowContext $context): Generator $pivotValue = type_union(type_string(), type_integer())->assert($pivotValue); if (!array_key_exists($pivotValue, $pivotedTable[$indexValue])) { - // @mago-ignore analysis:invalid-property-assignment-value,possibly-invalid-clone $pivotedTable[$indexValue][$pivotValue] = clone current($this->aggregations); } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php index a8d3f1c4bb..892ba2e276 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php @@ -4,7 +4,6 @@ namespace Flow\ETL\GroupBy\BucketsCache; -use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\GroupBy\GroupBucketsCache; use Flow\ETL\GroupBy\GroupState; use Flow\ETL\Hash\NativePHPHash; @@ -30,14 +29,12 @@ public function __construct( private int $chunkSize = 100, ?Path $cacheDir = null, ) { - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->chunkSize < 1) { - throw new InvalidArgumentException('Chunk size must be greater than 0'); - } - $this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); } + /** + * @return Generator + */ public function get(string $bucketId): Generator { $path = $this->keyPath($bucketId); @@ -50,6 +47,11 @@ public function get(string $bucketId): Generator foreach ($stream->readLines() as $line) { $separator = strpos($line, ' '); + + if ($separator === false) { + continue; + } + $hash = substr($line, 0, $separator); $serialized = substr($line, $separator + 1); diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php index 1c1fd5352b..40c329a3c9 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php @@ -12,7 +12,7 @@ interface GroupBucketsCache { /** - * @return Generator + * @return Generator */ public function get(string $bucketId): Generator; diff --git a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php index cd2bb712c1..06d265fade 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php @@ -113,18 +113,14 @@ private function merge(array $buckets, array $hot): Generator { ksort($hot); - /** @var array> $cursors */ + /** @var array> $cursors */ $cursors = []; foreach ($buckets as $index => $bucketId) { $cursors[$index] = $this->cache->get($bucketId); } - $cursors[count($buckets)] = (static function () use ($hot): Generator { - foreach ($hot as $hash => $state) { - yield [$hash, $state]; - } - })(); + $cursors[count($buckets)] = $this->hotCursor($hot); $heap = new GroupsMinHeap(); @@ -156,14 +152,29 @@ private function merge(array $buckets, array $hot): Generator } /** - * @param Generator $cursor + * @param Generator $cursor */ private function advance(Generator $cursor, int $bucketIndex, GroupsMinHeap $heap): void { - if ($cursor->valid()) { - [$hash, $state] = $cursor->current(); - $heap->insert(new BucketGroup($hash, $bucketIndex, $state)); - $cursor->next(); + if (!$cursor->valid()) { + return; + } + + /** @var array{0: string, 1: GroupState} $item */ + $item = $cursor->current(); + $heap->insert(new BucketGroup($item[0], $bucketIndex, $item[1])); + $cursor->next(); + } + + /** + * @param array $hot + * + * @return Generator + */ + private function hotCursor(array $hot): Generator + { + foreach ($hot as $hash => $state) { + yield [$hash, $state]; } } @@ -176,11 +187,7 @@ private function spill(array $hot): string $bucketId = bin2hex(random_bytes(16)); - $this->cache->set($bucketId, (static function () use ($hot): Generator { - foreach ($hot as $hash => $state) { - yield [$hash, $state]; - } - })()); + $this->cache->set($bucketId, $this->hotCursor($hot)); return $bucketId; } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php new file mode 100644 index 0000000000..4358b59b28 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php @@ -0,0 +1,50 @@ +fs(), $this->serializer(), 100, $this->cacheDir); + + self::assertSame([], iterator_to_array($cache->get('does-not-exist'))); + } + + public function test_round_trip_and_remove(): void + { + $cache = new FilesystemGroupBucketsCache($this->fs(), $this->serializer(), 2, $this->cacheDir); + + $groups = [ + ['aaa', new GroupState(['type' => 'a'], [sum(ref('v'))])], + ['bbb', new GroupState(['type' => 'b'], [sum(ref('v'))])], + ['ccc', new GroupState(['type' => 'c'], [sum(ref('v'))])], + ]; + + $cache->set('bucket-1', (static function () use ($groups) { + yield from $groups; + })()); + + $read = []; + + foreach ($cache->get('bucket-1') as [$hash, $state]) { + $read[$hash] = $state->keyValues; + } + + self::assertSame(['aaa' => ['type' => 'a'], 'bbb' => ['type' => 'b'], 'ccc' => ['type' => 'c']], $read); + + $cache->remove('bucket-1'); + + self::assertSame([], iterator_to_array($cache->get('bucket-1'))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php index 7a5bf424c5..1dc0315a04 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Tests\Unit\Config\Aggregation; +use Flow\ETL\Config\Aggregation\AggregationConfig; use Flow\ETL\Config\Aggregation\AggregationConfigBuilder; use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\FlowTestCase; +use function putenv; + use const PHP_INT_MAX; final class AggregationConfigBuilderTest extends FlowTestCase @@ -20,6 +23,19 @@ public function test_default_memory_limit_is_unbounded(): void self::assertSame('file', $config->filesystemProtocol); } + public function test_memory_limit_is_read_from_the_environment(): void + { + putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV . '=32M'); + + try { + $config = (new AggregationConfigBuilder())->build(); + + self::assertSame(Unit::fromMb(32)->inBytes(), $config->memoryLimit->inBytes()); + } finally { + putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); + } + } + public function test_setting_a_memory_limit_and_filesystem(): void { $config = (new AggregationConfigBuilder()) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php index a5e8b43c11..6e0c564827 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php @@ -5,17 +5,23 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Function\StringAggregate; +use Flow\ETL\Row\SortOrder; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\collect; +use function Flow\ETL\DSL\collect_unique; use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_entry; use function Flow\ETL\DSL\last; +use function Flow\ETL\DSL\max; +use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\str_entry; use function Flow\ETL\DSL\sum; final class AggregatingFunctionMergeTest extends FlowTestCase @@ -36,6 +42,13 @@ public function test_average_merges_partial_sums_and_counts(): void self::assertSame(3, $left->result($context->entryFactory())->value()); } + public function test_average_rejects_a_different_scale(): void + { + $this->expectException(InvalidArgumentException::class); + + average(ref('v'), 2)->merge(average(ref('v'), 4)); + } + public function test_collect_merges_in_order(): void { $context = flow_context(); @@ -45,10 +58,66 @@ public function test_collect_merges_in_order(): void $right = collect(ref('v')); $right->aggregate(row(int_entry('v', 2)), $context); + $right->aggregate(row(int_entry('v', 3)), $context); $left->merge($right); - self::assertSame([1, 2], $left->result($context->entryFactory())->value()); + self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + } + + public function test_collect_unique_merges_as_a_union(): void + { + $context = flow_context(); + + $left = collect_unique(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = collect_unique(ref('v')); + $right->aggregate(row(int_entry('v', 2)), $context); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + } + + public function test_count_merges_counts(): void + { + $context = flow_context(); + + $left = count(ref('v')); + $left->aggregate(row(int_entry('v', 1)), $context); + $left->aggregate(row(int_entry('v', 2)), $context); + + $right = count(ref('v')); + $right->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + self::assertSame(3, $left->result($context->entryFactory())->value()); + } + + public function test_count_without_reference_merges(): void + { + $context = flow_context(); + + $left = count(); + $left->aggregate(row(int_entry('v', 1)), $context); + + $right = count(); + $right->aggregate(row(int_entry('v', 2)), $context); + + $left->merge($right); + + self::assertSame(2, $left->result($context->entryFactory())->value()); + } + + public function test_count_rejects_a_referenced_and_unreferenced_mix(): void + { + $this->expectException(InvalidArgumentException::class); + + count(ref('v'))->merge(count()); } public function test_first_keeps_the_earlier_partial(): void @@ -66,6 +135,20 @@ public function test_first_keeps_the_earlier_partial(): void self::assertSame(10, $left->result($context->entryFactory())->value()); } + public function test_first_takes_the_other_when_empty(): void + { + $context = flow_context(); + + $left = first(ref('v')); + + $right = first(ref('v')); + $right->aggregate(row(int_entry('v', 20)), $context); + + $left->merge($right); + + self::assertSame(20, $left->result($context->entryFactory())->value()); + } + public function test_last_keeps_the_later_partial(): void { $context = flow_context(); @@ -81,6 +164,59 @@ public function test_last_keeps_the_later_partial(): void self::assertSame(20, $left->result($context->entryFactory())->value()); } + public function test_last_keeps_current_when_other_is_empty(): void + { + $context = flow_context(); + + $left = last(ref('v')); + $left->aggregate(row(int_entry('v', 10)), $context); + + $left->merge(last(ref('v'))); + + self::assertSame(10, $left->result($context->entryFactory())->value()); + } + + public function test_max_keeps_the_greatest(): void + { + $context = flow_context(); + + $left = max(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + + $right = max(ref('v')); + $right->aggregate(row(int_entry('v', 9)), $context); + + $left->merge($right); + + self::assertSame(9, $left->result($context->entryFactory())->value()); + } + + public function test_max_takes_the_other_when_empty(): void + { + $context = flow_context(); + + $left = max(ref('v')); + + $right = max(ref('v')); + $right->aggregate(row(int_entry('v', 9)), $context); + + $left->merge($right); + + self::assertSame(9, $left->result($context->entryFactory())->value()); + } + + public function test_max_keeps_current_when_other_is_empty(): void + { + $context = flow_context(); + + $left = max(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + + $left->merge(max(ref('v'))); + + self::assertSame(5, $left->result($context->entryFactory())->value()); + } + public function test_merging_a_different_function_type_throws(): void { $this->expectException(InvalidArgumentException::class); @@ -95,6 +231,70 @@ public function test_merging_the_same_function_on_a_different_reference_throws() sum(ref('a'))->merge(sum(ref('b'))); } + public function test_min_keeps_the_smallest(): void + { + $context = flow_context(); + + $left = min(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + + $right = min(ref('v')); + $right->aggregate(row(int_entry('v', 2)), $context); + + $left->merge($right); + + self::assertSame(2, $left->result($context->entryFactory())->value()); + } + + public function test_min_takes_the_other_when_empty(): void + { + $context = flow_context(); + + $left = min(ref('v')); + + $right = min(ref('v')); + $right->aggregate(row(int_entry('v', 2)), $context); + + $left->merge($right); + + self::assertSame(2, $left->result($context->entryFactory())->value()); + } + + public function test_min_keeps_current_when_other_is_empty(): void + { + $context = flow_context(); + + $left = min(ref('v')); + $left->aggregate(row(int_entry('v', 5)), $context); + + $left->merge(min(ref('v'))); + + self::assertSame(5, $left->result($context->entryFactory())->value()); + } + + public function test_string_aggregate_merges_values(): void + { + $context = flow_context(); + + $left = new StringAggregate(ref('v'), ',', SortOrder::ASC); + $left->aggregate(row(str_entry('v', 'b')), $context); + + $right = new StringAggregate(ref('v'), ',', SortOrder::ASC); + $right->aggregate(row(str_entry('v', 'a')), $context); + $right->aggregate(row(str_entry('v', 'c')), $context); + + $left->merge($right); + + self::assertSame('a,b,c', $left->result($context->entryFactory())->value()); + } + + public function test_string_aggregate_rejects_a_different_separator(): void + { + $this->expectException(InvalidArgumentException::class); + + (new StringAggregate(ref('v'), ','))->merge(new StringAggregate(ref('v'), ';')); + } + public function test_sum_merges_partial_sums(): void { $context = flow_context(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php new file mode 100644 index 0000000000..d8aa49159f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php @@ -0,0 +1,38 @@ +insert(new BucketGroup('b', 0, new GroupState([], []))); + $heap->insert(new BucketGroup('a', 2, new GroupState([], []))); + $heap->insert(new BucketGroup('a', 1, new GroupState([], []))); + + $order = []; + + while (!$heap->isEmpty()) { + $group = $heap->extract(); + $order[] = $group->hash . $group->bucketIndex; + } + + self::assertSame(['a1', 'a2', 'b0'], $order); + } + + public function test_rejects_a_non_bucket_group(): void + { + $this->expectException(InvalidArgumentException::class); + + (new GroupsMinHeap())->insert('not-a-bucket-group'); + } +} From 2f6072a719ab2390b2b8d2b9c40e3b559c234483 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 08:49:40 +0200 Subject: [PATCH 05/13] fix(flow-php/etl): resolve remaining static analysis findings - drop the unused mago pragma in Sum::merge - hoist the pivot prototype behind an instanceof guard so clone is never applied to a possibly-false current() - drop over-specific @return docblocks that toArray() cannot satisfy --- src/core/etl/src/Flow/ETL/Function/Sum.php | 1 - src/core/etl/src/Flow/ETL/GroupBy.php | 8 +++++++- .../Integration/DataFrame/GroupByAggregationTest.php | 6 ------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/core/etl/src/Flow/ETL/Function/Sum.php b/src/core/etl/src/Flow/ETL/Function/Sum.php index 4289b5263a..9addb4f814 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sum.php +++ b/src/core/etl/src/Flow/ETL/Function/Sum.php @@ -74,7 +74,6 @@ public function merge(AggregatingFunction $other): void throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); } - // @mago-ignore analysis:possibly-invalid-argument $this->sum = (new Calculator())->add($this->sum, $other->sum); } diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 7c816a3a1d..264fbb886e 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -181,6 +181,12 @@ public function pivotResult(Generator $rows, FlowContext $context): Generator throw new RuntimeException('pivotResult() called without a pivot reference'); } + $aggregation = current($this->aggregations); + + if (!$aggregation instanceof AggregatingFunction) { + throw new RuntimeException('Pivot requires exactly one aggregation'); + } + /** @var array|bool|float|int|object|string> $pivotColumns */ $pivotColumns = []; /** @var array|bool|float|int|object|string>> $pivotedTable */ @@ -220,7 +226,7 @@ public function pivotResult(Generator $rows, FlowContext $context): Generator $pivotValue = type_union(type_string(), type_integer())->assert($pivotValue); if (!array_key_exists($pivotValue, $pivotedTable[$indexValue])) { - $pivotedTable[$indexValue][$pivotValue] = clone current($this->aggregations); + $pivotedTable[$indexValue][$pivotValue] = clone $aggregation; } $aggregator = $pivotedTable[$indexValue][$pivotValue]; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 857c7edd72..631c975c34 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -100,17 +100,11 @@ public function test_seller_aggregation_is_identical_when_spilling(): void self::assertCount(5, $memory); } - /** - * @return array> - */ private function aggregateByEmail(ConfigBuilder $config): array { return df($config)->read(from_array($this->orders()))->groupBy('email')->aggregate(count())->fetch()->toArray(); } - /** - * @return array> - */ private function aggregateBySeller(ConfigBuilder $config): array { return df($config) From b854046a42fc79019ea7cebee5faf02443c3ef0c Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 08:56:15 +0200 Subject: [PATCH 06/13] style(flow-php/etl): apply mago formatting --- src/core/etl/src/Flow/ETL/Function/Count.php | 2 +- .../Integration/DataFrame/GroupByAggregationTest.php | 6 +++--- .../GroupBy/FilesystemGroupBucketsCacheTest.php | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/core/etl/src/Flow/ETL/Function/Count.php b/src/core/etl/src/Flow/ETL/Function/Count.php index ddd9459dfc..8fc456391a 100644 --- a/src/core/etl/src/Flow/ETL/Function/Count.php +++ b/src/core/etl/src/Flow/ETL/Function/Count.php @@ -85,7 +85,7 @@ public function merge(AggregatingFunction $other): void if ( !$other instanceof self || ($this->ref === null) !== ($other->ref === null) - || ($this->ref !== null && $other->ref !== null && !$this->ref->is($other->ref)) + || $this->ref !== null && $other->ref !== null && !$this->ref->is($other->ref) ) { throw new InvalidArgumentException('Cannot merge ' . self::class . ' aggregating a different reference'); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 631c975c34..6e1e837b41 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -34,10 +34,10 @@ public function test_first_and_last_preserve_input_order_across_spilled_buckets( $data = []; for ($i = 0; $i < 400; $i++) { - $data[] = ['g' => $i % 2 === 0 ? 'a' : 'b', 'v' => $i]; + $data[] = ['g' => ($i % 2) === 0 ? 'a' : 'b', 'v' => $i]; } - $run = static fn (ConfigBuilder $config): array => df($config) + $run = static fn(ConfigBuilder $config): array => df($config) ->read(from_array($data)) ->groupBy('g') ->aggregate(first(ref('v')), last(ref('v'))) @@ -74,7 +74,7 @@ public function test_multi_column_keys_do_not_collide(): void ['x' => 'a', 'y' => 'bc'], ]; - $run = static fn (ConfigBuilder $config): array => df($config) + $run = static fn(ConfigBuilder $config): array => df($config) ->read(from_array($data)) ->groupBy('x', 'y') ->aggregate(count()) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php index 4358b59b28..ecafda4acb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php @@ -31,9 +31,12 @@ public function test_round_trip_and_remove(): void ['ccc', new GroupState(['type' => 'c'], [sum(ref('v'))])], ]; - $cache->set('bucket-1', (static function () use ($groups) { - yield from $groups; - })()); + $cache->set( + 'bucket-1', + (static function () use ($groups) { + yield from $groups; + })(), + ); $read = []; From a59d0feaff52666911f39ee2acaf4dd92aa2ff50 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 09:02:19 +0200 Subject: [PATCH 07/13] style(flow-php/etl): use static:: assertions in new aggregation tests --- .../DataFrame/GroupByAggregationTest.php | 14 ++++---- .../FilesystemGroupBucketsCacheTest.php | 6 ++-- .../AggregationConfigBuilderTest.php | 10 +++--- .../Function/AggregatingFunctionMergeTest.php | 34 +++++++++---------- .../Tests/Unit/GroupBy/GroupsMinHeapTest.php | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 6e1e837b41..87ec052ccb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -48,8 +48,8 @@ public function test_first_and_last_preserve_input_order_across_spilled_buckets( $memory = $run(config_builder()); $spilled = $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1))); - self::assertSame($memory, $spilled); - self::assertSame( + static::assertSame($memory, $spilled); + static::assertSame( [ ['g' => 'a', 'v_first' => 0, 'v_last' => 398], ['g' => 'b', 'v_first' => 1, 'v_last' => 399], @@ -63,7 +63,7 @@ public function test_high_cardinality_group_by_is_identical_when_spilling(): voi $memory = $this->aggregateByEmail(config_builder()); $spilled = $this->aggregateByEmail(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); - self::assertEqualsCanonicalizing($memory, $spilled); + static::assertEqualsCanonicalizing($memory, $spilled); } public function test_multi_column_keys_do_not_collide(): void @@ -87,8 +87,8 @@ public function test_multi_column_keys_do_not_collide(): void ['x' => 'ab', 'y' => 'c', '_count' => 1], ]; - self::assertSame($expected, $run(config_builder())); - self::assertSame($expected, $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1)))); + static::assertSame($expected, $run(config_builder())); + static::assertSame($expected, $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1)))); } public function test_seller_aggregation_is_identical_when_spilling(): void @@ -96,8 +96,8 @@ public function test_seller_aggregation_is_identical_when_spilling(): void $memory = $this->aggregateBySeller(config_builder()); $spilled = $this->aggregateBySeller(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); - self::assertEqualsCanonicalizing($memory, $spilled); - self::assertCount(5, $memory); + static::assertEqualsCanonicalizing($memory, $spilled); + static::assertCount(5, $memory); } private function aggregateByEmail(ConfigBuilder $config): array diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php index ecafda4acb..2cc4628b0c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php @@ -18,7 +18,7 @@ public function test_missing_bucket_yields_nothing(): void { $cache = new FilesystemGroupBucketsCache($this->fs(), $this->serializer(), 100, $this->cacheDir); - self::assertSame([], iterator_to_array($cache->get('does-not-exist'))); + static::assertSame([], iterator_to_array($cache->get('does-not-exist'))); } public function test_round_trip_and_remove(): void @@ -44,10 +44,10 @@ public function test_round_trip_and_remove(): void $read[$hash] = $state->keyValues; } - self::assertSame(['aaa' => ['type' => 'a'], 'bbb' => ['type' => 'b'], 'ccc' => ['type' => 'c']], $read); + static::assertSame(['aaa' => ['type' => 'a'], 'bbb' => ['type' => 'b'], 'ccc' => ['type' => 'c']], $read); $cache->remove('bucket-1'); - self::assertSame([], iterator_to_array($cache->get('bucket-1'))); + static::assertSame([], iterator_to_array($cache->get('bucket-1'))); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php index 1dc0315a04..3dd6226210 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php @@ -19,8 +19,8 @@ public function test_default_memory_limit_is_unbounded(): void { $config = (new AggregationConfigBuilder())->build(); - self::assertSame(PHP_INT_MAX, $config->memoryLimit->inBytes()); - self::assertSame('file', $config->filesystemProtocol); + static::assertSame(PHP_INT_MAX, $config->memoryLimit->inBytes()); + static::assertSame('file', $config->filesystemProtocol); } public function test_memory_limit_is_read_from_the_environment(): void @@ -30,7 +30,7 @@ public function test_memory_limit_is_read_from_the_environment(): void try { $config = (new AggregationConfigBuilder())->build(); - self::assertSame(Unit::fromMb(32)->inBytes(), $config->memoryLimit->inBytes()); + static::assertSame(Unit::fromMb(32)->inBytes(), $config->memoryLimit->inBytes()); } finally { putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); } @@ -43,7 +43,7 @@ public function test_setting_a_memory_limit_and_filesystem(): void ->filesystemProtocol('memory') ->build(); - self::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); - self::assertSame('memory', $config->filesystemProtocol); + static::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); + static::assertSame('memory', $config->filesystemProtocol); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php index 6e0c564827..1c3b07defe 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AggregatingFunctionMergeTest.php @@ -39,7 +39,7 @@ public function test_average_merges_partial_sums_and_counts(): void $left->merge($right); - self::assertSame(3, $left->result($context->entryFactory())->value()); + static::assertSame(3, $left->result($context->entryFactory())->value()); } public function test_average_rejects_a_different_scale(): void @@ -62,7 +62,7 @@ public function test_collect_merges_in_order(): void $left->merge($right); - self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + static::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); } public function test_collect_unique_merges_as_a_union(): void @@ -79,7 +79,7 @@ public function test_collect_unique_merges_as_a_union(): void $left->merge($right); - self::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); + static::assertSame([1, 2, 3], $left->result($context->entryFactory())->value()); } public function test_count_merges_counts(): void @@ -95,7 +95,7 @@ public function test_count_merges_counts(): void $left->merge($right); - self::assertSame(3, $left->result($context->entryFactory())->value()); + static::assertSame(3, $left->result($context->entryFactory())->value()); } public function test_count_without_reference_merges(): void @@ -110,7 +110,7 @@ public function test_count_without_reference_merges(): void $left->merge($right); - self::assertSame(2, $left->result($context->entryFactory())->value()); + static::assertSame(2, $left->result($context->entryFactory())->value()); } public function test_count_rejects_a_referenced_and_unreferenced_mix(): void @@ -132,7 +132,7 @@ public function test_first_keeps_the_earlier_partial(): void $left->merge($right); - self::assertSame(10, $left->result($context->entryFactory())->value()); + static::assertSame(10, $left->result($context->entryFactory())->value()); } public function test_first_takes_the_other_when_empty(): void @@ -146,7 +146,7 @@ public function test_first_takes_the_other_when_empty(): void $left->merge($right); - self::assertSame(20, $left->result($context->entryFactory())->value()); + static::assertSame(20, $left->result($context->entryFactory())->value()); } public function test_last_keeps_the_later_partial(): void @@ -161,7 +161,7 @@ public function test_last_keeps_the_later_partial(): void $left->merge($right); - self::assertSame(20, $left->result($context->entryFactory())->value()); + static::assertSame(20, $left->result($context->entryFactory())->value()); } public function test_last_keeps_current_when_other_is_empty(): void @@ -173,7 +173,7 @@ public function test_last_keeps_current_when_other_is_empty(): void $left->merge(last(ref('v'))); - self::assertSame(10, $left->result($context->entryFactory())->value()); + static::assertSame(10, $left->result($context->entryFactory())->value()); } public function test_max_keeps_the_greatest(): void @@ -188,7 +188,7 @@ public function test_max_keeps_the_greatest(): void $left->merge($right); - self::assertSame(9, $left->result($context->entryFactory())->value()); + static::assertSame(9, $left->result($context->entryFactory())->value()); } public function test_max_takes_the_other_when_empty(): void @@ -202,7 +202,7 @@ public function test_max_takes_the_other_when_empty(): void $left->merge($right); - self::assertSame(9, $left->result($context->entryFactory())->value()); + static::assertSame(9, $left->result($context->entryFactory())->value()); } public function test_max_keeps_current_when_other_is_empty(): void @@ -214,7 +214,7 @@ public function test_max_keeps_current_when_other_is_empty(): void $left->merge(max(ref('v'))); - self::assertSame(5, $left->result($context->entryFactory())->value()); + static::assertSame(5, $left->result($context->entryFactory())->value()); } public function test_merging_a_different_function_type_throws(): void @@ -243,7 +243,7 @@ public function test_min_keeps_the_smallest(): void $left->merge($right); - self::assertSame(2, $left->result($context->entryFactory())->value()); + static::assertSame(2, $left->result($context->entryFactory())->value()); } public function test_min_takes_the_other_when_empty(): void @@ -257,7 +257,7 @@ public function test_min_takes_the_other_when_empty(): void $left->merge($right); - self::assertSame(2, $left->result($context->entryFactory())->value()); + static::assertSame(2, $left->result($context->entryFactory())->value()); } public function test_min_keeps_current_when_other_is_empty(): void @@ -269,7 +269,7 @@ public function test_min_keeps_current_when_other_is_empty(): void $left->merge(min(ref('v'))); - self::assertSame(5, $left->result($context->entryFactory())->value()); + static::assertSame(5, $left->result($context->entryFactory())->value()); } public function test_string_aggregate_merges_values(): void @@ -285,7 +285,7 @@ public function test_string_aggregate_merges_values(): void $left->merge($right); - self::assertSame('a,b,c', $left->result($context->entryFactory())->value()); + static::assertSame('a,b,c', $left->result($context->entryFactory())->value()); } public function test_string_aggregate_rejects_a_different_separator(): void @@ -308,6 +308,6 @@ public function test_sum_merges_partial_sums(): void $left->merge($right); - self::assertSame(6, $left->result($context->entryFactory())->value()); + static::assertSame(6, $left->result($context->entryFactory())->value()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php index d8aa49159f..5fe062c8cc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php @@ -26,7 +26,7 @@ public function test_orders_by_hash_then_bucket_index(): void $order[] = $group->hash . $group->bucketIndex; } - self::assertSame(['a1', 'a2', 'b0'], $order); + static::assertSame(['a1', 'a2', 'b0'], $order); } public function test_rejects_a_non_bucket_group(): void From 69df77e74b736907b9e451eb5f7190b9c1fe960d Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Tue, 7 Jul 2026 12:13:37 +0200 Subject: [PATCH 08/13] fix: --- .../BucketsCache/FilesystemGroupBucketsCache.php | 9 +++++---- .../src/Flow/ETL/GroupBy/GroupBucketsCache.php | 4 ++-- .../etl/src/Flow/ETL/GroupBy/HashAggregation.php | 14 +++++++------- .../etl/src/Flow/ETL/GroupBy/HashedGroup.php | 16 ++++++++++++++++ .../GroupBy/FilesystemGroupBucketsCacheTest.php | 11 ++++++----- 5 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/HashedGroup.php diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php index 892ba2e276..3e35c3897e 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php @@ -6,6 +6,7 @@ use Flow\ETL\GroupBy\GroupBucketsCache; use Flow\ETL\GroupBy\GroupState; +use Flow\ETL\GroupBy\HashedGroup; use Flow\ETL\Hash\NativePHPHash; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; @@ -33,7 +34,7 @@ public function __construct( } /** - * @return Generator + * @return Generator */ public function get(string $bucketId): Generator { @@ -55,7 +56,7 @@ public function get(string $bucketId): Generator $hash = substr($line, 0, $separator); $serialized = substr($line, $separator + 1); - yield [$hash, $this->serializer->unserialize($serialized, [GroupState::class])]; + yield new HashedGroup($hash, $this->serializer->unserialize($serialized, [GroupState::class])); } $stream->close(); @@ -73,8 +74,8 @@ public function set(string $bucketId, iterable $groups): void $buffer = ''; $counter = 0; - foreach ($groups as [$hash, $state]) { - $buffer .= $hash . ' ' . $this->serializer->serialize($state) . "\n"; + foreach ($groups as $group) { + $buffer .= $group->hash . ' ' . $this->serializer->serialize($group->state) . "\n"; $counter++; if ($counter >= $this->chunkSize) { diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php index 40c329a3c9..0db68173c1 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php @@ -12,14 +12,14 @@ interface GroupBucketsCache { /** - * @return Generator + * @return Generator */ public function get(string $bucketId): Generator; public function remove(string $bucketId): void; /** - * @param iterable $groups + * @param iterable $groups */ public function set(string $bucketId, iterable $groups): void; } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php index 06d265fade..1519c1799c 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php @@ -113,7 +113,7 @@ private function merge(array $buckets, array $hot): Generator { ksort($hot); - /** @var array> $cursors */ + /** @var array> $cursors */ $cursors = []; foreach ($buckets as $index => $bucketId) { @@ -152,7 +152,7 @@ private function merge(array $buckets, array $hot): Generator } /** - * @param Generator $cursor + * @param Generator $cursor */ private function advance(Generator $cursor, int $bucketIndex, GroupsMinHeap $heap): void { @@ -160,21 +160,21 @@ private function advance(Generator $cursor, int $bucketIndex, GroupsMinHeap $hea return; } - /** @var array{0: string, 1: GroupState} $item */ - $item = $cursor->current(); - $heap->insert(new BucketGroup($item[0], $bucketIndex, $item[1])); + /** @var HashedGroup $group */ + $group = $cursor->current(); + $heap->insert(new BucketGroup($group->hash, $bucketIndex, $group->state)); $cursor->next(); } /** * @param array $hot * - * @return Generator + * @return Generator */ private function hotCursor(array $hot): Generator { foreach ($hot as $hash => $state) { - yield [$hash, $state]; + yield new HashedGroup($hash, $state); } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/HashedGroup.php b/src/core/etl/src/Flow/ETL/GroupBy/HashedGroup.php new file mode 100644 index 0000000000..15830f06aa --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/HashedGroup.php @@ -0,0 +1,16 @@ +fs(), $this->serializer(), 2, $this->cacheDir); $groups = [ - ['aaa', new GroupState(['type' => 'a'], [sum(ref('v'))])], - ['bbb', new GroupState(['type' => 'b'], [sum(ref('v'))])], - ['ccc', new GroupState(['type' => 'c'], [sum(ref('v'))])], + new HashedGroup('aaa', new GroupState(['type' => 'a'], [sum(ref('v'))])), + new HashedGroup('bbb', new GroupState(['type' => 'b'], [sum(ref('v'))])), + new HashedGroup('ccc', new GroupState(['type' => 'c'], [sum(ref('v'))])), ]; $cache->set( @@ -40,8 +41,8 @@ public function test_round_trip_and_remove(): void $read = []; - foreach ($cache->get('bucket-1') as [$hash, $state]) { - $read[$hash] = $state->keyValues; + foreach ($cache->get('bucket-1') as $group) { + $read[$group->hash] = $group->state->keyValues; } static::assertSame(['aaa' => ['type' => 'a'], 'bbb' => ['type' => 'b'], 'ccc' => ['type' => 'c']], $read); From e5b8addd853777baf540efbda1846e3a0062eac1 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Wed, 8 Jul 2026 20:24:23 +0200 Subject: [PATCH 09/13] feat(flow-php/etl): memory-safe GroupBy with external aggregation and grouping Make GroupBy memory-safe for datasets and group counts larger than RAM, selectable via a Grouping backend (Memory by default, Filesystem opt-in), mirroring the Sort MemorySort/ExternalSort pattern. Aggregation on the Filesystem backend now uses external hash aggregation (ExternalBuckets): map-side combine into a bounded hot map, spill partial aggregate state as key-ordered runs, then a bounded-fan-in k-way merge folding equal-key partials via AggregatingFunction::merge. This replaces the previous one-file-per-group store and its super-linear cardinality cost. Add grouping without aggregation via GroupedDataFrame::getGroups(): sort by the group references (MemorySort/ExternalSort) and stream consecutive-key runs, emitting each group's rows in batches. Requires the group columns to be present. Rename the aggregation config to Grouping* (GroupingConfig, GroupingBackend, Config::$grouping, groupingFilesystem(), groupingMemoryLimit()). 100k FakeRandomOrders: Filesystem aggregation peak stays flat (~108MB) independent of cardinality; high-cardinality (~98k groups) completes in ~2s. --- src/core/etl/src/Flow/ETL/Config.php | 4 +- .../Config/Aggregation/AggregationConfig.php | 17 - .../Aggregation/AggregationConfigBuilder.php | 44 --- .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 38 +-- .../ETL/Config/Grouping/GroupingBackend.php | 11 + .../ETL/Config/Grouping/GroupingConfig.php | 22 ++ .../Config/Grouping/GroupingConfigBuilder.php | 78 +++++ .../Flow/ETL/DataFrame/GroupedDataFrame.php | 17 + src/core/etl/src/Flow/ETL/GroupBy.php | 17 + .../GroupBy/{GroupState.php => Bucket.php} | 9 +- .../Flow/ETL/GroupBy/BucketAggregation.php | 73 +++++ .../etl/src/Flow/ETL/GroupBy/BucketGroup.php | 17 - .../{HashedGroup.php => BucketRun.php} | 5 +- .../src/Flow/ETL/GroupBy/BucketRunCache.php | 25 ++ src/core/etl/src/Flow/ETL/GroupBy/Buckets.php | 20 ++ .../BucketsCache/FilesystemBucketRunCache.php | 135 ++++++++ .../FilesystemGroupBucketsCache.php | 99 ------ .../src/Flow/ETL/GroupBy/BucketsMinHeap.php | 44 +++ .../src/Flow/ETL/GroupBy/ExternalBuckets.php | 197 ++++++++++++ .../Flow/ETL/GroupBy/GroupBucketsCache.php | 25 -- .../etl/src/Flow/ETL/GroupBy/GroupRun.php | 23 ++ .../etl/src/Flow/ETL/GroupBy/GroupedRuns.php | 82 +++++ .../src/Flow/ETL/GroupBy/GroupsMinHeap.php | 36 --- .../src/Flow/ETL/GroupBy/HashAggregation.php | 194 ------------ .../src/Flow/ETL/GroupBy/InMemoryBuckets.php | 35 +++ .../Flow/ETL/Processor/GroupByProcessor.php | 111 ++++++- .../Flow/ETL/Processor/SortingProcessor.php | 13 +- .../src/Flow/ETL/Sort/ExternalSortFactory.php | 42 +++ .../Integration/DataFrame/GetGroupsTest.php | 292 ++++++++++++++++++ .../DataFrame/GroupByAggregationTest.php | 283 ++++++++++++----- .../DataFrame/GroupByScaleTest.php | 59 ++++ .../GroupBy/ExternalBucketsTest.php | 199 ++++++++++++ .../GroupBy/FilesystemBucketRunCacheTest.php | 109 +++++++ .../FilesystemGroupBucketsCacheTest.php | 54 ---- .../AggregationConfigBuilderTest.php | 49 --- .../Grouping/GroupingConfigBuilderTest.php | 43 +++ .../Unit/GroupBy/BucketAggregationTest.php | 113 +++++++ .../ETL/Tests/Unit/GroupBy/BucketTest.php | 43 +++ .../Tests/Unit/GroupBy/BucketsMinHeapTest.php | 38 +++ .../Tests/Unit/GroupBy/GroupedRunsTest.php | 47 +++ .../Tests/Unit/GroupBy/GroupsMinHeapTest.php | 38 --- .../Unit/GroupBy/InMemoryBucketsTest.php | 51 +++ .../tests/Flow/ETL/Tests/Unit/GroupByTest.php | 8 +- .../Unit/Processor/GroupByProcessorTest.php | 36 ++- 44 files changed, 2179 insertions(+), 716 deletions(-) delete mode 100644 src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfigBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/GroupingBackend.php create mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php rename src/core/etl/src/Flow/ETL/GroupBy/{GroupState.php => Bucket.php} (61%) create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketGroup.php rename src/core/etl/src/Flow/ETL/GroupBy/{HashedGroup.php => BucketRun.php} (63%) create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketRunCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Buckets.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php create mode 100644 src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php diff --git a/src/core/etl/src/Flow/ETL/Config.php b/src/core/etl/src/Flow/ETL/Config.php index 4cf96bd43b..a8cf5937bb 100644 --- a/src/core/etl/src/Flow/ETL/Config.php +++ b/src/core/etl/src/Flow/ETL/Config.php @@ -4,9 +4,9 @@ namespace Flow\ETL; -use Flow\ETL\Config\Aggregation\AggregationConfig; use Flow\ETL\Config\Cache\CacheConfig; use Flow\ETL\Config\ConfigBuilder; +use Flow\ETL\Config\Grouping\GroupingConfig; use Flow\ETL\Config\Sort\SortConfig; use Flow\ETL\Config\Telemetry\TelemetryConfig; use Flow\ETL\Filesystem\FilesystemStreams; @@ -37,7 +37,7 @@ public function __construct( public SortConfig $sort, private ?Analyze $analyze, public TelemetryConfig $telemetry, - public AggregationConfig $aggregation, + public GroupingConfig $grouping, ) {} public static function builder(): ConfigBuilder diff --git a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php b/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php deleted file mode 100644 index cfcb9bd5ea..0000000000 --- a/src/core/etl/src/Flow/ETL/Config/Aggregation/AggregationConfig.php +++ /dev/null @@ -1,17 +0,0 @@ -memoryLimit === null) { - $envLimit = getenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); - - $this->memoryLimit = is_string($envLimit) ? Unit::fromString($envLimit) : Unit::fromBytes(PHP_INT_MAX); - } - - return new AggregationConfig($this->memoryLimit, $this->filesystemProtocol); - } - - public function filesystemProtocol(string $protocol): self - { - $this->filesystemProtocol = $protocol; - - return $this; - } - - public function memoryLimit(Unit $memoryLimit): self - { - $this->memoryLimit = $memoryLimit; - - return $this; - } -} diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index 609add4da6..e71698da24 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -8,8 +8,8 @@ use Flow\ETL\Analyze; use Flow\ETL\Cache; use Flow\ETL\Config; -use Flow\ETL\Config\Aggregation\AggregationConfigBuilder; use Flow\ETL\Config\Cache\CacheConfigBuilder; +use Flow\ETL\Config\Grouping\GroupingConfigBuilder; use Flow\ETL\Config\Sort\SortConfigBuilder; use Flow\ETL\Config\Telemetry\TelemetryConfig; use Flow\ETL\Config\Telemetry\TelemetryOptions; @@ -35,10 +35,10 @@ final class ConfigBuilder { - public readonly AggregationConfigBuilder $aggregation; - public readonly CacheConfigBuilder $cache; + public readonly GroupingConfigBuilder $grouping; + public readonly SortConfigBuilder $sort; private ?Analyze $analyze; @@ -72,8 +72,8 @@ public function __construct() $this->putInputIntoRows = false; $this->optimizer = null; $this->clock = null; - $this->aggregation = new AggregationConfigBuilder(); $this->cache = new CacheConfigBuilder(); + $this->grouping = new GroupingConfigBuilder(); $this->sort = new SortConfigBuilder(); $this->randomValueGenerator = new NativePHPRandomValueGenerator(); $this->analyze = null; @@ -83,20 +83,6 @@ public function __construct() : PackageVersion::get('flow-php/etl'); } - public function aggregationFilesystem(string $protocol): self - { - $this->aggregation->filesystemProtocol($protocol); - - return $this; - } - - public function aggregationMemoryLimit(Unit $unit): self - { - $this->aggregation->memoryLimit($unit); - - return $this; - } - public function analyze(Analyze $analyze): self { $this->analyze = $analyze; @@ -129,7 +115,7 @@ public function build(EntryFactory $entryFactory = new EntryFactory()): Config $this->sort->build(), $this->analyze, $this->telemetryConfig ?? TelemetryConfig::default($this->getClock()), - $this->aggregation->build(), + $this->grouping->build(), ); } @@ -178,6 +164,20 @@ public function externalSortFilesystem(string $protocol): self return $this; } + public function groupingFilesystem(string $protocol = 'file'): self + { + $this->grouping->filesystemProtocol($protocol); + + return $this; + } + + public function groupingMemoryLimit(Unit $unit): self + { + $this->grouping->memoryLimit($unit); + + return $this; + } + public function id(string $id): self { $this->id = $id; diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingBackend.php b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingBackend.php new file mode 100644 index 0000000000..01cd5c1561 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingBackend.php @@ -0,0 +1,11 @@ + $bucketsCount + */ + public function __construct( + public GroupingBackend $backend, + public Unit $memoryLimit, + public int $bucketsCount = 10, + public string $filesystemProtocol = 'file', + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php new file mode 100644 index 0000000000..2dbadd615d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php @@ -0,0 +1,78 @@ + */ + private int $bucketsCount = 10; + + private string $filesystemProtocol = 'file'; + + private ?Unit $memoryLimit = null; + + public function backend(GroupingBackend $backend): self + { + $this->backend = $backend; + + return $this; + } + + /** + * @param int<1, max> $bucketsCount + */ + public function bucketsCount(int $bucketsCount): self + { + $this->bucketsCount = $bucketsCount; + + return $this; + } + + public function build(): GroupingConfig + { + if ($this->memoryLimit === null) { + $env = getenv(GroupingConfig::GROUPING_MAX_MEMORY_ENV); + + if ($env !== false) { + $this->memoryLimit = Unit::fromString($env); + } else { + $iniLimit = ini_get('memory_limit'); + + $this->memoryLimit = + $iniLimit === false || $iniLimit === '-1' + ? Unit::fromBytes(PHP_INT_MAX) + : Unit::fromString($iniLimit)->percentage(self::DEFAULT_GROUPING_MEMORY_PERCENTAGE); + } + } + + return new GroupingConfig($this->backend, $this->memoryLimit, $this->bucketsCount, $this->filesystemProtocol); + } + + public function filesystemProtocol(string $protocol): self + { + $this->backend = GroupingBackend::Filesystem; + $this->filesystemProtocol = $protocol; + + return $this; + } + + public function memoryLimit(Unit $memoryLimit): self + { + $this->memoryLimit = $memoryLimit; + + return $this; + } +} diff --git a/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php b/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php index f4d2f48506..6297b7c741 100644 --- a/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php +++ b/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php @@ -32,6 +32,23 @@ public function aggregate(AggregatingFunction ...$aggregations): DataFrame return $this->df; } + public function getGroups(?int $size = null): DataFrame + { + if ($size !== null) { + $this->groupBy->batchSize($size); + } + + $pipelineAdder = function (GroupBy $groupBy): void { + // @mago-ignore analysis:non-existent-property,method-access-on-null + $this->pipeline->add(new GroupByProcessor($groupBy)); + }; + + // @mago-ignore analysis:invalid-method-access + $pipelineAdder->bindTo($this->df, $this->df)($this->groupBy); + + return $this->df; + } + public function pivot(Reference $ref): self { $this->groupBy->pivot($ref); diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 264fbb886e..c0172e5809 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -41,6 +41,8 @@ final class GroupBy */ private array $aggregations = []; + private int $batchSize = self::RESULT_BATCH_SIZE; + private ?Reference $pivot = null; private readonly References $refs; @@ -93,6 +95,21 @@ public function aggregations(): array return $this->aggregations; } + public function batchSize(int $size): void + { + $this->batchSize = $size; + } + + public function getBatchSize(): int + { + return $this->batchSize; + } + + public function hasAggregations(): bool + { + return $this->aggregations !== []; + } + /** * @param array $values */ diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupState.php b/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php similarity index 61% rename from src/core/etl/src/Flow/ETL/GroupBy/GroupState.php rename to src/core/etl/src/Flow/ETL/GroupBy/Bucket.php index ded39a6a25..df4618d437 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupState.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php @@ -9,7 +9,7 @@ /** * @internal */ -final readonly class GroupState +final readonly class Bucket { /** * @param array $keyValues @@ -19,4 +19,11 @@ public function __construct( public array $keyValues, public array $aggregators, ) {} + + public function merge(self $other): void + { + foreach ($this->aggregators as $index => $aggregator) { + $aggregator->merge($other->aggregators[$index]); + } + } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php new file mode 100644 index 0000000000..493caf4787 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php @@ -0,0 +1,73 @@ + $rows + * + * @return Generator + */ + public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator + { + try { + foreach ($rows as $batch) { + /** @var array $local */ + $local = []; + + foreach ($batch as $row) { + $keyValues = $groupBy->keyValues($row); + $hash = $groupBy->hash($keyValues); + + if (!isset($local[$hash])) { + $local[$hash] = new Bucket($keyValues, $groupBy->newAggregators()); + } + + foreach ($local[$hash]->aggregators as $aggregator) { + $aggregator->aggregate($row, $context); + } + } + + foreach ($local as $hash => $bucket) { + $this->buckets->merge($hash, $bucket); + } + } + + $buffer = []; + + foreach ($this->buckets->all() as $bucket) { + $buffer[] = $groupBy->aggregatedRow($bucket->keyValues, $bucket->aggregators, $context->entryFactory()); + + if (count($buffer) >= self::RESULT_BATCH_SIZE) { + yield new Rows(...$buffer); + $buffer = []; + } + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + } + } finally { + $this->buckets->clear(); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketGroup.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketGroup.php deleted file mode 100644 index 1737d13297..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketGroup.php +++ /dev/null @@ -1,17 +0,0 @@ - yields hash => Bucket in ascending-hash order + */ + public function get(string $runId): Generator; + + public function remove(string $runId): void; + + /** + * @param iterable $buckets hash => Bucket, ascending-hash order + */ + public function set(string $runId, iterable $buckets): void; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php b/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php new file mode 100644 index 0000000000..97e95b5836 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php @@ -0,0 +1,20 @@ + + */ + public function all(): iterable; + + public function clear(): void; + + public function merge(string $hash, Bucket $bucket): void; +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php new file mode 100644 index 0000000000..b36d8a4b06 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php @@ -0,0 +1,135 @@ + $chunkSize + */ + public function __construct( + private Filesystem $filesystem, + private Serializer $serializer = new NativePHPSerializer(), + private int $chunkSize = self::DEFAULT_CHUNK_SIZE, + ?Path $cacheDir = null, + ) { + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->chunkSize < 1) { + throw new InvalidArgumentException('Chunk size must be greater than 0'); + } + + $this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); + } + + /** + * @return Generator + */ + public function get(string $runId): Generator + { + $path = $this->keyPath($runId); + + if (!$this->filesystem->status($path)) { + return; + } + + $stream = $this->filesystem->readFrom($path); + + try { + foreach ($stream->readLines() as $line) { + if ($line === '') { + continue; + } + + $parts = explode(' ', $line, 2); + + if (count($parts) !== 2) { + throw new RuntimeException(sprintf( + 'Corrupted bucket-cache line for run "%s": missing hash/bucket separator', + $runId, + )); + } + + [$hashB64, $bucketB64] = $parts; + + $hash = base64_decode($hashB64, true); + $serializedBucket = base64_decode($bucketB64, true); + + if ($hash === false || $serializedBucket === false) { + throw new RuntimeException(sprintf( + 'Failed to decode base64 bucket-cache line for run "%s"', + $runId, + )); + } + + $bucket = $this->serializer->unserialize($serializedBucket, [Bucket::class]); + + yield $hash => $bucket; + } + } finally { + $stream->close(); + } + } + + public function remove(string $runId): void + { + $this->filesystem->rm($this->keyPath($runId)->parentDirectory()); + } + + public function set(string $runId, iterable $buckets): void + { + $stream = $this->filesystem->writeTo($this->keyPath($runId)); + + $buffer = ''; + $counter = 0; + + foreach ($buckets as $hash => $bucket) { + // @mago-ignore analysis:redundant-cast - numeric-string hashes are coerced to int array keys by PHP + $buffer .= + base64_encode((string) $hash) . ' ' . base64_encode($this->serializer->serialize($bucket)) . "\n"; + $counter++; + + if ($counter >= $this->chunkSize) { + $stream->append($buffer); + $buffer = ''; + $counter = 0; + } + } + + if ($counter > 0) { + $stream->append($buffer); + } + + $stream->close(); + } + + private function keyPath(string $key): Path + { + return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php deleted file mode 100644 index 3e35c3897e..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemGroupBucketsCache.php +++ /dev/null @@ -1,99 +0,0 @@ - $chunkSize - */ - public function __construct( - private Filesystem $filesystem, - private Serializer $serializer = new NativePHPSerializer(), - private int $chunkSize = 100, - ?Path $cacheDir = null, - ) { - $this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); - } - - /** - * @return Generator - */ - public function get(string $bucketId): Generator - { - $path = $this->keyPath($bucketId); - - if (!$this->filesystem->status($path)) { - return; - } - - $stream = $this->filesystem->readFrom($path); - - foreach ($stream->readLines() as $line) { - $separator = strpos($line, ' '); - - if ($separator === false) { - continue; - } - - $hash = substr($line, 0, $separator); - $serialized = substr($line, $separator + 1); - - yield new HashedGroup($hash, $this->serializer->unserialize($serialized, [GroupState::class])); - } - - $stream->close(); - } - - public function remove(string $bucketId): void - { - $this->filesystem->rm($this->keyPath($bucketId)->parentDirectory()); - } - - public function set(string $bucketId, iterable $groups): void - { - $stream = $this->filesystem->writeTo($this->keyPath($bucketId)); - - $buffer = ''; - $counter = 0; - - foreach ($groups as $group) { - $buffer .= $group->hash . ' ' . $this->serializer->serialize($group->state) . "\n"; - $counter++; - - if ($counter >= $this->chunkSize) { - $stream->append($buffer); - $buffer = ''; - $counter = 0; - } - } - - if ($counter > 0) { - $stream->append($buffer); - } - - $stream->close(); - } - - private function keyPath(string $key): Path - { - return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php new file mode 100644 index 0000000000..c016340425 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php @@ -0,0 +1,44 @@ + + */ +final class BucketsMinHeap extends SplMinHeap +{ + /** + * @return BucketRun + */ + public function extract(): mixed + { + return parent::extract(); + } + + public function insert(mixed $value): true + { + if (!$value instanceof BucketRun) { + throw new InvalidArgumentException('Value inserted into BucketsMinHeap must be a ' . BucketRun::class); + } + + parent::insert($value); + + return true; + } + + /** + * @param BucketRun $value1 + * @param BucketRun $value2 + */ + protected function compare($value1, $value2): int + { + return [$value2->hash, $value2->runIndex] <=> [$value1->hash, $value1->runIndex]; + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php new file mode 100644 index 0000000000..fe47ad6b1b --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php @@ -0,0 +1,197 @@ + + */ + private array $hot = []; + + /** + * @var list + */ + private array $runIds = []; + + /** + * @param int<1, max> $bucketsCount + */ + public function __construct( + private readonly BucketRunCache $cache, + private readonly Unit $memoryLimit, + private readonly int $bucketsCount = 10, + ) { + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->bucketsCount < 1) { + throw new InvalidArgumentException('Buckets count must be greater than 0, given: ' . $this->bucketsCount); + } + + $this->consumption = new Consumption(false); + } + + public function all(): iterable + { + if ($this->runIds === []) { + ksort($this->hot); + + yield from $this->hot; + + return; + } + + if ($this->hot !== []) { + $this->spillRun(); + } + + // A fan-in of 1 would chunk runs into size-1 groups that never shrink the run count, so the + // recursive merge could loop forever; clamp to at least 2 to guarantee progress. + $fanIn = max(2, $this->bucketsCount); + + // $this->runIds is kept as the LIVE set of on-disk runs at every point below: each level + // creates all of its merged runs first, only then removes the consumed inputs, and only + // then repoints $this->runIds at the new level. This loop body is fully synchronous (no + // yield inside it), so it can never be interrupted mid-cascade by an abandoned consumer — + // if the caller abandons `all()` it can only happen in the final merge/yield loop below, + // at which point $this->runIds already reflects exactly the runs still on disk, so + // clear() can safely remove them. + while (count($this->runIds) > $this->bucketsCount) { + $consumedRunIds = $this->runIds; + $newRunIds = []; + + foreach (array_chunk($consumedRunIds, $fanIn) as $chunk) { + $newRunId = bin2hex(random_bytes(16)); + $this->cache->set($newRunId, $this->mergeCursors($chunk)); + + $newRunIds[] = $newRunId; + } + + foreach ($consumedRunIds as $id) { + $this->cache->remove($id); + } + + $this->runIds = $newRunIds; + } + + foreach ($this->mergeCursors($this->runIds) as $bucket) { + yield $bucket; + } + + foreach ($this->runIds as $id) { + $this->cache->remove($id); + } + + $this->runIds = []; + $this->hot = []; + } + + public function clear(): void + { + foreach ($this->runIds as $id) { + $this->cache->remove($id); + } + + $this->hot = []; + $this->runIds = []; + } + + public function merge(string $hash, Bucket $bucket): void + { + if (isset($this->hot[$hash])) { + $this->hot[$hash]->merge($bucket); + } else { + $this->hot[$hash] = $bucket; + } + + if ($this->consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { + $this->spillRun(); + } + } + + private function advance(Generator $cursor, int $index, BucketsMinHeap $heap): void + { + if ($cursor->valid()) { + /** @var string $key */ + $key = $cursor->key(); + /** @var Bucket $bucket */ + $bucket = $cursor->current(); + + $heap->insert(new BucketRun($key, $index, $bucket)); + $cursor->next(); + } + } + + /** + * K-way merge over run cursors, folding equal-hash Buckets via Bucket::merge, yielding merged + * Buckets in ascending hash. Keyed by hash so the result feeds both BucketRunCache::set() + * (which needs hash => Bucket) and value-only consumers. + * + * @param list $runIds + * + * @return Generator + */ + private function mergeCursors(array $runIds): Generator + { + /** @var array> $cursors */ + $cursors = []; + + foreach ($runIds as $i => $runId) { + $cursors[$i] = $this->cache->get($runId); + } + + $heap = new BucketsMinHeap(); + + foreach ($cursors as $i => $cursor) { + $this->advance($cursor, $i, $heap); + } + + while (!$heap->isEmpty()) { + $top = $heap->extract(); + $merged = $top->bucket; + $this->advance($cursors[$top->runIndex], $top->runIndex, $heap); + + while (!$heap->isEmpty() && $heap->top()->hash === $top->hash) { + $nextRun = $heap->extract(); + $merged->merge($nextRun->bucket); + $this->advance($cursors[$nextRun->runIndex], $nextRun->runIndex, $heap); + } + + yield $top->hash => $merged; + } + } + + private function spillRun(): void + { + ksort($this->hot); + + $runId = bin2hex(random_bytes(16)); + $this->cache->set($runId, $this->hot); + $this->runIds[] = $runId; + $this->hot = []; + $this->consumption = new Consumption(false); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php deleted file mode 100644 index 0db68173c1..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupBucketsCache.php +++ /dev/null @@ -1,25 +0,0 @@ - - */ - public function get(string $bucketId): Generator; - - public function remove(string $bucketId): void; - - /** - * @param iterable $groups - */ - public function set(string $bucketId, iterable $groups): void; -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php new file mode 100644 index 0000000000..bad835a474 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php @@ -0,0 +1,23 @@ + $keyValues + * @param Generator $rows + */ + public function __construct( + public array $keyValues, + public Generator $rows, + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php new file mode 100644 index 0000000000..febb52e91f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php @@ -0,0 +1,82 @@ + $sortedRows key-ordered stream of rows + * + * @return Generator + */ + public function groups(Generator $sortedRows, GroupBy $groupBy): Generator + { + $rows = $this->flatten($sortedRows); + + while ($rows->valid()) { + $keyValues = $groupBy->keyValues($rows->current()); + $hash = $groupBy->hash($keyValues); + + yield new GroupRun($keyValues, $this->groupRows($rows, $groupBy, $hash)); + } + } + + /** + * @param Generator<\Flow\ETL\Rows> $sortedRows + * + * @return Generator + */ + private function flatten(Generator $sortedRows): Generator + { + foreach ($sortedRows as $batch) { + foreach ($batch as $row) { + yield $row; + } + } + } + + /** + * @param Generator $rows + * + * @return Generator + */ + private function groupRows(Generator $rows, GroupBy $groupBy, string $groupHash): Generator + { + while ($rows->valid()) { + /** @var Row $row */ + $row = $rows->current(); + + if ($groupBy->hash($groupBy->keyValues($row)) !== $groupHash) { + return; + } + + yield $row; + $rows->next(); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php deleted file mode 100644 index 8bd8180101..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupsMinHeap.php +++ /dev/null @@ -1,36 +0,0 @@ - - */ -final class GroupsMinHeap extends SplMinHeap -{ - public function insert(mixed $value): true - { - if (!$value instanceof BucketGroup) { - throw new InvalidArgumentException('Value inserted into GroupsMinHeap must be a ' . BucketGroup::class); - } - - parent::insert($value); - - return true; - } - - /** - * @param BucketGroup $value1 - * @param BucketGroup $value2 - */ - protected function compare($value1, $value2): int - { - return [$value2->hash, $value2->bucketIndex] <=> [$value1->hash, $value1->bucketIndex]; - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php deleted file mode 100644 index 1519c1799c..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/HashAggregation.php +++ /dev/null @@ -1,194 +0,0 @@ - $rows - * - * @return Generator - */ - public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator - { - $consumption = new Consumption(false); - /** @var array $hot */ - $hot = []; - /** @var list $buckets */ - $buckets = []; - - foreach ($rows as $batch) { - foreach ($batch as $row) { - $keyValues = $groupBy->keyValues($row); - $hash = $groupBy->hash($keyValues); - - if (!array_key_exists($hash, $hot)) { - $hot[$hash] = new GroupState($keyValues, $groupBy->newAggregators()); - } - - foreach ($hot[$hash]->aggregators as $aggregator) { - $aggregator->aggregate($row, $context); - } - } - - if ($consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { - $buckets[] = $this->spill($hot); - $hot = []; - $consumption = new Consumption(false); - } - } - - $groups = $buckets === [] ? $this->iterate($hot) : $this->merge($buckets, $hot); - - yield from $this->emit($groups, $groupBy, $context); - } - - /** - * @param Generator $groups - * - * @return Generator - */ - private function emit(Generator $groups, GroupBy $groupBy, FlowContext $context): Generator - { - $buffer = []; - - foreach ($groups as $state) { - $buffer[] = $groupBy->aggregatedRow($state->keyValues, $state->aggregators, $context->entryFactory()); - - if (count($buffer) >= self::BATCH_SIZE) { - yield new Rows(...$buffer); - $buffer = []; - } - } - - if ($buffer !== []) { - yield new Rows(...$buffer); - } - } - - /** - * @param array $hot - * - * @return Generator - */ - private function iterate(array $hot): Generator - { - foreach ($hot as $state) { - yield $state; - } - } - - /** - * @param list $buckets - * @param array $hot - * - * @return Generator - */ - private function merge(array $buckets, array $hot): Generator - { - ksort($hot); - - /** @var array> $cursors */ - $cursors = []; - - foreach ($buckets as $index => $bucketId) { - $cursors[$index] = $this->cache->get($bucketId); - } - - $cursors[count($buckets)] = $this->hotCursor($hot); - - $heap = new GroupsMinHeap(); - - foreach ($cursors as $index => $cursor) { - $this->advance($cursor, $index, $heap); - } - - while (!$heap->isEmpty()) { - $current = $heap->extract(); - $merged = $current->state; - $this->advance($cursors[$current->bucketIndex], $current->bucketIndex, $heap); - - while (!$heap->isEmpty() && $heap->top()->hash === $current->hash) { - $next = $heap->extract(); - - foreach ($merged->aggregators as $index => $aggregator) { - $aggregator->merge($next->state->aggregators[$index]); - } - - $this->advance($cursors[$next->bucketIndex], $next->bucketIndex, $heap); - } - - yield $merged; - } - - foreach ($buckets as $bucketId) { - $this->cache->remove($bucketId); - } - } - - /** - * @param Generator $cursor - */ - private function advance(Generator $cursor, int $bucketIndex, GroupsMinHeap $heap): void - { - if (!$cursor->valid()) { - return; - } - - /** @var HashedGroup $group */ - $group = $cursor->current(); - $heap->insert(new BucketGroup($group->hash, $bucketIndex, $group->state)); - $cursor->next(); - } - - /** - * @param array $hot - * - * @return Generator - */ - private function hotCursor(array $hot): Generator - { - foreach ($hot as $hash => $state) { - yield new HashedGroup($hash, $state); - } - } - - /** - * @param array $hot - */ - private function spill(array $hot): string - { - ksort($hot); - - $bucketId = bin2hex(random_bytes(16)); - - $this->cache->set($bucketId, $this->hotCursor($hot)); - - return $bucketId; - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php new file mode 100644 index 0000000000..96ec8618a0 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php @@ -0,0 +1,35 @@ + + */ + private array $buckets = []; + + public function all(): iterable + { + yield from $this->buckets; + } + + public function clear(): void + { + $this->buckets = []; + } + + public function merge(string $hash, Bucket $bucket): void + { + if (isset($this->buckets[$hash])) { + $this->buckets[$hash]->merge($bucket); + } else { + $this->buckets[$hash] = $bucket; + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index ccc4eb59e5..7c256608b2 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -4,13 +4,27 @@ namespace Flow\ETL\Processor; +use Flow\ETL\Config\Grouping\GroupingBackend; use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; -use Flow\ETL\GroupBy\BucketsCache\FilesystemGroupBucketsCache; -use Flow\ETL\GroupBy\HashAggregation; +use Flow\ETL\GroupBy\BucketAggregation; +use Flow\ETL\GroupBy\BucketsCache\FilesystemBucketRunCache; +use Flow\ETL\GroupBy\ExternalBuckets; +use Flow\ETL\GroupBy\GroupedRuns; +use Flow\ETL\GroupBy\InMemoryBuckets; use Flow\ETL\Processor; +use Flow\ETL\Row\EntryReference; +use Flow\ETL\Row\Reference; +use Flow\ETL\Row\References; +use Flow\ETL\Rows; +use Flow\ETL\Sort\ExternalSortFactory; +use Flow\ETL\Sort\MemorySort; use Generator; +use function array_map; +use function bin2hex; +use function random_bytes; + /** * Groups all rows and applies aggregation functions. * @@ -30,18 +44,93 @@ public function process(Generator $rows, FlowContext $context): Generator return; } - $config = $context->config->aggregation; + if (!$this->groupBy->hasAggregations()) { + yield from $this->group($rows, $context); + + return; + } + + $config = $context->config->grouping; + + $buckets = match ($config->backend) { + GroupingBackend::Filesystem => new ExternalBuckets( + new FilesystemBucketRunCache( + $context->filesystem($config->filesystemProtocol), + $context->config->serializer(), + cacheDir: $context + ->config + ->cache + ->localFilesystemCacheDir + ->suffix('/flow-php-group-by/' . bin2hex(random_bytes(16)) . '/'), + ), + $config->memoryLimit, + $config->bucketsCount, + ), + GroupingBackend::Memory => new InMemoryBuckets(), + }; - $aggregation = new HashAggregation( - $config->memoryLimit, - new FilesystemGroupBucketsCache( - $context->filesystem($config->filesystemProtocol), + yield from (new BucketAggregation($buckets))->aggregate($rows, $context, $this->groupBy); + } + + /** + * Grouping without aggregation: sort the stream by the group references, then stream the + * consecutive-equal-key runs and re-chunk each run's rows into batches. A new group always + * starts a new batch, so every emitted {@see Rows} belongs to exactly one group. + * + * Note: `groupingMemoryLimit` does NOT bound the Filesystem no-agg spill - {@see ExternalSort} + * spills by row batches (its own `minBatchSize`/`bucketsCount`), not by byte limit. On the + * Memory backend it still caps the in-memory {@see MemorySort} (its OOM threshold). + * + * @param Generator $rows + * + * @return Generator + */ + private function group(Generator $rows, FlowContext $context): Generator + { + $refs = References::init(...array_map(static fn(Reference $ref): Reference => $ref instanceof EntryReference + ? $ref->asc() + : $ref, $this->groupBy->references()->all())); + + /** @var Generator $sorted */ + $sorted = match ($context->config->grouping->backend) { + GroupingBackend::Filesystem => ExternalSortFactory::create( + $context->filesystem($context->config->grouping->filesystemProtocol), $context->config->serializer(), - 100, - $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-group-by/'), + $context + ->config + ->cache + ->localFilesystemCacheDir + ->suffix('/flow-php-group-by/' . bin2hex(random_bytes(16)) . '/'), + $context->config->grouping->bucketsCount, + )->sortGenerator($rows, $context, $refs), + GroupingBackend::Memory => (new MemorySort($context->config->grouping->memoryLimit))->sortGenerator( + $rows, + $context, + $refs, ), - ); + }; + + $batchSize = $this->groupBy->getBatchSize(); + $buffer = []; + $counter = 0; - yield from $aggregation->aggregate($rows, $context, $this->groupBy); + foreach ((new GroupedRuns())->groups($sorted, $this->groupBy) as $groupRun) { + foreach ($groupRun->rows as $row) { + $buffer[] = $row; + $counter++; + + if ($counter >= $batchSize) { + yield new Rows(...$buffer); + $buffer = []; + $counter = 0; + } + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + $buffer = []; + $counter = 0; + } + } } } diff --git a/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php index d8d34a0877..8593831c3c 100644 --- a/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/SortingProcessor.php @@ -9,8 +9,7 @@ use Flow\ETL\Processor; use Flow\ETL\Row\References; use Flow\ETL\Rows; -use Flow\ETL\Sort\ExternalSort; -use Flow\ETL\Sort\ExternalSort\BucketsCache\FilesystemBucketsCache; +use Flow\ETL\Sort\ExternalSortFactory; use Flow\ETL\Sort\MemorySort; use Generator; @@ -52,14 +51,6 @@ public function process(Generator $rows, FlowContext $context): Generator */ private function externalSort(Generator $rows, FlowContext $context): Generator { - return (new ExternalSort( - new FilesystemBucketsCache( - $context->filesystem($context->config->sort->filesystemProtocol), - $context->config->serializer(), - 100, - $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-external-sort/'), - ), - $context->config->cache->externalSortBucketsCount, - ))->sortGenerator($rows, $context, $this->refs); + return ExternalSortFactory::fromContext($context)->sortGenerator($rows, $context, $this->refs); } } diff --git a/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php b/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php new file mode 100644 index 0000000000..b3e48d3423 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php @@ -0,0 +1,42 @@ + $bucketsCount + */ + public static function create( + Filesystem $filesystem, + Serializer $serializer, + Path $cacheDir, + int $bucketsCount, + ): ExternalSort { + return new ExternalSort( + new FilesystemBucketsCache($filesystem, $serializer, cacheDir: $cacheDir), + $bucketsCount, + ); + } + + public static function fromContext(FlowContext $context): ExternalSort + { + return self::create( + $context->filesystem($context->config->sort->filesystemProtocol), + $context->config->serializer(), + $context->config->cache->localFilesystemCacheDir->suffix('/flow-php-external-sort/'), + $context->config->cache->externalSortBucketsCount, + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php new file mode 100644 index 0000000000..8869c8cf13 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php @@ -0,0 +1,292 @@ +expectException(InvalidArgumentException::class); + + iterator_to_array( + data_frame() + ->read(from_array([['type' => 'a'], ['not_type' => 'b'], ['type' => 'c']])) + ->groupBy(ref('type')) + ->getGroups() + ->get(), + ); + } + + public function test_get_groups_memory_and_filesystem_emit_the_same_groups(): void + { + $input = [ + ['k' => 'a', 'v' => 1], + ['k' => 'b', 'v' => 2], + ['k' => 'a', 'v' => 3], + ['k' => 'c', 'v' => 4], + ['k' => 'b', 'v' => 5], + ['k' => 'a', 'v' => 6], + ]; + + $run = static function (ConfigBuilder $config) use ($input): array { + $groups = []; + + foreach (data_frame($config)->read(from_array($input))->groupBy(ref('k'))->getGroups()->get() as $rows) { + // every emitted batch belongs to exactly one group + $keys = []; + + foreach ($rows as $row) { + $value = $row->valueOf(ref('k')); + + if (is_string($value)) { + $keys[$value] = true; + } + } + + static::assertCount(1, $keys); + + $key = array_key_first($keys); + + if ($key === null) { + continue; + } + + $groups[$key] = ($groups[$key] ?? 0) + $rows->count(); + } + + ksort($groups); + + return $groups; + }; + + static::assertSame(['a' => 3, 'b' => 2, 'c' => 1], $run(config_builder())); + static::assertSame(['a' => 3, 'b' => 2, 'c' => 1], $run(config_builder()->groupingFilesystem('file'))); + } + + public function test_get_groups_splits_a_large_group_across_batches(): void + { + $input = [ + ['k' => 'a', 'v' => 1], + ['k' => 'a', 'v' => 2], + ['k' => 'a', 'v' => 3], + ['k' => 'a', 'v' => 4], + ['k' => 'a', 'v' => 5], + ['k' => 'b', 'v' => 6], + ]; + + $run = static function (ConfigBuilder $config) use ($input): array { + $batchSizes = []; + + foreach (data_frame($config)->read(from_array($input))->groupBy(ref('k'))->getGroups(2)->get() as $rows) { + // every emitted batch belongs to exactly one group + $keys = []; + + foreach ($rows as $row) { + $value = $row->valueOf(ref('k')); + + if (is_string($value)) { + $keys[$value] = true; + } + } + + static::assertCount(1, $keys); + static::assertLessThanOrEqual(2, $rows->count()); + + $key = array_key_first($keys); + + if ($key === null) { + continue; + } + + $batchSizes[$key][] = $rows->count(); + } + + return $batchSizes; + }; + + $batchSizes = $run(config_builder()); + + static::assertSame(['a' => [2, 2, 1], 'b' => [1]], $batchSizes); + + static::assertSame(['a' => [2, 2, 1], 'b' => [1]], $run(config_builder()->groupingFilesystem('file'))); + } + + public function test_get_groups_multi_column_key(): void + { + // ('x','yz') and ('xy','z') would collide under a naive concatenation-based hash (both + // produce "xyz"); GroupBy::hash() length-prefixes each value to tell them apart. + $input = [ + ['a' => 'x', 'b' => 'yz', 'v' => 1], + ['a' => 'xy', 'b' => 'z', 'v' => 2], + ['a' => 'x', 'b' => 'yz', 'v' => 3], + ]; + + $run = static function (ConfigBuilder $config) use ($input): array { + $groups = []; + + foreach (data_frame($config) + ->read(from_array($input)) + ->groupBy(ref('a'), ref('b')) + ->getGroups() + ->get() as $rows) { + $keys = []; + + foreach ($rows as $row) { + $a = $row->valueOf(ref('a')); + $b = $row->valueOf(ref('b')); + + if (is_string($a) && is_string($b)) { + $keys[$a . '|' . $b] = true; + } + } + + static::assertCount(1, $keys); + + $key = array_key_first($keys); + + if ($key === null) { + continue; + } + + $groups[$key] = ($groups[$key] ?? 0) + $rows->count(); + } + + ksort($groups); + + return $groups; + }; + + // ksort() on the runner's $groups keys orders 'xy|z' before 'x|yz' (the '|' byte sorts + // after 'y'), so the expectation must match that key order for assertSame(). + $expected = ['xy|z' => 1, 'x|yz' => 2]; + + static::assertCount(2, $expected); + static::assertSame($expected, $run(config_builder())); + static::assertSame($expected, $run(config_builder()->groupingFilesystem('file'))); + } + + public function test_get_groups_groups_null_value_under_null(): void + { + $input = [ + ['k' => 'a'], + ['k' => null], + ['k' => 'a'], + ]; + + $groups = []; + + foreach (data_frame()->read(from_array($input))->groupBy(ref('k'))->getGroups()->get() as $rows) { + $keys = []; + + foreach ($rows as $row) { + $value = $row->valueOf(ref('k')); + $keys[is_string($value) ? $value : 'NULL'] = true; + } + + static::assertCount(1, $keys); + + $key = array_key_first($keys); + + if ($key === null) { + continue; + } + + $groups[$key] = ($groups[$key] ?? 0) + $rows->count(); + } + + ksort($groups); + + static::assertSame(['NULL' => 1, 'a' => 2], $groups); + } + + public function test_get_groups_empty_input_yields_nothing(): void + { + $batches = iterator_to_array(data_frame()->read(from_array([]))->groupBy(ref('k'))->getGroups()->get()); + + static::assertSame([], $batches); + } + + public function test_get_groups_filesystem_multi_bucket_matches_memory(): void + { + // 1500 rows over 3 keys, evenly interleaved: the external sort's minBatchSize (500) is + // exceeded, so createBucketsFromGenerator() spills 3 sorted buckets (500 rows each) that + // must go through a real k-way merge in Buckets::sort() rather than a single in-memory sort. + $rowCount = 1500; + $keys = ['a', 'b', 'c']; + $keysCount = count($keys); + + $input = []; + + for ($i = 0; $i < $rowCount; $i++) { + $input[] = ['k' => $keys[$i % $keysCount], 'v' => $i]; + } + + $run = static function (ConfigBuilder $config) use ($input): array { + $groups = []; + + foreach (data_frame($config)->read(from_array($input))->groupBy(ref('k'))->getGroups()->get() as $rows) { + $keysSeen = []; + $values = []; + + foreach ($rows as $row) { + $k = $row->valueOf(ref('k')); + + if (is_string($k)) { + $keysSeen[$k] = true; + } + + $values[] = $row->valueOf(ref('v')); + } + + static::assertCount(1, $keysSeen); + + $key = array_key_first($keysSeen); + + if ($key === null) { + continue; + } + + if (!isset($groups[$key])) { + $groups[$key] = ['count' => 0, 'values' => []]; + } + + $groups[$key]['count'] += $rows->count(); + $groups[$key]['values'] = [...$groups[$key]['values'], ...$values]; + } + + ksort($groups); + + foreach ($groups as $key => $group) { + sort($group['values']); + $groups[$key]['values'] = $group['values']; + } + + return $groups; + }; + + $memory = $run(config_builder()); + $filesystem = $run(config_builder()->groupingFilesystem('file')); + + static::assertSame($memory, $filesystem); + static::assertSame(500, $memory['a']['count']); + static::assertSame(500, $memory['b']['count']); + static::assertSame(500, $memory['c']['count']); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 87ec052ccb..d0eb49b2de 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -9,137 +9,252 @@ use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; +use function array_map; +use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\collect; +use function Flow\ETL\DSL\collect_unique; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\count; -use function Flow\ETL\DSL\df; +use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\last; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\sum; -use function is_numeric; +use function iterator_to_array; final class GroupByAggregationTest extends FlowIntegrationTestCase { - private const int ORDERS = 20_000; + public function test_memory_and_filesystem_backends_produce_identical_results(): void + { + $input = [ + ['seller' => 'a', 'amount' => 10], + ['seller' => 'b', 'amount' => 5], + ['seller' => 'a', 'amount' => 20], + ['seller' => 'c', 'amount' => 1], + ['seller' => 'b', 'amount' => 7], + ['seller' => 'a', 'amount' => 3], + ]; - /** - * @var null|list - */ - private static ?array $orders = null; + $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( + data_frame($config) + ->read(from_array($input)) + ->groupBy(ref('seller')) + ->aggregate(count(ref('seller')), sum(ref('amount'))) + ->sortBy(ref('seller')->asc()) + ->getEachAsArray(), + ); - public function test_first_and_last_preserve_input_order_across_spilled_buckets(): void - { - $data = []; + $memory = $pipeline(config_builder()); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); - for ($i = 0; $i < 400; $i++) { - $data[] = ['g' => ($i % 2) === 0 ? 'a' : 'b', 'v' => $i]; - } + static::assertSame($memory, $filesystem); + static::assertCount(3, $memory); - $run = static fn(ConfigBuilder $config): array => df($config) - ->read(from_array($data)) - ->groupBy('g') - ->aggregate(first(ref('v')), last(ref('v'))) - ->sortBy(ref('g')) - ->fetch() - ->toArray(); + $bySeller = []; - $memory = $run(config_builder()); - $spilled = $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1))); + foreach ($memory as $row) { + $bySeller[$row['seller']] = ['count' => $row['seller_count'], 'sum' => $row['amount_sum']]; + } - static::assertSame($memory, $spilled); static::assertSame( [ - ['g' => 'a', 'v_first' => 0, 'v_last' => 398], - ['g' => 'b', 'v_first' => 1, 'v_last' => 399], + 'a' => ['count' => 3, 'sum' => 33], + 'b' => ['count' => 2, 'sum' => 12], + 'c' => ['count' => 1, 'sum' => 1], ], - $spilled, + $bySeller, ); } - public function test_high_cardinality_group_by_is_identical_when_spilling(): void + public function test_multi_column_key_does_not_collide(): void { - $memory = $this->aggregateByEmail(config_builder()); - $spilled = $this->aggregateByEmail(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); + $input = [ + ['a' => 'x', 'b' => 'yz', 'v' => 1], + ['a' => 'xy', 'b' => 'z', 'v' => 1], + ['a' => 'x', 'b' => 'yz', 'v' => 1], + ]; + + $result = iterator_to_array( + data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + ->read(from_array($input)) + ->groupBy(ref('a'), ref('b')) + ->aggregate(sum(ref('v'))) + ->sortBy(ref('a')->asc()) + ->getEachAsArray(), + ); + + static::assertCount(2, $result); + + $byGroup = []; + + foreach ($result as $row) { + $byGroup[(string) $row['a'] . '|' . (string) $row['b']] = $row['v_sum']; + } - static::assertEqualsCanonicalizing($memory, $spilled); + static::assertSame(2, $byGroup['x|yz']); + static::assertSame(1, $byGroup['xy|z']); } - public function test_multi_column_keys_do_not_collide(): void + public function test_chained_filesystem_group_by_stages_do_not_corrupt_each_other(): void { - $data = [ - ['x' => 'a', 'y' => 'bc'], - ['x' => 'ab', 'y' => 'c'], - ['x' => 'a', 'y' => 'bc'], + $input = [ + ['seller' => 'a', 'region' => 'south'], + ['seller' => 'b', 'region' => 'north'], + ['seller' => 'c', 'region' => 'south'], + ['seller' => 'd', 'region' => 'north'], + ['seller' => 'e', 'region' => 'south'], + ['seller' => 'f', 'region' => 'north'], ]; - $run = static fn(ConfigBuilder $config): array => df($config) - ->read(from_array($data)) - ->groupBy('x', 'y') - ->aggregate(count()) - ->sortBy(ref('x')) - ->fetch() - ->toArray(); - - $expected = [ - ['x' => 'a', 'y' => 'bc', '_count' => 2], - ['x' => 'ab', 'y' => 'c', '_count' => 1], - ]; + $result = iterator_to_array( + data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + ->read(from_array($input)) + ->groupBy(ref('seller'), ref('region')) + ->aggregate(count(ref('seller'))) + ->groupBy(ref('region')) + ->aggregate(count(ref('region'))) + ->sortBy(ref('region')->asc()) + ->getEachAsArray(), + ); - static::assertSame($expected, $run(config_builder())); - static::assertSame($expected, $run(config_builder()->aggregationMemoryLimit(Unit::fromBytes(1)))); - } + // 6 distinct sellers -> 6 first-stage groups; second stage counts 3 north + 3 south + $byRegion = []; - public function test_seller_aggregation_is_identical_when_spilling(): void - { - $memory = $this->aggregateBySeller(config_builder()); - $spilled = $this->aggregateBySeller(config_builder()->aggregationMemoryLimit(Unit::fromKb(256))); + foreach ($result as $row) { + $byRegion[$row['region']] = $row['region_count']; + } - static::assertEqualsCanonicalizing($memory, $spilled); - static::assertCount(5, $memory); + static::assertSame(['north' => 3, 'south' => 3], $byRegion); } - private function aggregateByEmail(ConfigBuilder $config): array + public function test_first_and_last_survive_the_filesystem_round_trip(): void { - return df($config)->read(from_array($this->orders()))->groupBy('email')->aggregate(count())->fetch()->toArray(); + $input = [ + ['k' => 'a', 'v' => 1], + ['k' => 'a', 'v' => 2], + ['k' => 'a', 'v' => 3], + ]; + + $result = iterator_to_array( + data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + ->read(from_array($input)) + ->groupBy(ref('k')) + ->aggregate(first(ref('v')), last(ref('v'))) + ->sortBy(ref('k')->asc()) + ->getEachAsArray(), + ); + + static::assertSame(1, $result[0]['v_first']); + static::assertSame(3, $result[0]['v_last']); } - private function aggregateBySeller(ConfigBuilder $config): array + public function test_collect_matches_between_backends_through_spill(): void { - return df($config) - ->read(from_array($this->orders())) - ->groupBy('seller_id') - ->aggregate(count(), sum('discount'), collect('customer')) - ->fetch() - ->toArray(); + $input = [ + ['k' => 'a', 'v' => 1], + ['k' => 'b', 'v' => 4], + ['k' => 'a', 'v' => 2], + ['k' => 'c', 'v' => 6], + ['k' => 'b', 'v' => 5], + ['k' => 'a', 'v' => 3], + ]; + + $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( + data_frame($config) + ->read(from_array($input)) + ->groupBy(ref('k')) + ->aggregate(collect(ref('v')), collect_unique(ref('v'))) + ->sortBy(ref('k')->asc()) + ->getEachAsArray(), + ); + + $memory = $pipeline(config_builder()); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); + + // The k-way merge folds partial collections in run-creation order, which for this + // single-extractor pipeline matches input order, so the raw arrays come out identical + // between backends without needing an order-insensitive comparison. + static::assertSame($memory, $filesystem); + + $byKey = []; + + foreach ($memory as $row) { + $byKey[$row['k']] = [ + 'collection' => $row['v_collection'], + 'collection_unique' => $row['v_collection_unique'], + ]; + } + + static::assertSame( + [ + 'a' => ['collection' => [1, 2, 3], 'collection_unique' => [1, 2, 3]], + 'b' => ['collection' => [4, 5], 'collection_unique' => [4, 5]], + 'c' => ['collection' => [6], 'collection_unique' => [6]], + ], + $byKey, + ); } - /** - * @return list - */ - private function orders(): array + public function test_average_matches_between_backends_through_spill(): void { - if (self::$orders !== null) { - return self::$orders; - } + $input = [ + ['k' => 'a', 'v' => 2], + ['k' => 'b', 'v' => 10], + ['k' => 'c', 'v' => 7], + ['k' => 'a', 'v' => 4], + ['k' => 'b', 'v' => 20], + ['k' => 'b', 'v' => 30], + ]; - $orders = []; + $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( + data_frame($config) + ->read(from_array($input)) + ->groupBy(ref('k')) + ->aggregate(average(ref('v'))) + ->sortBy(ref('k')->asc()) + ->getEachAsArray(), + ); - foreach ((new FakeRandomOrdersExtractor(self::ORDERS))->rawData() as $order) { - /** @var mixed $discount */ - $discount = $order['discount']; + $memory = $pipeline(config_builder()); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); - $orders[] = [ - 'seller_id' => (string) $order['seller_id'], - 'email' => (string) $order['email'], - 'customer' => (string) $order['customer'], - 'discount' => is_numeric($discount) ? (float) $discount : null, - ]; + // Each row lands in its own single-value partial Bucket under a byte-tiny memory limit, so + // the k-way merge must fold sum+count across (at least) two partial states per group. If the + // merge instead averaged partial averages (average-of-averages) rather than summing the two + // fields, a differently-weighted fold would drift from the true value. + static::assertSame($memory, $filesystem); + + $byKey = []; + + foreach ($memory as $row) { + $byKey[$row['k']] = $row['v_avg']; } - self::$orders = $orders; + static::assertSame(['a' => 3, 'b' => 20, 'c' => 7], $byKey); + } + + public function test_backends_match_on_a_realistic_dataset(): void + { + $raw = iterator_to_array((new FakeRandomOrdersExtractor(1000))->rawData(), false); + $data = array_map(static fn(array $order): array => [ + 'seller_id' => $order['seller_id'], + 'email' => $order['email'], + 'discount' => $order['discount'], + ], $raw); + + $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( + data_frame($config) + ->read(from_array($data)) + ->groupBy(ref('email')) + ->aggregate(count(ref('email')), sum(ref('discount'))) + ->sortBy(ref('email')->asc()) + ->getEachAsArray(), + ); - return $orders; + static::assertSame( + $pipeline(config_builder()), + $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php new file mode 100644 index 0000000000..0c6a945b8a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php @@ -0,0 +1,59 @@ +rawData(), false); + + $data = array_map(static fn(array $order): array => [ + 'seller_id' => $order['seller_id'], + 'email' => $order['email'], + 'discount' => $order['discount'], + ], $raw); + + $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( + data_frame($config) + ->read(from_array($data)) + ->groupBy(ref('email')) + ->aggregate(count(ref('email')), sum(ref('discount'))) + ->sortBy(ref('email')->asc()) + ->getEachAsArray(), + ); + + $memory = $pipeline(config_builder()); + + $start = microtime(true); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromMb(4))); + $elapsed = microtime(true) - $start; + + static::assertSame($memory, $filesystem); + + // Benchmarked at ~2s; 30s is a generous, non-flaky ceiling that still catches a regression + // toward the old super-linear spill cliff (~80s). + static::assertLessThan( + 30.0, + $elapsed, + 'filesystem aggregation at 100k regressed toward the old super-linear cliff', + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php new file mode 100644 index 0000000000..4e82a10116 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php @@ -0,0 +1,199 @@ +fs(), cacheDir: $this->cacheDir), + Unit::fromBytes(1), // force a spill on essentially every merge + 2, // small fan-in to exercise recursive merge + ); + + // 50 groups, each seen 3 times (so a group is folded across runs) + for ($round = 0; $round < 3; $round++) { + for ($g = 0; $g < 50; $g++) { + foreach ([$memory, $external] as $store) { + $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); + $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); + $store->merge((string) $g, $bucket); + } + } + } + + $normalize = static function (iterable $buckets) use ($context): array { + /** @var iterable $buckets */ + $out = []; + + foreach ($buckets as $bucket) { + $out[(string) $bucket->keyValues['k']] = $bucket->aggregators[0] + ->result($context->entryFactory()) + ->value(); + } + + ksort($out); + + return $out; + }; + + $expected = $normalize($memory->all()); // each group summed 3x its own value + $actual = $normalize($external->all()); + + static::assertSame($expected, $actual); + static::assertCount(50, $actual); + + $external->clear(); + } + + /** + * Tunes the memory limit so that each spilled run holds MANY ascending-hash buckets (not one), + * then folds every group across several of those multi-entry runs. Exercises the subtlest + * property: a k-way merge that must fold equal hashes drawn from different, many-entry runs. + */ + public function test_parity_with_multi_entry_runs(): void + { + $context = flow_context(); + $memory = new InMemoryBuckets(); + $external = new ExternalBuckets( + new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + // large enough that a run holds many groups (multi-entry), small enough to spill several times + Unit::fromKb(64), + 2, // small fan-in forces recursive merge of the multi-entry runs + ); + + // 2000 groups, each seen in 2 rounds (so a group is folded across several multi-entry runs) + for ($round = 0; $round < 2; $round++) { + for ($g = 0; $g < 2000; $g++) { + foreach ([$memory, $external] as $store) { + $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); + $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); + $store->merge((string) $g, $bucket); + } + } + } + + $normalize = static function (iterable $buckets) use ($context): array { + /** @var iterable $buckets */ + $out = []; + + foreach ($buckets as $bucket) { + $out[(string) $bucket->keyValues['k']] = $bucket->aggregators[0] + ->result($context->entryFactory()) + ->value(); + } + + ksort($out); + + return $out; + }; + + $expected = $normalize($memory->all()); // each group summed to 2x its own value + $actual = $normalize($external->all()); + + static::assertSame($expected, $actual); + static::assertCount(2000, $actual); + + $external->clear(); + } + + /** + * Regression test for a temp-file leak: abandoning `all()` mid-yield used to leave the + * final-level intermediate merged runs orphaned on disk, because the cascade tracked the + * live run ids in a LOCAL variable instead of $this->runIds, so clear() only ever removed + * the (already deleted) original runs. + */ + public function test_clear_removes_intermediate_runs_after_abandoned_all(): void + { + $context = flow_context(); + $external = new ExternalBuckets( + new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + Unit::fromBytes(1), // force a spill on essentially every merge -> many runs + 2, // small fan-in -> multi-level cascade + ); + + for ($g = 0; $g < 20; $g++) { + $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); + $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); + $external->merge((string) $g, $bucket); + } + + foreach ($external->all() as $_bucket) { + break; + } + + $external->clear(); + + $cacheFiles = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($this->cacheDir->path(), FilesystemIterator::SKIP_DOTS), + ); + + $leftoverRunFiles = 0; + + foreach ($cacheFiles as $file) { + /** @var SplFileInfo $file */ + if (str_ends_with($file->getFilename(), '.php.cache')) { + $leftoverRunFiles++; + } + } + + static::assertSame(0, $leftoverRunFiles); + } + + public function test_empty_yields_nothing(): void + { + $external = new ExternalBuckets( + new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + Unit::fromMb(64), + ); + + static::assertSame([], iterator_to_array($external->all(), false)); + } + + public function test_single_group_fast_path(): void + { + $context = flow_context(); + $external = new ExternalBuckets( + new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + Unit::fromMb(64), // large enough that nothing spills -> exercises the no-runs fast path + ); + + $bucket = new Bucket(['k' => 7], [sum(ref('v'))]); + $bucket->aggregators[0]->aggregate(row(int_entry('v', 7)), $context); + $external->merge('7', $bucket); + + $result = iterator_to_array($external->all(), false); + + static::assertCount(1, $result); + static::assertSame(7, $result[0]->keyValues['k']); + static::assertSame(7, $result[0]->aggregators[0]->result($context->entryFactory())->value()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php new file mode 100644 index 0000000000..37c140a140 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php @@ -0,0 +1,109 @@ +fs(), cacheDir: $this->cacheDir); + + $a = new Bucket(['k' => 'a'], [sum(ref('v'))]); + $a->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); + $b = new Bucket(['k' => 'b'], [sum(ref('v'))]); + $b->aggregators[0]->aggregate(row(int_entry('v', 5)), $context); + + $cache->set('run-1', ['aaa' => $a, 'bbb' => $b]); + + $read = []; + + foreach ($cache->get('run-1') as $hash => $bucket) { + $read[$hash] = $bucket->aggregators[0]->result($context->entryFactory())->value(); + } + + static::assertSame(['aaa' => 2, 'bbb' => 5], $read); + + $cache->remove('run-1'); + + static::assertSame([], iterator_to_array($cache->get('run-1'), false)); + } + + public function test_corrupt_run_raises_a_runtime_exception(): void + { + $cache = new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir); + + $cache->set('run-corrupt', ['aaa' => new Bucket(['k' => 'a'], [sum(ref('v'))])]); + + $path = $this->cacheDir->suffix( + 'flow-php-group-by/' . NativePHPHash::xxh128('run-corrupt') . '/run-corrupt.php.cache', + ); + + $this->fs()->writeTo($path)->append('not-a-valid-line')->close(); + + static::expectException(RuntimeException::class); + + iterator_to_array($cache->get('run-corrupt')); + } + + public function test_invalid_base64_payload_raises_runtime_exception(): void + { + $cache = new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir); + + $cache->set('run-bad-b64', ['aaa' => new Bucket(['k' => 'a'], [sum(ref('v'))])]); + + $path = $this->cacheDir->suffix( + 'flow-php-group-by/' . NativePHPHash::xxh128('run-bad-b64') . '/run-bad-b64.php.cache', + ); + + // Valid hash/bucket separator, but the payload after the space is not valid base64. + $this + ->fs() + ->writeTo($path) + ->append(base64_encode('aaa') . ' !!!not-base64!!!') + ->close(); + + static::expectException(RuntimeException::class); + + iterator_to_array($cache->get('run-bad-b64')); + } + + public function test_preserves_caller_order_across_chunk_flushes(): void + { + $context = flow_context(); + $cache = new FilesystemBucketRunCache($this->fs(), chunkSize: 1, cacheDir: $this->cacheDir); + + $ccc = new Bucket(['k' => 'c'], [sum(ref('v'))]); + $ccc->aggregators[0]->aggregate(row(int_entry('v', 3)), $context); + $bbb = new Bucket(['k' => 'b'], [sum(ref('v'))]); + $bbb->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); + $aaa = new Bucket(['k' => 'a'], [sum(ref('v'))]); + $aaa->aggregators[0]->aggregate(row(int_entry('v', 1)), $context); + + $cache->set('run-2', ['ccc' => $ccc, 'bbb' => $bbb, 'aaa' => $aaa]); + + $read = []; + + foreach ($cache->get('run-2') as $hash => $bucket) { + $read[$hash] = $bucket->aggregators[0]->result($context->entryFactory())->value(); + } + + static::assertSame(['ccc' => 3, 'bbb' => 2, 'aaa' => 1], $read); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php deleted file mode 100644 index a042c3ebfc..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemGroupBucketsCacheTest.php +++ /dev/null @@ -1,54 +0,0 @@ -fs(), $this->serializer(), 100, $this->cacheDir); - - static::assertSame([], iterator_to_array($cache->get('does-not-exist'))); - } - - public function test_round_trip_and_remove(): void - { - $cache = new FilesystemGroupBucketsCache($this->fs(), $this->serializer(), 2, $this->cacheDir); - - $groups = [ - new HashedGroup('aaa', new GroupState(['type' => 'a'], [sum(ref('v'))])), - new HashedGroup('bbb', new GroupState(['type' => 'b'], [sum(ref('v'))])), - new HashedGroup('ccc', new GroupState(['type' => 'c'], [sum(ref('v'))])), - ]; - - $cache->set( - 'bucket-1', - (static function () use ($groups) { - yield from $groups; - })(), - ); - - $read = []; - - foreach ($cache->get('bucket-1') as $group) { - $read[$group->hash] = $group->state->keyValues; - } - - static::assertSame(['aaa' => ['type' => 'a'], 'bbb' => ['type' => 'b'], 'ccc' => ['type' => 'c']], $read); - - $cache->remove('bucket-1'); - - static::assertSame([], iterator_to_array($cache->get('bucket-1'))); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php deleted file mode 100644 index 3dd6226210..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Aggregation/AggregationConfigBuilderTest.php +++ /dev/null @@ -1,49 +0,0 @@ -build(); - - static::assertSame(PHP_INT_MAX, $config->memoryLimit->inBytes()); - static::assertSame('file', $config->filesystemProtocol); - } - - public function test_memory_limit_is_read_from_the_environment(): void - { - putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV . '=32M'); - - try { - $config = (new AggregationConfigBuilder())->build(); - - static::assertSame(Unit::fromMb(32)->inBytes(), $config->memoryLimit->inBytes()); - } finally { - putenv(AggregationConfig::AGGREGATION_MAX_MEMORY_ENV); - } - } - - public function test_setting_a_memory_limit_and_filesystem(): void - { - $config = (new AggregationConfigBuilder()) - ->memoryLimit(Unit::fromMb(16)) - ->filesystemProtocol('memory') - ->build(); - - static::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); - static::assertSame('memory', $config->filesystemProtocol); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php new file mode 100644 index 0000000000..856f51e553 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php @@ -0,0 +1,43 @@ +build(); + + static::assertSame(GroupingBackend::Memory, $config->backend); + static::assertSame('file', $config->filesystemProtocol); + static::assertSame(10, $config->bucketsCount); + } + + public function test_filesystem_protocol_flips_backend(): void + { + $config = (new GroupingConfigBuilder()) + ->filesystemProtocol('local') + ->build(); + + static::assertSame(GroupingBackend::Filesystem, $config->backend); + static::assertSame('local', $config->filesystemProtocol); + } + + public function test_memory_limit_and_buckets_count_are_set(): void + { + $config = (new GroupingConfigBuilder()) + ->memoryLimit(Unit::fromMb(16)) + ->bucketsCount(4) + ->build(); + + static::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); + static::assertSame(4, $config->bucketsCount); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php new file mode 100644 index 0000000000..d4ce714a7a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php @@ -0,0 +1,113 @@ +aggregate(sum(ref('v'))); + + $batches = (static function () { + yield rows(row(str_entry('k', 'a'), int_entry('v', 1)), row(str_entry('k', 'b'), int_entry('v', 10))); + yield rows(row(str_entry('k', 'a'), int_entry('v', 2)), row(str_entry('k', 'b'), int_entry('v', 20))); + })(); + + $sums = []; + + foreach ((new BucketAggregation(new InMemoryBuckets()))->aggregate( + $batches, + $context, + $groupBy, + ) as $resultRows) { + foreach ($resultRows as $resultRow) { + $key = $resultRow->valueOf(ref('k')); + assert(is_string($key)); + + $sums[$key] = $resultRow->valueOf(ref('v_sum')); + } + } + + static::assertSame(['a' => 3, 'b' => 30], $sums); + } + + public function test_first_preserves_input_order_across_batches(): void + { + $context = flow_context(); + + $groupBy = new GroupBy(ref('k')); + $groupBy->aggregate(first(ref('v'))); + + $batches = (static function () { + yield rows(row(str_entry('k', 'a'), int_entry('v', 100))); + yield rows(row(str_entry('k', 'a'), int_entry('v', 200))); + })(); + + $firsts = []; + + foreach ((new BucketAggregation(new InMemoryBuckets()))->aggregate( + $batches, + $context, + $groupBy, + ) as $resultRows) { + foreach ($resultRows as $resultRow) { + $key = $resultRow->valueOf(ref('k')); + assert(is_string($key)); + + $firsts[$key] = $resultRow->valueOf(ref('v_first')); + } + } + + static::assertSame(['a' => 100], $firsts); + } + + public function test_last_keeps_the_latest_across_batches(): void + { + $context = flow_context(); + + $groupBy = new GroupBy(ref('k')); + $groupBy->aggregate(last(ref('v'))); + + $batches = (static function () { + yield rows(row(str_entry('k', 'a'), int_entry('v', 100))); + yield rows(row(str_entry('k', 'a'), int_entry('v', 200))); + })(); + + $lasts = []; + + foreach ((new BucketAggregation(new InMemoryBuckets()))->aggregate( + $batches, + $context, + $groupBy, + ) as $resultRows) { + foreach ($resultRows as $resultRow) { + $key = $resultRow->valueOf(ref('k')); + assert(is_string($key)); + + $lasts[$key] = $resultRow->valueOf(ref('v_last')); + } + } + + static::assertSame(['a' => 200], $lasts); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php new file mode 100644 index 0000000000..a2ba6a5a02 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php @@ -0,0 +1,43 @@ + 'a'], [sum(ref('v'))]); + $left->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); + + $right = new Bucket(['k' => 'a'], [sum(ref('v'))]); + $right->aggregators[0]->aggregate(row(int_entry('v', 3)), $context); + + $left->merge($right); + + static::assertSame(5, $left->aggregators[0]->result($context->entryFactory())->value()); + } + + public function test_merge_delegates_validation_to_the_aggregators(): void + { + $this->expectException(InvalidArgumentException::class); + + $left = new Bucket(['k' => 'a'], [sum(ref('a'))]); + $right = new Bucket(['k' => 'a'], [sum(ref('b'))]); + + $left->merge($right); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php new file mode 100644 index 0000000000..ec4b307219 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php @@ -0,0 +1,38 @@ +insert(new BucketRun('b', 0, new Bucket([], []))); + $heap->insert(new BucketRun('a', 2, new Bucket([], []))); + $heap->insert(new BucketRun('a', 1, new Bucket([], []))); + + $order = []; + + while (!$heap->isEmpty()) { + $run = $heap->extract(); + $order[] = $run->hash . $run->runIndex; + } + + static::assertSame(['a1', 'a2', 'b0'], $order); + } + + public function test_rejects_non_bucket_run(): void + { + $this->expectException(InvalidArgumentException::class); + + (new BucketsMinHeap())->insert('nope'); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php new file mode 100644 index 0000000000..cea646f9d3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php @@ -0,0 +1,47 @@ +groups($sorted, $groupBy) as $run) { + $result[(string) $run->keyValues['k']] = array_map( + static fn($r) => $r->valueOf(ref('v')), + iterator_to_array($run->rows, false), + ); + } + + static::assertSame(['a' => [1, 2, 3], 'b' => [4], 'c' => [5, 6]], $result); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php deleted file mode 100644 index 5fe062c8cc..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupsMinHeapTest.php +++ /dev/null @@ -1,38 +0,0 @@ -insert(new BucketGroup('b', 0, new GroupState([], []))); - $heap->insert(new BucketGroup('a', 2, new GroupState([], []))); - $heap->insert(new BucketGroup('a', 1, new GroupState([], []))); - - $order = []; - - while (!$heap->isEmpty()) { - $group = $heap->extract(); - $order[] = $group->hash . $group->bucketIndex; - } - - static::assertSame(['a1', 'a2', 'b0'], $order); - } - - public function test_rejects_a_non_bucket_group(): void - { - $this->expectException(InvalidArgumentException::class); - - (new GroupsMinHeap())->insert('not-a-bucket-group'); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php new file mode 100644 index 0000000000..37361aad1b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php @@ -0,0 +1,51 @@ + 'x'], [sum(ref('v'))]); + $a->aggregators[0]->aggregate(row(int_entry('v', 1)), $context); + $buckets->merge('h1', $a); + + $b = new Bucket(['k' => 'x'], [sum(ref('v'))]); + $b->aggregators[0]->aggregate(row(int_entry('v', 4)), $context); + $buckets->merge('h1', $b); + + $all = iterator_to_array($buckets->all(), false); + + static::assertCount(1, $all); + static::assertSame(5, $all[0]->aggregators[0]->result($context->entryFactory())->value()); + } + + public function test_distinct_hashes_stay_separate_and_clear_empties(): void + { + $buckets = new InMemoryBuckets(); + $buckets->merge('h1', new Bucket(['k' => 'x'], [])); + $buckets->merge('h2', new Bucket(['k' => 'y'], [])); + + static::assertCount(2, iterator_to_array($buckets->all(), false)); + + $buckets->clear(); + + static::assertCount(0, iterator_to_array($buckets->all(), false)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php index 89926efe6f..a34e301a1c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_entry; use function Flow\ETL\DSL\null_entry; @@ -25,9 +26,14 @@ final class GroupByTest extends FlowTestCase public function test_group_by_missing_entry(): void { $groupBy = new GroupBy('type'); + $groupBy->aggregate(count()); static::assertEquals( - rows(row(str_entry('type', 'a')), row(null_entry('type')), row(str_entry('type', 'c'))), + rows( + row(str_entry('type', 'a'), int_entry('_count', 1)), + row(null_entry('type'), int_entry('_count', 1)), + row(str_entry('type', 'c'), int_entry('_count', 1)), + ), $this->aggregate($groupBy, rows( row(str_entry('type', 'a')), row(str_entry('not-type', 'b')), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php index 0659e97f71..4bd42fca48 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByProcessorTest.php @@ -35,14 +35,21 @@ public function test_groups_across_multiple_batches(): void /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); - static::assertCount(1, $result); + $aggregated = []; - $resultArray = $result[0]->toArray(); - static::assertCount(2, $resultArray); + foreach ($result as $batch) { + foreach ($batch->toArray() as $groupRow) { + $aggregated[] = $groupRow; + } + } + + static::assertCount(2, $aggregated); - $categoryA = array_values(array_filter($resultArray, static fn(array $r): bool => $r['category'] === 'a'))[0]; + $categoryA = array_values(array_filter($aggregated, static fn(array $r): bool => $r['category'] === 'a'))[0]; + $categoryB = array_values(array_filter($aggregated, static fn(array $r): bool => $r['category'] === 'b'))[0]; - static::assertEquals(30, $categoryA['amount_sum']); + static::assertSame(30, $categoryA['amount_sum']); + static::assertSame(15, $categoryB['amount_sum']); } public function test_groups_and_aggregates_rows(): void @@ -63,16 +70,21 @@ public function test_groups_and_aggregates_rows(): void /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); - static::assertCount(1, $result); + $aggregated = []; + + foreach ($result as $batch) { + foreach ($batch->toArray() as $groupRow) { + $aggregated[] = $groupRow; + } + } - $resultArray = $result[0]->toArray(); - static::assertCount(2, $resultArray); + static::assertCount(2, $aggregated); - $categoryA = array_values(array_filter($resultArray, static fn(array $r): bool => $r['category'] === 'a'))[0]; - $categoryB = array_values(array_filter($resultArray, static fn(array $r): bool => $r['category'] === 'b'))[0]; + $categoryA = array_values(array_filter($aggregated, static fn(array $r): bool => $r['category'] === 'a'))[0]; + $categoryB = array_values(array_filter($aggregated, static fn(array $r): bool => $r['category'] === 'b'))[0]; - static::assertEquals(30, $categoryA['amount_sum']); - static::assertEquals(15, $categoryB['amount_sum']); + static::assertSame(30, $categoryA['amount_sum']); + static::assertSame(15, $categoryB['amount_sum']); } public function test_handles_empty_input(): void From 497ad35584e7f3dd613136b92a476f4d7399bbc0 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Wed, 8 Jul 2026 21:57:40 +0200 Subject: [PATCH 10/13] refactor(flow-php/etl): typed grouping collections and simpler grouping internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the group identity onto the bucket and replace raw array seams with typed objects: Bucket is now {hash, GroupKey, Aggregators}, Buckets::merge() takes the bucket alone, and the run cache reads/writes plain serialized buckets (one field per line) instead of hash-keyed generators. GroupKey owns the group key and its hashing (moved from GroupBy::hash()); Aggregators owns the aggregator set behaviour (aggregate/merge/cloned/first, replacing the raw array and newAggregators()). Simplify the engine: collapse the single-implementation BucketRunCache interface into one concrete class, collapse GroupedRuns + GroupRun into a single GroupBatches iterator, move the result batch size from GroupBy onto GroupByProcessor, and share the spill-dir construction between both Filesystem arms. getGroups() on the Memory backend now hash-partitions instead of sorting (per the design: one pass, first-seen group order, no sort cost) — 100k rows groups in 1.1s instead of 2.8s; group columns must still be present on every row on both backends. The lazy GroupBatches also drops the Filesystem getGroups() peak by ~50MB. --- .../Flow/ETL/DataFrame/GroupedDataFrame.php | 40 +++--- src/core/etl/src/Flow/ETL/GroupBy.php | 109 ++------------ .../etl/src/Flow/ETL/GroupBy/Aggregators.php | 84 +++++++++++ src/core/etl/src/Flow/ETL/GroupBy/Bucket.php | 15 +- .../Flow/ETL/GroupBy/BucketAggregation.php | 19 +-- .../src/Flow/ETL/GroupBy/BucketRunCache.php | 105 +++++++++++++- src/core/etl/src/Flow/ETL/GroupBy/Buckets.php | 2 +- .../BucketsCache/FilesystemBucketRunCache.php | 135 ------------------ .../src/Flow/ETL/GroupBy/BucketsMinHeap.php | 14 +- .../src/Flow/ETL/GroupBy/ExternalBuckets.php | 33 +++-- .../etl/src/Flow/ETL/GroupBy/GroupBatches.php | 66 +++++++++ .../etl/src/Flow/ETL/GroupBy/GroupKey.php | 84 +++++++++++ .../etl/src/Flow/ETL/GroupBy/GroupRun.php | 23 --- .../etl/src/Flow/ETL/GroupBy/GroupedRuns.php | 82 ----------- .../src/Flow/ETL/GroupBy/InMemoryBuckets.php | 12 +- .../GroupBy/{BucketRun.php => RunBucket.php} | 3 +- .../Flow/ETL/Processor/GroupByProcessor.php | 129 ++++++++++------- .../src/Flow/ETL/Sort/ExternalSortFactory.php | 2 +- .../GroupBy/BucketRunCacheTest.php | 113 +++++++++++++++ .../GroupBy/ExternalBucketsTest.php | 56 ++++---- .../GroupBy/FilesystemBucketRunCacheTest.php | 109 -------------- .../ETL/Tests/Unit/GroupBy/BucketTest.php | 16 ++- .../Tests/Unit/GroupBy/BucketsMinHeapTest.php | 14 +- .../Tests/Unit/GroupBy/GroupBatchesTest.php | 99 +++++++++++++ .../Tests/Unit/GroupBy/GroupedRunsTest.php | 47 ------ .../Unit/GroupBy/InMemoryBucketsTest.php | 20 +-- 26 files changed, 757 insertions(+), 674 deletions(-) create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php create mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php rename src/core/etl/src/Flow/ETL/GroupBy/{BucketRun.php => RunBucket.php} (76%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php diff --git a/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php b/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php index 6297b7c741..5ed292dfbd 100644 --- a/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php +++ b/src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php @@ -21,32 +21,15 @@ public function aggregate(AggregatingFunction ...$aggregations): DataFrame { $this->groupBy->aggregate(...$aggregations); - $pipelineAdder = function (GroupBy $groupBy): void { - // @mago-ignore analysis:non-existent-property,method-access-on-null - $this->pipeline->add(new GroupByProcessor($groupBy)); - }; - - // @mago-ignore analysis:invalid-method-access - $pipelineAdder->bindTo($this->df, $this->df)($this->groupBy); - - return $this->df; + return $this->addProcessor(new GroupByProcessor($this->groupBy)); } + /** + * @param null|int<1, max> $size + */ public function getGroups(?int $size = null): DataFrame { - if ($size !== null) { - $this->groupBy->batchSize($size); - } - - $pipelineAdder = function (GroupBy $groupBy): void { - // @mago-ignore analysis:non-existent-property,method-access-on-null - $this->pipeline->add(new GroupByProcessor($groupBy)); - }; - - // @mago-ignore analysis:invalid-method-access - $pipelineAdder->bindTo($this->df, $this->df)($this->groupBy); - - return $this->df; + return $this->addProcessor(new GroupByProcessor($this->groupBy, $size ?? GroupByProcessor::DEFAULT_BATCH_SIZE)); } public function pivot(Reference $ref): self @@ -55,4 +38,17 @@ public function pivot(Reference $ref): self return $this; } + + private function addProcessor(GroupByProcessor $processor): DataFrame + { + $pipelineAdder = function (GroupByProcessor $processor): void { + // @mago-ignore analysis:non-existent-property,method-access-on-null + $this->pipeline->add($processor); + }; + + // @mago-ignore analysis:invalid-method-access + $pipelineAdder->bindTo($this->df, $this->df)($processor); + + return $this->df; + } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index c0172e5809..86c4e288bb 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -4,44 +4,31 @@ namespace Flow\ETL; -use DateTimeImmutable; -use DateTimeInterface; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Function\AggregatingFunction; -use Flow\ETL\Hash\NativePHPHash; +use Flow\ETL\GroupBy\Aggregators; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\Row\EntryFactory; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Generator; -use Stringable; use function array_filter; use function array_key_exists; -use function array_map; use function array_unique; use function array_values; use function count; -use function current; use function Flow\ETL\DSL\array_to_rows; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_union; -use function is_array; -use function is_scalar; -use function serialize; -use function strlen; final class GroupBy { private const int RESULT_BATCH_SIZE = 1000; - /** - * @var array - */ - private array $aggregations = []; - - private int $batchSize = self::RESULT_BATCH_SIZE; + private Aggregators $aggregations; private ?Reference $pivot = null; @@ -50,6 +37,7 @@ final class GroupBy public function __construct(string|Reference ...$entries) { $this->refs = References::init(...array_unique($entries)); + $this->aggregations = new Aggregators(); } public function aggregate(AggregatingFunction ...$aggregator): void @@ -64,19 +52,15 @@ public function aggregate(AggregatingFunction ...$aggregator): void ); } - $this->aggregations = $aggregator; + $this->aggregations = new Aggregators(...$aggregator); } - /** - * @param array $keyValues - * @param array $aggregators - */ - public function aggregatedRow(array $keyValues, array $aggregators, EntryFactory $entryFactory): Row + public function aggregatedRow(GroupKey $key, Aggregators $aggregators, EntryFactory $entryFactory): Row { $entries = []; /** @var mixed $value */ - foreach ($keyValues as $name => $value) { + foreach ($key as $name => $value) { $entries[] = $entryFactory->create($name, $value); } @@ -87,63 +71,14 @@ public function aggregatedRow(array $keyValues, array $aggregators, EntryFactory return Row::create(...$entries); } - /** - * @return array - */ - public function aggregations(): array + public function aggregations(): Aggregators { return $this->aggregations; } - public function batchSize(int $size): void - { - $this->batchSize = $size; - } - - public function getBatchSize(): int - { - return $this->batchSize; - } - public function hasAggregations(): bool { - return $this->aggregations !== []; - } - - /** - * @param array $values - */ - public function hash(array $values): string - { - /** @var array $stringValues */ - $stringValues = []; - - /** @var mixed $value */ - foreach ($values as $value) { - if ($value === null) { - $stringValues[] = 'null'; - } elseif (is_scalar($value)) { - $stringValues[] = (string) $value; - } else { - if ($value instanceof Stringable) { - $stringValues[] = $value->__toString(); - } elseif ($value instanceof DateTimeInterface) { - $stringValues[] = $value->format(DateTimeImmutable::ATOM); - } elseif (is_array($value)) { - $stringValues[] = $this->hash($value); - } else { - $stringValues[] = serialize($value); - } - } - } - - $key = ''; - - foreach ($stringValues as $stringValue) { - $key .= strlen($stringValue) . ':' . $stringValue; - } - - return NativePHPHash::xxh128($key); + return $this->aggregations->count() > 0; } public function isPivot(): bool @@ -151,10 +86,7 @@ public function isPivot(): bool return $this->pivot !== null; } - /** - * @return array - */ - public function keyValues(Row $row): array + public function keyValues(Row $row): GroupKey { $values = []; @@ -166,18 +98,7 @@ public function keyValues(Row $row): array } } - return $values; - } - - /** - * @return array - */ - public function newAggregators(): array - { - return array_map( - static fn(AggregatingFunction $aggregation): AggregatingFunction => clone $aggregation, - $this->aggregations, - ); + return new GroupKey($values); } public function pivot(Reference $ref): void @@ -198,12 +119,12 @@ public function pivotResult(Generator $rows, FlowContext $context): Generator throw new RuntimeException('pivotResult() called without a pivot reference'); } - $aggregation = current($this->aggregations); - - if (!$aggregation instanceof AggregatingFunction) { + if ($this->aggregations->count() === 0) { throw new RuntimeException('Pivot requires exactly one aggregation'); } + $aggregation = $this->aggregations->first(); + /** @var array|bool|float|int|object|string> $pivotColumns */ $pivotColumns = []; /** @var array|bool|float|int|object|string>> $pivotedTable */ @@ -225,7 +146,7 @@ public function pivotResult(Generator $rows, FlowContext $context): Generator $values[$ref->name()] = $row->valueOf($ref); } - $indexValue = $this->hash($values); + $indexValue = (new GroupKey($values))->hash(); $pivotValue = $row->valueOf($pivot); if (!array_key_exists($indexValue, $pivotedTable)) { diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php b/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php new file mode 100644 index 0000000000..e1b58eb8a8 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php @@ -0,0 +1,84 @@ + + */ +final readonly class Aggregators implements Countable, IteratorAggregate +{ + /** + * @var list + */ + private array $aggregators; + + public function __construct(AggregatingFunction ...$aggregators) + { + $this->aggregators = array_values($aggregators); + } + + public function aggregate(Row $row, FlowContext $context): void + { + foreach ($this->aggregators as $aggregator) { + $aggregator->aggregate($row, $context); + } + } + + public function cloned(): self + { + return new self(...array_map( + static fn(AggregatingFunction $aggregator): AggregatingFunction => clone $aggregator, + $this->aggregators, + )); + } + + public function count(): int + { + return count($this->aggregators); + } + + public function first(): AggregatingFunction + { + if ($this->aggregators === []) { + throw new InvalidArgumentException('Aggregators are empty.'); + } + + return $this->aggregators[0]; + } + + public function getIterator(): Traversable + { + return new ArrayIterator($this->aggregators); + } + + public function merge(self $other): void + { + if (count($this->aggregators) !== count($other->aggregators)) { + throw new InvalidArgumentException(sprintf( + 'Cannot merge aggregators, count mismatch: %d != %d', + count($this->aggregators), + count($other->aggregators), + )); + } + + foreach ($this->aggregators as $index => $aggregator) { + $aggregator->merge($other->aggregators[$index]); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php b/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php index df4618d437..c97396e659 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/Bucket.php @@ -4,26 +4,19 @@ namespace Flow\ETL\GroupBy; -use Flow\ETL\Function\AggregatingFunction; - /** * @internal */ final readonly class Bucket { - /** - * @param array $keyValues - * @param array $aggregators - */ public function __construct( - public array $keyValues, - public array $aggregators, + public string $hash, + public GroupKey $key, + public Aggregators $aggregators, ) {} public function merge(self $other): void { - foreach ($this->aggregators as $index => $aggregator) { - $aggregator->merge($other->aggregators[$index]); - } + $this->aggregators->merge($other->aggregators); } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php index 493caf4787..0301e20e90 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php @@ -35,27 +35,22 @@ public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupB $local = []; foreach ($batch as $row) { - $keyValues = $groupBy->keyValues($row); - $hash = $groupBy->hash($keyValues); + $key = $groupBy->keyValues($row); + $hash = $key->hash(); - if (!isset($local[$hash])) { - $local[$hash] = new Bucket($keyValues, $groupBy->newAggregators()); - } - - foreach ($local[$hash]->aggregators as $aggregator) { - $aggregator->aggregate($row, $context); - } + $local[$hash] ??= new Bucket($hash, $key, $groupBy->aggregations()->cloned()); + $local[$hash]->aggregators->aggregate($row, $context); } - foreach ($local as $hash => $bucket) { - $this->buckets->merge($hash, $bucket); + foreach ($local as $bucket) { + $this->buckets->merge($bucket); } } $buffer = []; foreach ($this->buckets->all() as $bucket) { - $buffer[] = $groupBy->aggregatedRow($bucket->keyValues, $bucket->aggregators, $context->entryFactory()); + $buffer[] = $groupBy->aggregatedRow($bucket->key, $bucket->aggregators, $context->entryFactory()); if (count($buffer) >= self::RESULT_BATCH_SIZE) { yield new Rows(...$buffer); diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketRunCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketRunCache.php index baf9f4523b..f06e4fbf8b 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketRunCache.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketRunCache.php @@ -4,22 +4,115 @@ namespace Flow\ETL\GroupBy; +use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Hash\NativePHPHash; +use Flow\Filesystem\Filesystem; +use Flow\Filesystem\Path; +use Flow\Serializer\NativePHPSerializer; +use Flow\Serializer\Serializer; use Generator; +use function base64_decode; +use function base64_encode; +use function sprintf; + /** * @internal */ -interface BucketRunCache +final readonly class BucketRunCache { + private const int DEFAULT_CHUNK_SIZE = 100; + + private Path $cacheDir; + /** - * @return Generator yields hash => Bucket in ascending-hash order + * @param int<1, max> $chunkSize */ - public function get(string $runId): Generator; + public function __construct( + private Filesystem $filesystem, + private Serializer $serializer = new NativePHPSerializer(), + private int $chunkSize = self::DEFAULT_CHUNK_SIZE, + ?Path $cacheDir = null, + ) { + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->chunkSize < 1) { + throw new InvalidArgumentException('Chunk size must be greater than 0'); + } - public function remove(string $runId): void; + $this->cacheDir = $cacheDir ?? $this->filesystem->getSystemTmpDir()->suffix('/flow-php-group-by/'); + } /** - * @param iterable $buckets hash => Bucket, ascending-hash order + * @return Generator yields Buckets in ascending-hash order */ - public function set(string $runId, iterable $buckets): void; + public function get(string $runId): Generator + { + $path = $this->keyPath($runId); + + if (!$this->filesystem->status($path)) { + return; + } + + $stream = $this->filesystem->readFrom($path); + + try { + foreach ($stream->readLines() as $line) { + if ($line === '') { + continue; + } + + $serializedBucket = base64_decode($line, true); + + if ($serializedBucket === false) { + throw new RuntimeException(sprintf( + 'Failed to decode base64 bucket-cache line for run "%s"', + $runId, + )); + } + + yield $this->serializer->unserialize($serializedBucket, [Bucket::class]); + } + } finally { + $stream->close(); + } + } + + public function remove(string $runId): void + { + $this->filesystem->rm($this->keyPath($runId)->parentDirectory()); + } + + /** + * @param iterable $buckets ascending-hash order + */ + public function set(string $runId, iterable $buckets): void + { + $stream = $this->filesystem->writeTo($this->keyPath($runId)); + + $buffer = ''; + $counter = 0; + + foreach ($buckets as $bucket) { + $buffer .= base64_encode($this->serializer->serialize($bucket)) . "\n"; + $counter++; + + if ($counter >= $this->chunkSize) { + $stream->append($buffer); + $buffer = ''; + $counter = 0; + } + } + + if ($counter > 0) { + $stream->append($buffer); + } + + $stream->close(); + } + + private function keyPath(string $key): Path + { + return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); + } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php b/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php index 97e95b5836..58c77ffecd 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/Buckets.php @@ -16,5 +16,5 @@ public function all(): iterable; public function clear(): void; - public function merge(string $hash, Bucket $bucket): void; + public function merge(Bucket $bucket): void; } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php deleted file mode 100644 index b36d8a4b06..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketsCache/FilesystemBucketRunCache.php +++ /dev/null @@ -1,135 +0,0 @@ - $chunkSize - */ - public function __construct( - private Filesystem $filesystem, - private Serializer $serializer = new NativePHPSerializer(), - private int $chunkSize = self::DEFAULT_CHUNK_SIZE, - ?Path $cacheDir = null, - ) { - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->chunkSize < 1) { - throw new InvalidArgumentException('Chunk size must be greater than 0'); - } - - $this->cacheDir = ($cacheDir ?? $this->filesystem->getSystemTmpDir())->suffix('/flow-php-group-by/'); - } - - /** - * @return Generator - */ - public function get(string $runId): Generator - { - $path = $this->keyPath($runId); - - if (!$this->filesystem->status($path)) { - return; - } - - $stream = $this->filesystem->readFrom($path); - - try { - foreach ($stream->readLines() as $line) { - if ($line === '') { - continue; - } - - $parts = explode(' ', $line, 2); - - if (count($parts) !== 2) { - throw new RuntimeException(sprintf( - 'Corrupted bucket-cache line for run "%s": missing hash/bucket separator', - $runId, - )); - } - - [$hashB64, $bucketB64] = $parts; - - $hash = base64_decode($hashB64, true); - $serializedBucket = base64_decode($bucketB64, true); - - if ($hash === false || $serializedBucket === false) { - throw new RuntimeException(sprintf( - 'Failed to decode base64 bucket-cache line for run "%s"', - $runId, - )); - } - - $bucket = $this->serializer->unserialize($serializedBucket, [Bucket::class]); - - yield $hash => $bucket; - } - } finally { - $stream->close(); - } - } - - public function remove(string $runId): void - { - $this->filesystem->rm($this->keyPath($runId)->parentDirectory()); - } - - public function set(string $runId, iterable $buckets): void - { - $stream = $this->filesystem->writeTo($this->keyPath($runId)); - - $buffer = ''; - $counter = 0; - - foreach ($buckets as $hash => $bucket) { - // @mago-ignore analysis:redundant-cast - numeric-string hashes are coerced to int array keys by PHP - $buffer .= - base64_encode((string) $hash) . ' ' . base64_encode($this->serializer->serialize($bucket)) . "\n"; - $counter++; - - if ($counter >= $this->chunkSize) { - $stream->append($buffer); - $buffer = ''; - $counter = 0; - } - } - - if ($counter > 0) { - $stream->append($buffer); - } - - $stream->close(); - } - - private function keyPath(string $key): Path - { - return $this->cacheDir->suffix(NativePHPHash::xxh128($key) . '/' . $key . '.php.cache'); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php index c016340425..26191e0224 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/BucketsMinHeap.php @@ -10,12 +10,12 @@ /** * @internal * - * @extends SplMinHeap + * @extends SplMinHeap */ final class BucketsMinHeap extends SplMinHeap { /** - * @return BucketRun + * @return RunBucket */ public function extract(): mixed { @@ -24,8 +24,8 @@ public function extract(): mixed public function insert(mixed $value): true { - if (!$value instanceof BucketRun) { - throw new InvalidArgumentException('Value inserted into BucketsMinHeap must be a ' . BucketRun::class); + if (!$value instanceof RunBucket) { + throw new InvalidArgumentException('Value inserted into BucketsMinHeap must be a ' . RunBucket::class); } parent::insert($value); @@ -34,11 +34,11 @@ public function insert(mixed $value): true } /** - * @param BucketRun $value1 - * @param BucketRun $value2 + * @param RunBucket $value1 + * @param RunBucket $value2 */ protected function compare($value1, $value2): int { - return [$value2->hash, $value2->runIndex] <=> [$value1->hash, $value1->runIndex]; + return [$value2->bucket->hash, $value2->runIndex] <=> [$value1->bucket->hash, $value1->runIndex]; } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php index fe47ad6b1b..c76e3625db 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php @@ -59,7 +59,9 @@ public function all(): iterable if ($this->runIds === []) { ksort($this->hot); - yield from $this->hot; + foreach ($this->hot as $bucket) { + yield $bucket; + } return; } @@ -119,12 +121,12 @@ public function clear(): void $this->runIds = []; } - public function merge(string $hash, Bucket $bucket): void + public function merge(Bucket $bucket): void { - if (isset($this->hot[$hash])) { - $this->hot[$hash]->merge($bucket); + if (isset($this->hot[$bucket->hash])) { + $this->hot[$bucket->hash]->merge($bucket); } else { - $this->hot[$hash] = $bucket; + $this->hot[$bucket->hash] = $bucket; } if ($this->consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { @@ -132,31 +134,28 @@ public function merge(string $hash, Bucket $bucket): void } } + /** + * @param Generator $cursor + */ private function advance(Generator $cursor, int $index, BucketsMinHeap $heap): void { if ($cursor->valid()) { - /** @var string $key */ - $key = $cursor->key(); - /** @var Bucket $bucket */ - $bucket = $cursor->current(); - - $heap->insert(new BucketRun($key, $index, $bucket)); + $heap->insert(new RunBucket($index, $cursor->current())); $cursor->next(); } } /** * K-way merge over run cursors, folding equal-hash Buckets via Bucket::merge, yielding merged - * Buckets in ascending hash. Keyed by hash so the result feeds both BucketRunCache::set() - * (which needs hash => Bucket) and value-only consumers. + * Buckets in ascending hash. * * @param list $runIds * - * @return Generator + * @return Generator */ private function mergeCursors(array $runIds): Generator { - /** @var array> $cursors */ + /** @var array> $cursors */ $cursors = []; foreach ($runIds as $i => $runId) { @@ -174,13 +173,13 @@ private function mergeCursors(array $runIds): Generator $merged = $top->bucket; $this->advance($cursors[$top->runIndex], $top->runIndex, $heap); - while (!$heap->isEmpty() && $heap->top()->hash === $top->hash) { + while (!$heap->isEmpty() && $heap->top()->bucket->hash === $merged->hash) { $nextRun = $heap->extract(); $merged->merge($nextRun->bucket); $this->advance($cursors[$nextRun->runIndex], $nextRun->runIndex, $heap); } - yield $top->hash => $merged; + yield $merged; } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php new file mode 100644 index 0000000000..dc1b94dce5 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php @@ -0,0 +1,66 @@ + + */ +final class GroupBatches implements IteratorAggregate +{ + /** + * @param Generator $sortedRows key-ordered by the group references + * @param int<1, max> $batchSize + */ + public function __construct( + private readonly Generator $sortedRows, + private readonly GroupBy $groupBy, + private readonly int $batchSize, + ) {} + + /** + * @return Generator + */ + public function getIterator(): Generator + { + $buffer = []; + $currentHash = null; + + foreach ($this->sortedRows as $rows) { + foreach ($rows as $row) { + $hash = $this->groupBy->keyValues($row)->hash(); + + if ($currentHash !== null && $hash !== $currentHash && $buffer !== []) { + yield new Rows(...$buffer); + $buffer = []; + } + + $currentHash = $hash; + $buffer[] = $row; + + if (count($buffer) >= $this->batchSize) { + yield new Rows(...$buffer); + $buffer = []; + } + } + } + + if ($buffer !== []) { + yield new Rows(...$buffer); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php new file mode 100644 index 0000000000..5083067f2f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php @@ -0,0 +1,84 @@ + + */ +final readonly class GroupKey implements Countable, IteratorAggregate +{ + /** + * @param array $values ref name => key value + */ + public function __construct( + private array $values, + ) {} + + public function count(): int + { + return count($this->values); + } + + public function getIterator(): Traversable + { + return new ArrayIterator($this->values); + } + + public function hash(): string + { + return self::hashValues($this->values); + } + + /** + * @param array $values + */ + private static function hashValues(array $values): string + { + /** @var array $stringValues */ + $stringValues = []; + + /** @var mixed $value */ + foreach ($values as $value) { + if ($value === null) { + $stringValues[] = 'null'; + } elseif (is_scalar($value)) { + $stringValues[] = (string) $value; + } else { + if ($value instanceof Stringable) { + $stringValues[] = $value->__toString(); + } elseif ($value instanceof DateTimeInterface) { + $stringValues[] = $value->format(DateTimeImmutable::ATOM); + } elseif (is_array($value)) { + $stringValues[] = self::hashValues($value); + } else { + $stringValues[] = serialize($value); + } + } + } + + $key = ''; + + foreach ($stringValues as $stringValue) { + $key .= strlen($stringValue) . ':' . $stringValue; + } + + return NativePHPHash::xxh128($key); + } +} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php deleted file mode 100644 index bad835a474..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupRun.php +++ /dev/null @@ -1,23 +0,0 @@ - $keyValues - * @param Generator $rows - */ - public function __construct( - public array $keyValues, - public Generator $rows, - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php deleted file mode 100644 index febb52e91f..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupedRuns.php +++ /dev/null @@ -1,82 +0,0 @@ - $sortedRows key-ordered stream of rows - * - * @return Generator - */ - public function groups(Generator $sortedRows, GroupBy $groupBy): Generator - { - $rows = $this->flatten($sortedRows); - - while ($rows->valid()) { - $keyValues = $groupBy->keyValues($rows->current()); - $hash = $groupBy->hash($keyValues); - - yield new GroupRun($keyValues, $this->groupRows($rows, $groupBy, $hash)); - } - } - - /** - * @param Generator<\Flow\ETL\Rows> $sortedRows - * - * @return Generator - */ - private function flatten(Generator $sortedRows): Generator - { - foreach ($sortedRows as $batch) { - foreach ($batch as $row) { - yield $row; - } - } - } - - /** - * @param Generator $rows - * - * @return Generator - */ - private function groupRows(Generator $rows, GroupBy $groupBy, string $groupHash): Generator - { - while ($rows->valid()) { - /** @var Row $row */ - $row = $rows->current(); - - if ($groupBy->hash($groupBy->keyValues($row)) !== $groupHash) { - return; - } - - yield $row; - $rows->next(); - } - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php index 96ec8618a0..10222039e6 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/InMemoryBuckets.php @@ -16,7 +16,9 @@ final class InMemoryBuckets implements Buckets public function all(): iterable { - yield from $this->buckets; + foreach ($this->buckets as $bucket) { + yield $bucket; + } } public function clear(): void @@ -24,12 +26,12 @@ public function clear(): void $this->buckets = []; } - public function merge(string $hash, Bucket $bucket): void + public function merge(Bucket $bucket): void { - if (isset($this->buckets[$hash])) { - $this->buckets[$hash]->merge($bucket); + if (isset($this->buckets[$bucket->hash])) { + $this->buckets[$bucket->hash]->merge($bucket); } else { - $this->buckets[$hash] = $bucket; + $this->buckets[$bucket->hash] = $bucket; } } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketRun.php b/src/core/etl/src/Flow/ETL/GroupBy/RunBucket.php similarity index 76% rename from src/core/etl/src/Flow/ETL/GroupBy/BucketRun.php rename to src/core/etl/src/Flow/ETL/GroupBy/RunBucket.php index 0ede21993b..8540402931 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketRun.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/RunBucket.php @@ -7,10 +7,9 @@ /** * @internal */ -final readonly class BucketRun +final readonly class RunBucket { public function __construct( - public string $hash, public int $runIndex, public Bucket $bucket, ) {} diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index 7c256608b2..f8ca3039d0 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -5,22 +5,26 @@ namespace Flow\ETL\Processor; use Flow\ETL\Config\Grouping\GroupingBackend; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; use Flow\ETL\GroupBy\BucketAggregation; -use Flow\ETL\GroupBy\BucketsCache\FilesystemBucketRunCache; +use Flow\ETL\GroupBy\BucketRunCache; use Flow\ETL\GroupBy\ExternalBuckets; -use Flow\ETL\GroupBy\GroupedRuns; +use Flow\ETL\GroupBy\GroupBatches; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\GroupBy\InMemoryBuckets; use Flow\ETL\Processor; +use Flow\ETL\Row; use Flow\ETL\Row\EntryReference; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Flow\ETL\Rows; use Flow\ETL\Sort\ExternalSortFactory; -use Flow\ETL\Sort\MemorySort; +use Flow\Filesystem\Path; use Generator; +use function array_chunk; use function array_map; use function bin2hex; use function random_bytes; @@ -32,9 +36,21 @@ */ final readonly class GroupByProcessor implements Processor { + public const int DEFAULT_BATCH_SIZE = 1000; + + /** + * @param int<1, max> $batchSize + */ public function __construct( public GroupBy $groupBy, - ) {} + private int $batchSize = self::DEFAULT_BATCH_SIZE, + ) { + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->batchSize < 1) { + throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->batchSize); + } + } public function process(Generator $rows, FlowContext $context): Generator { @@ -54,14 +70,10 @@ public function process(Generator $rows, FlowContext $context): Generator $buckets = match ($config->backend) { GroupingBackend::Filesystem => new ExternalBuckets( - new FilesystemBucketRunCache( + new BucketRunCache( $context->filesystem($config->filesystemProtocol), $context->config->serializer(), - cacheDir: $context - ->config - ->cache - ->localFilesystemCacheDir - ->suffix('/flow-php-group-by/' . bin2hex(random_bytes(16)) . '/'), + cacheDir: $this->spillDir($context), ), $config->memoryLimit, $config->bucketsCount, @@ -73,13 +85,15 @@ public function process(Generator $rows, FlowContext $context): Generator } /** - * Grouping without aggregation: sort the stream by the group references, then stream the - * consecutive-equal-key runs and re-chunk each run's rows into batches. A new group always - * starts a new batch, so every emitted {@see Rows} belongs to exactly one group. - * - * Note: `groupingMemoryLimit` does NOT bound the Filesystem no-agg spill - {@see ExternalSort} - * spills by row batches (its own `minBatchSize`/`bucketsCount`), not by byte limit. On the - * Memory backend it still caps the in-memory {@see MemorySort} (its OOM threshold). + * Grouping without aggregation. Memory backend: single-pass hash partitioning - every row + * stays resident (O(rows)) and groups are emitted in first-seen order; group columns must + * exist on every row, a missing column throws the same + * {@see \Flow\ETL\Exception\InvalidArgumentException} the sort path throws (a present-but-null + * value is a regular group key). Filesystem backend: {@see \Flow\ETL\Sort\ExternalSort} over + * the group references (bounded memory, spills by row batches - the byte + * `groupingMemoryLimit` does not apply) streamed through {@see GroupBatches}, emitting groups + * in key order. Either way every emitted {@see Rows} belongs to exactly one group and holds + * at most the processor's batch size. * * @param Generator $rows * @@ -87,50 +101,65 @@ public function process(Generator $rows, FlowContext $context): Generator */ private function group(Generator $rows, FlowContext $context): Generator { + $config = $context->config->grouping; + + if ($config->backend === GroupingBackend::Memory) { + yield from $this->hashPartition($rows); + + return; + } + $refs = References::init(...array_map(static fn(Reference $ref): Reference => $ref instanceof EntryReference ? $ref->asc() : $ref, $this->groupBy->references()->all())); - /** @var Generator $sorted */ - $sorted = match ($context->config->grouping->backend) { - GroupingBackend::Filesystem => ExternalSortFactory::create( - $context->filesystem($context->config->grouping->filesystemProtocol), - $context->config->serializer(), - $context - ->config - ->cache - ->localFilesystemCacheDir - ->suffix('/flow-php-group-by/' . bin2hex(random_bytes(16)) . '/'), - $context->config->grouping->bucketsCount, - )->sortGenerator($rows, $context, $refs), - GroupingBackend::Memory => (new MemorySort($context->config->grouping->memoryLimit))->sortGenerator( - $rows, - $context, - $refs, - ), - }; + $sorted = ExternalSortFactory::create( + $context->filesystem($config->filesystemProtocol), + $context->config->serializer(), + $this->spillDir($context), + $config->bucketsCount, + )->sortGenerator($rows, $context, $refs); - $batchSize = $this->groupBy->getBatchSize(); - $buffer = []; - $counter = 0; + yield from new GroupBatches($sorted, $this->groupBy, $this->batchSize); + } + + /** + * @param Generator $rows + * + * @return Generator + */ + private function hashPartition(Generator $rows): Generator + { + $references = $this->groupBy->references()->all(); + + /** @var array> $groups */ + $groups = []; - foreach ((new GroupedRuns())->groups($sorted, $this->groupBy) as $groupRun) { - foreach ($groupRun->rows as $row) { - $buffer[] = $row; - $counter++; + foreach ($rows as $batch) { + foreach ($batch as $row) { + $values = []; - if ($counter >= $batchSize) { - yield new Rows(...$buffer); - $buffer = []; - $counter = 0; + foreach ($references as $reference) { + $values[$reference->name()] = $row->valueOf($reference); } + + $groups[(new GroupKey($values))->hash()][] = $row; } + } - if ($buffer !== []) { - yield new Rows(...$buffer); - $buffer = []; - $counter = 0; + foreach ($groups as $groupRows) { + foreach (array_chunk($groupRows, $this->batchSize) as $chunk) { + yield new Rows(...$chunk); } } } + + private function spillDir(FlowContext $context): Path + { + return $context + ->config + ->cache + ->localFilesystemCacheDir + ->suffix('/flow-php-group-by/' . bin2hex(random_bytes(16)) . '/'); + } } diff --git a/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php b/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php index b3e48d3423..9e780b8e5d 100644 --- a/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php +++ b/src/core/etl/src/Flow/ETL/Sort/ExternalSortFactory.php @@ -13,7 +13,7 @@ /** * @internal */ -final readonly class ExternalSortFactory +final class ExternalSortFactory { /** * @param int<1, max> $bucketsCount diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php new file mode 100644 index 0000000000..a1ec29d124 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php @@ -0,0 +1,113 @@ +fs(), cacheDir: $this->cacheDir); + + $a = new Bucket('aaa', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v')))); + $a->aggregators->first()->aggregate(row(int_entry('v', 2)), $context); + $b = new Bucket('bbb', new GroupKey(['k' => 'b']), new Aggregators(sum(ref('v')))); + $b->aggregators->first()->aggregate(row(int_entry('v', 5)), $context); + + $cache->set('run-1', [$a, $b]); + + $read = []; + $keys = []; + + foreach ($cache->get('run-1') as $bucket) { + $read[$bucket->hash] = $bucket->aggregators->first()->result($context->entryFactory())->value(); + $keys[$bucket->hash] = iterator_to_array($bucket->key); + } + + static::assertSame(['aaa' => 2, 'bbb' => 5], $read); + static::assertSame(['aaa' => ['k' => 'a'], 'bbb' => ['k' => 'b']], $keys); + + $cache->remove('run-1'); + + static::assertSame([], iterator_to_array($cache->get('run-1'), false)); + } + + public function test_invalid_base64_line_raises_runtime_exception(): void + { + $cache = new BucketRunCache($this->fs(), cacheDir: $this->cacheDir); + + $cache->set('run-bad-b64', [new Bucket('aaa', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v'))))]); + + $path = $this->cacheDir->suffix(NativePHPHash::xxh128('run-bad-b64') . '/run-bad-b64.php.cache'); + + // The line is not valid base64. + $this->fs()->writeTo($path)->append('!!!not-base64!!!')->close(); + + static::expectException(RuntimeException::class); + + iterator_to_array($cache->get('run-bad-b64')); + } + + public function test_garbage_serialized_payload_raises_runtime_exception(): void + { + $cache = new BucketRunCache($this->fs(), cacheDir: $this->cacheDir); + + $cache->set('run-garbage', [new Bucket('aaa', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v'))))]); + + $path = $this->cacheDir->suffix(NativePHPHash::xxh128('run-garbage') . '/run-garbage.php.cache'); + + // Valid base64, but the payload does not unserialize into a Bucket. + $this + ->fs() + ->writeTo($path) + ->append(base64_encode(serialize(new stdClass()))) + ->close(); + + static::expectException(RuntimeException::class); + + iterator_to_array($cache->get('run-garbage')); + } + + public function test_preserves_caller_order_across_chunk_flushes(): void + { + $context = flow_context(); + $cache = new BucketRunCache($this->fs(), chunkSize: 1, cacheDir: $this->cacheDir); + + $ccc = new Bucket('ccc', new GroupKey(['k' => 'c']), new Aggregators(sum(ref('v')))); + $ccc->aggregators->first()->aggregate(row(int_entry('v', 3)), $context); + $bbb = new Bucket('bbb', new GroupKey(['k' => 'b']), new Aggregators(sum(ref('v')))); + $bbb->aggregators->first()->aggregate(row(int_entry('v', 2)), $context); + $aaa = new Bucket('aaa', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v')))); + $aaa->aggregators->first()->aggregate(row(int_entry('v', 1)), $context); + + $cache->set('run-2', [$ccc, $bbb, $aaa]); + + $read = []; + + foreach ($cache->get('run-2') as $bucket) { + $read[$bucket->hash] = $bucket->aggregators->first()->result($context->entryFactory())->value(); + } + + static::assertSame(['ccc' => 3, 'bbb' => 2, 'aaa' => 1], $read); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php index 4e82a10116..864c26b4d3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php @@ -6,9 +6,11 @@ use FilesystemIterator; use Flow\ETL\Dataset\Memory\Unit; +use Flow\ETL\GroupBy\Aggregators; use Flow\ETL\GroupBy\Bucket; -use Flow\ETL\GroupBy\BucketsCache\FilesystemBucketRunCache; +use Flow\ETL\GroupBy\BucketRunCache; use Flow\ETL\GroupBy\ExternalBuckets; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\GroupBy\InMemoryBuckets; use Flow\ETL\Tests\FlowIntegrationTestCase; use RecursiveDirectoryIterator; @@ -34,7 +36,7 @@ public function test_parity_with_in_memory_across_forced_spills(): void $context = flow_context(); $memory = new InMemoryBuckets(); $external = new ExternalBuckets( - new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromBytes(1), // force a spill on essentially every merge 2, // small fan-in to exercise recursive merge ); @@ -43,9 +45,9 @@ public function test_parity_with_in_memory_across_forced_spills(): void for ($round = 0; $round < 3; $round++) { for ($g = 0; $g < 50; $g++) { foreach ([$memory, $external] as $store) { - $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); - $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); - $store->merge((string) $g, $bucket); + $bucket = new Bucket((string) $g, new GroupKey(['k' => $g]), new Aggregators(sum(ref('v')))); + $bucket->aggregators->first()->aggregate(row(int_entry('v', $g)), $context); + $store->merge($bucket); } } } @@ -55,7 +57,9 @@ public function test_parity_with_in_memory_across_forced_spills(): void $out = []; foreach ($buckets as $bucket) { - $out[(string) $bucket->keyValues['k']] = $bucket->aggregators[0] + $out[(string) iterator_to_array($bucket->key)['k']] = $bucket + ->aggregators + ->first() ->result($context->entryFactory()) ->value(); } @@ -84,7 +88,7 @@ public function test_parity_with_multi_entry_runs(): void $context = flow_context(); $memory = new InMemoryBuckets(); $external = new ExternalBuckets( - new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), // large enough that a run holds many groups (multi-entry), small enough to spill several times Unit::fromKb(64), 2, // small fan-in forces recursive merge of the multi-entry runs @@ -94,9 +98,9 @@ public function test_parity_with_multi_entry_runs(): void for ($round = 0; $round < 2; $round++) { for ($g = 0; $g < 2000; $g++) { foreach ([$memory, $external] as $store) { - $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); - $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); - $store->merge((string) $g, $bucket); + $bucket = new Bucket((string) $g, new GroupKey(['k' => $g]), new Aggregators(sum(ref('v')))); + $bucket->aggregators->first()->aggregate(row(int_entry('v', $g)), $context); + $store->merge($bucket); } } } @@ -106,7 +110,9 @@ public function test_parity_with_multi_entry_runs(): void $out = []; foreach ($buckets as $bucket) { - $out[(string) $bucket->keyValues['k']] = $bucket->aggregators[0] + $out[(string) iterator_to_array($bucket->key)['k']] = $bucket + ->aggregators + ->first() ->result($context->entryFactory()) ->value(); } @@ -135,15 +141,15 @@ public function test_clear_removes_intermediate_runs_after_abandoned_all(): void { $context = flow_context(); $external = new ExternalBuckets( - new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), + new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromBytes(1), // force a spill on essentially every merge -> many runs 2, // small fan-in -> multi-level cascade ); for ($g = 0; $g < 20; $g++) { - $bucket = new Bucket(['k' => $g], [sum(ref('v'))]); - $bucket->aggregators[0]->aggregate(row(int_entry('v', $g)), $context); - $external->merge((string) $g, $bucket); + $bucket = new Bucket((string) $g, new GroupKey(['k' => $g]), new Aggregators(sum(ref('v')))); + $bucket->aggregators->first()->aggregate(row(int_entry('v', $g)), $context); + $external->merge($bucket); } foreach ($external->all() as $_bucket) { @@ -170,10 +176,7 @@ public function test_clear_removes_intermediate_runs_after_abandoned_all(): void public function test_empty_yields_nothing(): void { - $external = new ExternalBuckets( - new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromMb(64), - ); + $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); static::assertSame([], iterator_to_array($external->all(), false)); } @@ -181,19 +184,16 @@ public function test_empty_yields_nothing(): void public function test_single_group_fast_path(): void { $context = flow_context(); - $external = new ExternalBuckets( - new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromMb(64), // large enough that nothing spills -> exercises the no-runs fast path - ); + $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); // large enough that nothing spills -> exercises the no-runs fast path - $bucket = new Bucket(['k' => 7], [sum(ref('v'))]); - $bucket->aggregators[0]->aggregate(row(int_entry('v', 7)), $context); - $external->merge('7', $bucket); + $bucket = new Bucket('7', new GroupKey(['k' => 7]), new Aggregators(sum(ref('v')))); + $bucket->aggregators->first()->aggregate(row(int_entry('v', 7)), $context); + $external->merge($bucket); $result = iterator_to_array($external->all(), false); static::assertCount(1, $result); - static::assertSame(7, $result[0]->keyValues['k']); - static::assertSame(7, $result[0]->aggregators[0]->result($context->entryFactory())->value()); + static::assertSame(7, iterator_to_array($result[0]->key)['k']); + static::assertSame(7, $result[0]->aggregators->first()->result($context->entryFactory())->value()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php deleted file mode 100644 index 37c140a140..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/FilesystemBucketRunCacheTest.php +++ /dev/null @@ -1,109 +0,0 @@ -fs(), cacheDir: $this->cacheDir); - - $a = new Bucket(['k' => 'a'], [sum(ref('v'))]); - $a->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); - $b = new Bucket(['k' => 'b'], [sum(ref('v'))]); - $b->aggregators[0]->aggregate(row(int_entry('v', 5)), $context); - - $cache->set('run-1', ['aaa' => $a, 'bbb' => $b]); - - $read = []; - - foreach ($cache->get('run-1') as $hash => $bucket) { - $read[$hash] = $bucket->aggregators[0]->result($context->entryFactory())->value(); - } - - static::assertSame(['aaa' => 2, 'bbb' => 5], $read); - - $cache->remove('run-1'); - - static::assertSame([], iterator_to_array($cache->get('run-1'), false)); - } - - public function test_corrupt_run_raises_a_runtime_exception(): void - { - $cache = new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir); - - $cache->set('run-corrupt', ['aaa' => new Bucket(['k' => 'a'], [sum(ref('v'))])]); - - $path = $this->cacheDir->suffix( - 'flow-php-group-by/' . NativePHPHash::xxh128('run-corrupt') . '/run-corrupt.php.cache', - ); - - $this->fs()->writeTo($path)->append('not-a-valid-line')->close(); - - static::expectException(RuntimeException::class); - - iterator_to_array($cache->get('run-corrupt')); - } - - public function test_invalid_base64_payload_raises_runtime_exception(): void - { - $cache = new FilesystemBucketRunCache($this->fs(), cacheDir: $this->cacheDir); - - $cache->set('run-bad-b64', ['aaa' => new Bucket(['k' => 'a'], [sum(ref('v'))])]); - - $path = $this->cacheDir->suffix( - 'flow-php-group-by/' . NativePHPHash::xxh128('run-bad-b64') . '/run-bad-b64.php.cache', - ); - - // Valid hash/bucket separator, but the payload after the space is not valid base64. - $this - ->fs() - ->writeTo($path) - ->append(base64_encode('aaa') . ' !!!not-base64!!!') - ->close(); - - static::expectException(RuntimeException::class); - - iterator_to_array($cache->get('run-bad-b64')); - } - - public function test_preserves_caller_order_across_chunk_flushes(): void - { - $context = flow_context(); - $cache = new FilesystemBucketRunCache($this->fs(), chunkSize: 1, cacheDir: $this->cacheDir); - - $ccc = new Bucket(['k' => 'c'], [sum(ref('v'))]); - $ccc->aggregators[0]->aggregate(row(int_entry('v', 3)), $context); - $bbb = new Bucket(['k' => 'b'], [sum(ref('v'))]); - $bbb->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); - $aaa = new Bucket(['k' => 'a'], [sum(ref('v'))]); - $aaa->aggregators[0]->aggregate(row(int_entry('v', 1)), $context); - - $cache->set('run-2', ['ccc' => $ccc, 'bbb' => $bbb, 'aaa' => $aaa]); - - $read = []; - - foreach ($cache->get('run-2') as $hash => $bucket) { - $read[$hash] = $bucket->aggregators[0]->result($context->entryFactory())->value(); - } - - static::assertSame(['ccc' => 3, 'bbb' => 2, 'aaa' => 1], $read); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php index a2ba6a5a02..40aeea4738 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketTest.php @@ -5,7 +5,9 @@ namespace Flow\ETL\Tests\Unit\GroupBy; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\GroupBy\Aggregators; use Flow\ETL\GroupBy\Bucket; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; @@ -20,23 +22,23 @@ public function test_merge_folds_aligned_aggregators(): void { $context = flow_context(); - $left = new Bucket(['k' => 'a'], [sum(ref('v'))]); - $left->aggregators[0]->aggregate(row(int_entry('v', 2)), $context); + $left = new Bucket('h', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v')))); + $left->aggregators->first()->aggregate(row(int_entry('v', 2)), $context); - $right = new Bucket(['k' => 'a'], [sum(ref('v'))]); - $right->aggregators[0]->aggregate(row(int_entry('v', 3)), $context); + $right = new Bucket('h', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('v')))); + $right->aggregators->first()->aggregate(row(int_entry('v', 3)), $context); $left->merge($right); - static::assertSame(5, $left->aggregators[0]->result($context->entryFactory())->value()); + static::assertSame(5, $left->aggregators->first()->result($context->entryFactory())->value()); } public function test_merge_delegates_validation_to_the_aggregators(): void { $this->expectException(InvalidArgumentException::class); - $left = new Bucket(['k' => 'a'], [sum(ref('a'))]); - $right = new Bucket(['k' => 'a'], [sum(ref('b'))]); + $left = new Bucket('h', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('a')))); + $right = new Bucket('h', new GroupKey(['k' => 'a']), new Aggregators(sum(ref('b')))); $left->merge($right); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php index ec4b307219..efdbc344f6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketsMinHeapTest.php @@ -5,9 +5,11 @@ namespace Flow\ETL\Tests\Unit\GroupBy; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\GroupBy\Aggregators; use Flow\ETL\GroupBy\Bucket; -use Flow\ETL\GroupBy\BucketRun; use Flow\ETL\GroupBy\BucketsMinHeap; +use Flow\ETL\GroupBy\GroupKey; +use Flow\ETL\GroupBy\RunBucket; use Flow\ETL\Tests\FlowTestCase; final class BucketsMinHeapTest extends FlowTestCase @@ -15,21 +17,21 @@ final class BucketsMinHeapTest extends FlowTestCase public function test_orders_by_hash_then_run_index(): void { $heap = new BucketsMinHeap(); - $heap->insert(new BucketRun('b', 0, new Bucket([], []))); - $heap->insert(new BucketRun('a', 2, new Bucket([], []))); - $heap->insert(new BucketRun('a', 1, new Bucket([], []))); + $heap->insert(new RunBucket(0, new Bucket('b', new GroupKey([]), new Aggregators()))); + $heap->insert(new RunBucket(2, new Bucket('a', new GroupKey([]), new Aggregators()))); + $heap->insert(new RunBucket(1, new Bucket('a', new GroupKey([]), new Aggregators()))); $order = []; while (!$heap->isEmpty()) { $run = $heap->extract(); - $order[] = $run->hash . $run->runIndex; + $order[] = $run->bucket->hash . $run->runIndex; } static::assertSame(['a1', 'a2', 'b0'], $order); } - public function test_rejects_non_bucket_run(): void + public function test_rejects_non_run_bucket(): void { $this->expectException(InvalidArgumentException::class); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php new file mode 100644 index 0000000000..232061b2af --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php @@ -0,0 +1,99 @@ +valueOf(ref('k')); + + if (is_string($key)) { + $keys[$key] = true; + } + + $values[] = $row->valueOf(ref('v')); + } + + static::assertCount(1, $keys); + + $emitted[] = [array_key_first($keys), $values]; + } + + static::assertSame([['a', [1, 2, 3]], ['b', [4]], ['c', [5, 6]]], $emitted); + } + + public function test_splits_a_group_larger_than_batch_size_into_consecutive_chunks(): void + { + $groupBy = new GroupBy(ref('k')); + + // key-ordered: a group of five "a" rows, then one "b" row + $sorted = (static function () { + yield rows( + row(str_entry('k', 'a'), int_entry('v', 1)), + row(str_entry('k', 'a'), int_entry('v', 2)), + row(str_entry('k', 'a'), int_entry('v', 3)), + row(str_entry('k', 'a'), int_entry('v', 4)), + row(str_entry('k', 'a'), int_entry('v', 5)), + row(str_entry('k', 'b'), int_entry('v', 6)), + ); + })(); + + $emitted = []; + + foreach (new GroupBatches($sorted, $groupBy, 2) as $rows) { + $keys = []; + $values = []; + + foreach ($rows as $row) { + $key = $row->valueOf(ref('k')); + + if (is_string($key)) { + $keys[$key] = true; + } + + $values[] = $row->valueOf(ref('v')); + } + + static::assertCount(1, $keys); + + $emitted[] = [array_key_first($keys), $values]; + } + + static::assertSame([['a', [1, 2]], ['a', [3, 4]], ['a', [5]], ['b', [6]]], $emitted); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php deleted file mode 100644 index cea646f9d3..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupedRunsTest.php +++ /dev/null @@ -1,47 +0,0 @@ -groups($sorted, $groupBy) as $run) { - $result[(string) $run->keyValues['k']] = array_map( - static fn($r) => $r->valueOf(ref('v')), - iterator_to_array($run->rows, false), - ); - } - - static::assertSame(['a' => [1, 2, 3], 'b' => [4], 'c' => [5, 6]], $result); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php index 37361aad1b..9bc652fbf4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/InMemoryBucketsTest.php @@ -4,7 +4,9 @@ namespace Flow\ETL\Tests\Unit\GroupBy; +use Flow\ETL\GroupBy\Aggregators; use Flow\ETL\GroupBy\Bucket; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\GroupBy\InMemoryBuckets; use Flow\ETL\Tests\FlowTestCase; @@ -22,25 +24,25 @@ public function test_merge_folds_a_repeated_hash(): void $context = flow_context(); $buckets = new InMemoryBuckets(); - $a = new Bucket(['k' => 'x'], [sum(ref('v'))]); - $a->aggregators[0]->aggregate(row(int_entry('v', 1)), $context); - $buckets->merge('h1', $a); + $a = new Bucket('h1', new GroupKey(['k' => 'x']), new Aggregators(sum(ref('v')))); + $a->aggregators->first()->aggregate(row(int_entry('v', 1)), $context); + $buckets->merge($a); - $b = new Bucket(['k' => 'x'], [sum(ref('v'))]); - $b->aggregators[0]->aggregate(row(int_entry('v', 4)), $context); - $buckets->merge('h1', $b); + $b = new Bucket('h1', new GroupKey(['k' => 'x']), new Aggregators(sum(ref('v')))); + $b->aggregators->first()->aggregate(row(int_entry('v', 4)), $context); + $buckets->merge($b); $all = iterator_to_array($buckets->all(), false); static::assertCount(1, $all); - static::assertSame(5, $all[0]->aggregators[0]->result($context->entryFactory())->value()); + static::assertSame(5, $all[0]->aggregators->first()->result($context->entryFactory())->value()); } public function test_distinct_hashes_stay_separate_and_clear_empties(): void { $buckets = new InMemoryBuckets(); - $buckets->merge('h1', new Bucket(['k' => 'x'], [])); - $buckets->merge('h2', new Bucket(['k' => 'y'], [])); + $buckets->merge(new Bucket('h1', new GroupKey(['k' => 'x']), new Aggregators())); + $buckets->merge(new Bucket('h2', new GroupKey(['k' => 'y']), new Aggregators())); static::assertCount(2, iterator_to_array($buckets->all(), false)); From 201dcd9fc02842b48def6d50b570026c75bab275 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Wed, 8 Jul 2026 22:17:26 +0200 Subject: [PATCH 11/13] style(flow-php/etl): drop redundant comments from grouping code --- .../src/Flow/ETL/GroupBy/ExternalBuckets.php | 16 --------- .../etl/src/Flow/ETL/GroupBy/GroupBatches.php | 4 --- .../Flow/ETL/Processor/GroupByProcessor.php | 10 ------ .../Integration/DataFrame/GetGroupsTest.php | 9 ----- .../DataFrame/GroupByAggregationTest.php | 8 ----- .../DataFrame/GroupByScaleTest.php | 2 -- .../GroupBy/BucketRunCacheTest.php | 2 -- .../GroupBy/ExternalBucketsTest.php | 34 +++++-------------- .../Tests/Unit/GroupBy/GroupBatchesTest.php | 2 -- 9 files changed, 8 insertions(+), 79 deletions(-) diff --git a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php index c76e3625db..06eb4e686a 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php @@ -18,10 +18,6 @@ /** * @internal - * - * Spilling map-side hash aggregator: folds partial states into a hot map, spills sorted runs of - * partial-state Buckets when the memory limit is exceeded, then performs a bounded-fan-in k-way - * merge (mirrors Sort\ExternalSort) folding equal-hash Buckets across runs via Bucket::merge. */ final class ExternalBuckets implements Buckets { @@ -70,17 +66,8 @@ public function all(): iterable $this->spillRun(); } - // A fan-in of 1 would chunk runs into size-1 groups that never shrink the run count, so the - // recursive merge could loop forever; clamp to at least 2 to guarantee progress. $fanIn = max(2, $this->bucketsCount); - // $this->runIds is kept as the LIVE set of on-disk runs at every point below: each level - // creates all of its merged runs first, only then removes the consumed inputs, and only - // then repoints $this->runIds at the new level. This loop body is fully synchronous (no - // yield inside it), so it can never be interrupted mid-cascade by an abandoned consumer — - // if the caller abandons `all()` it can only happen in the final merge/yield loop below, - // at which point $this->runIds already reflects exactly the runs still on disk, so - // clear() can safely remove them. while (count($this->runIds) > $this->bucketsCount) { $consumedRunIds = $this->runIds; $newRunIds = []; @@ -146,9 +133,6 @@ private function advance(Generator $cursor, int $index, BucketsMinHeap $heap): v } /** - * K-way merge over run cursors, folding equal-hash Buckets via Bucket::merge, yielding merged - * Buckets in ascending hash. - * * @param list $runIds * * @return Generator diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php index dc1b94dce5..28e79dc8bd 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBatches.php @@ -12,10 +12,6 @@ use function count; /** - * Cuts a key-ordered stream of rows into batches of <= $batchSize where every batch belongs to - * exactly one group - a new group always starts a new batch, and a group larger than $batchSize - * spans consecutive batches. Single-pass: wraps the sorted cursor, iterate exactly once. - * * @internal * * @implements IteratorAggregate diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index f8ca3039d0..c2c6b3a583 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -85,16 +85,6 @@ public function process(Generator $rows, FlowContext $context): Generator } /** - * Grouping without aggregation. Memory backend: single-pass hash partitioning - every row - * stays resident (O(rows)) and groups are emitted in first-seen order; group columns must - * exist on every row, a missing column throws the same - * {@see \Flow\ETL\Exception\InvalidArgumentException} the sort path throws (a present-but-null - * value is a regular group key). Filesystem backend: {@see \Flow\ETL\Sort\ExternalSort} over - * the group references (bounded memory, spills by row batches - the byte - * `groupingMemoryLimit` does not apply) streamed through {@see GroupBatches}, emitting groups - * in key order. Either way every emitted {@see Rows} belongs to exactly one group and holds - * at most the processor's batch size. - * * @param Generator $rows * * @return Generator diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php index 8869c8cf13..db38272a77 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GetGroupsTest.php @@ -49,7 +49,6 @@ public function test_get_groups_memory_and_filesystem_emit_the_same_groups(): vo $groups = []; foreach (data_frame($config)->read(from_array($input))->groupBy(ref('k'))->getGroups()->get() as $rows) { - // every emitted batch belongs to exactly one group $keys = []; foreach ($rows as $row) { @@ -95,7 +94,6 @@ public function test_get_groups_splits_a_large_group_across_batches(): void $batchSizes = []; foreach (data_frame($config)->read(from_array($input))->groupBy(ref('k'))->getGroups(2)->get() as $rows) { - // every emitted batch belongs to exactly one group $keys = []; foreach ($rows as $row) { @@ -130,8 +128,6 @@ public function test_get_groups_splits_a_large_group_across_batches(): void public function test_get_groups_multi_column_key(): void { - // ('x','yz') and ('xy','z') would collide under a naive concatenation-based hash (both - // produce "xyz"); GroupBy::hash() length-prefixes each value to tell them apart. $input = [ ['a' => 'x', 'b' => 'yz', 'v' => 1], ['a' => 'xy', 'b' => 'z', 'v' => 2], @@ -173,8 +169,6 @@ public function test_get_groups_multi_column_key(): void return $groups; }; - // ksort() on the runner's $groups keys orders 'xy|z' before 'x|yz' (the '|' byte sorts - // after 'y'), so the expectation must match that key order for assertSame(). $expected = ['xy|z' => 1, 'x|yz' => 2]; static::assertCount(2, $expected); @@ -225,9 +219,6 @@ public function test_get_groups_empty_input_yields_nothing(): void public function test_get_groups_filesystem_multi_bucket_matches_memory(): void { - // 1500 rows over 3 keys, evenly interleaved: the external sort's minBatchSize (500) is - // exceeded, so createBucketsFromGenerator() spills 3 sorted buckets (500 rows each) that - // must go through a real k-way merge in Buckets::sort() rather than a single in-memory sort. $rowCount = 1500; $keys = ['a', 'b', 'c']; $keysCount = count($keys); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index d0eb49b2de..31a299b8cc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -118,7 +118,6 @@ public function test_chained_filesystem_group_by_stages_do_not_corrupt_each_othe ->getEachAsArray(), ); - // 6 distinct sellers -> 6 first-stage groups; second stage counts 3 north + 3 south $byRegion = []; foreach ($result as $row) { @@ -172,9 +171,6 @@ public function test_collect_matches_between_backends_through_spill(): void $memory = $pipeline(config_builder()); $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); - // The k-way merge folds partial collections in run-creation order, which for this - // single-extractor pipeline matches input order, so the raw arrays come out identical - // between backends without needing an order-insensitive comparison. static::assertSame($memory, $filesystem); $byKey = []; @@ -219,10 +215,6 @@ public function test_average_matches_between_backends_through_spill(): void $memory = $pipeline(config_builder()); $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); - // Each row lands in its own single-value partial Bucket under a byte-tiny memory limit, so - // the k-way merge must fold sum+count across (at least) two partial states per group. If the - // merge instead averaged partial averages (average-of-averages) rather than summing the two - // fields, a differently-weighted fold would drift from the true value. static::assertSame($memory, $filesystem); $byKey = []; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php index 0c6a945b8a..d3e6015dbc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php @@ -48,8 +48,6 @@ public function test_memory_and_filesystem_aggregation_match_at_100k(): void static::assertSame($memory, $filesystem); - // Benchmarked at ~2s; 30s is a generous, non-flaky ceiling that still catches a regression - // toward the old super-linear spill cliff (~80s). static::assertLessThan( 30.0, $elapsed, diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php index a1ec29d124..ce730ced00 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/BucketRunCacheTest.php @@ -60,7 +60,6 @@ public function test_invalid_base64_line_raises_runtime_exception(): void $path = $this->cacheDir->suffix(NativePHPHash::xxh128('run-bad-b64') . '/run-bad-b64.php.cache'); - // The line is not valid base64. $this->fs()->writeTo($path)->append('!!!not-base64!!!')->close(); static::expectException(RuntimeException::class); @@ -76,7 +75,6 @@ public function test_garbage_serialized_payload_raises_runtime_exception(): void $path = $this->cacheDir->suffix(NativePHPHash::xxh128('run-garbage') . '/run-garbage.php.cache'); - // Valid base64, but the payload does not unserialize into a Bucket. $this ->fs() ->writeTo($path) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php index 864c26b4d3..449ed8892e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php @@ -27,21 +27,16 @@ final class ExternalBucketsTest extends FlowIntegrationTestCase { - /** - * Drives many groups through both stores with a tiny memory limit to FORCE multiple spills, - * and asserts the external result equals the in-memory result (parity + fold-across-runs). - */ public function test_parity_with_in_memory_across_forced_spills(): void { $context = flow_context(); $memory = new InMemoryBuckets(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromBytes(1), // force a spill on essentially every merge - 2, // small fan-in to exercise recursive merge + Unit::fromBytes(1), + 2, ); - // 50 groups, each seen 3 times (so a group is folded across runs) for ($round = 0; $round < 3; $round++) { for ($g = 0; $g < 50; $g++) { foreach ([$memory, $external] as $store) { @@ -69,7 +64,7 @@ public function test_parity_with_in_memory_across_forced_spills(): void return $out; }; - $expected = $normalize($memory->all()); // each group summed 3x its own value + $expected = $normalize($memory->all()); $actual = $normalize($external->all()); static::assertSame($expected, $actual); @@ -78,23 +73,16 @@ public function test_parity_with_in_memory_across_forced_spills(): void $external->clear(); } - /** - * Tunes the memory limit so that each spilled run holds MANY ascending-hash buckets (not one), - * then folds every group across several of those multi-entry runs. Exercises the subtlest - * property: a k-way merge that must fold equal hashes drawn from different, many-entry runs. - */ public function test_parity_with_multi_entry_runs(): void { $context = flow_context(); $memory = new InMemoryBuckets(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - // large enough that a run holds many groups (multi-entry), small enough to spill several times Unit::fromKb(64), - 2, // small fan-in forces recursive merge of the multi-entry runs + 2, ); - // 2000 groups, each seen in 2 rounds (so a group is folded across several multi-entry runs) for ($round = 0; $round < 2; $round++) { for ($g = 0; $g < 2000; $g++) { foreach ([$memory, $external] as $store) { @@ -122,7 +110,7 @@ public function test_parity_with_multi_entry_runs(): void return $out; }; - $expected = $normalize($memory->all()); // each group summed to 2x its own value + $expected = $normalize($memory->all()); $actual = $normalize($external->all()); static::assertSame($expected, $actual); @@ -131,19 +119,13 @@ public function test_parity_with_multi_entry_runs(): void $external->clear(); } - /** - * Regression test for a temp-file leak: abandoning `all()` mid-yield used to leave the - * final-level intermediate merged runs orphaned on disk, because the cascade tracked the - * live run ids in a LOCAL variable instead of $this->runIds, so clear() only ever removed - * the (already deleted) original runs. - */ public function test_clear_removes_intermediate_runs_after_abandoned_all(): void { $context = flow_context(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromBytes(1), // force a spill on essentially every merge -> many runs - 2, // small fan-in -> multi-level cascade + Unit::fromBytes(1), + 2, ); for ($g = 0; $g < 20; $g++) { @@ -184,7 +166,7 @@ public function test_empty_yields_nothing(): void public function test_single_group_fast_path(): void { $context = flow_context(); - $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); // large enough that nothing spills -> exercises the no-runs fast path + $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); $bucket = new Bucket('7', new GroupKey(['k' => 7]), new Aggregators(sum(ref('v')))); $bucket->aggregators->first()->aggregate(row(int_entry('v', 7)), $context); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php index 232061b2af..737cc8aab8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupBatchesTest.php @@ -22,7 +22,6 @@ public function test_cuts_key_ordered_stream_into_single_group_batches(): void { $groupBy = new GroupBy(ref('k')); - // already key-ordered: a,a | a,b,c,c - group "a" spans the two input Rows batches $sorted = (static function () { yield rows(row(str_entry('k', 'a'), int_entry('v', 1)), row(str_entry('k', 'a'), int_entry('v', 2))); yield rows( @@ -61,7 +60,6 @@ public function test_splits_a_group_larger_than_batch_size_into_consecutive_chun { $groupBy = new GroupBy(ref('k')); - // key-ordered: a group of five "a" rows, then one "b" row $sorted = (static function () { yield rows( row(str_entry('k', 'a'), int_entry('v', 1)), From cf0b784d04ccd62443edc72280df557e3b65afd2 Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Wed, 8 Jul 2026 22:47:58 +0200 Subject: [PATCH 12/13] refactor(flow-php/etl): deterministic count-based run size for external grouping Replace the byte-based grouping memory limit (env var, ini percentage, Consumption sampling) with a count-based run size: ExternalBuckets spills its hot map once it holds runSize groups, mirroring ExternalSort's count-based runs. Spills are now reproducible functions of the data, and all grouping memory knobs are plain sizes (batch size, run size, fan-in). --- .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 7 ++-- .../ETL/Config/Grouping/GroupingConfig.php | 7 ++-- .../Config/Grouping/GroupingConfigBuilder.php | 36 +++++-------------- .../src/Flow/ETL/GroupBy/ExternalBuckets.php | 22 +++++++----- .../Flow/ETL/Processor/GroupByProcessor.php | 2 +- .../DataFrame/GroupByAggregationTest.php | 15 ++++---- .../DataFrame/GroupByScaleTest.php | 3 +- .../GroupBy/ExternalBucketsTest.php | 17 +++++---- .../Grouping/GroupingConfigBuilderTest.php | 10 +++--- 9 files changed, 50 insertions(+), 69 deletions(-) diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index e71698da24..a52adcaa1d 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -171,9 +171,12 @@ public function groupingFilesystem(string $protocol = 'file'): self return $this; } - public function groupingMemoryLimit(Unit $unit): self + /** + * @param int<1, max> $size + */ + public function groupingRunSize(int $size): self { - $this->grouping->memoryLimit($unit); + $this->grouping->runSize($size); return $this; } diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfig.php b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfig.php index f04c14a08a..11f144617d 100644 --- a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfig.php @@ -4,18 +4,15 @@ namespace Flow\ETL\Config\Grouping; -use Flow\ETL\Dataset\Memory\Unit; - final readonly class GroupingConfig { - public const string GROUPING_MAX_MEMORY_ENV = 'FLOW_GROUPING_MAX_MEMORY'; - /** + * @param int<1, max> $runSize * @param int<1, max> $bucketsCount */ public function __construct( public GroupingBackend $backend, - public Unit $memoryLimit, + public int $runSize = 10_000, public int $bucketsCount = 10, public string $filesystemProtocol = 'file', ) {} diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php index 2dbadd615d..e9cd4215d9 100644 --- a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupingConfigBuilder.php @@ -4,17 +4,8 @@ namespace Flow\ETL\Config\Grouping; -use Flow\ETL\Dataset\Memory\Unit; - -use function getenv; -use function ini_get; - -use const PHP_INT_MAX; - final class GroupingConfigBuilder { - private const int DEFAULT_GROUPING_MEMORY_PERCENTAGE = 70; - private GroupingBackend $backend = GroupingBackend::Memory; /** @var int<1, max> */ @@ -22,7 +13,8 @@ final class GroupingConfigBuilder private string $filesystemProtocol = 'file'; - private ?Unit $memoryLimit = null; + /** @var int<1, max> */ + private int $runSize = 10_000; public function backend(GroupingBackend $backend): self { @@ -43,22 +35,7 @@ public function bucketsCount(int $bucketsCount): self public function build(): GroupingConfig { - if ($this->memoryLimit === null) { - $env = getenv(GroupingConfig::GROUPING_MAX_MEMORY_ENV); - - if ($env !== false) { - $this->memoryLimit = Unit::fromString($env); - } else { - $iniLimit = ini_get('memory_limit'); - - $this->memoryLimit = - $iniLimit === false || $iniLimit === '-1' - ? Unit::fromBytes(PHP_INT_MAX) - : Unit::fromString($iniLimit)->percentage(self::DEFAULT_GROUPING_MEMORY_PERCENTAGE); - } - } - - return new GroupingConfig($this->backend, $this->memoryLimit, $this->bucketsCount, $this->filesystemProtocol); + return new GroupingConfig($this->backend, $this->runSize, $this->bucketsCount, $this->filesystemProtocol); } public function filesystemProtocol(string $protocol): self @@ -69,9 +46,12 @@ public function filesystemProtocol(string $protocol): self return $this; } - public function memoryLimit(Unit $memoryLimit): self + /** + * @param int<1, max> $runSize + */ + public function runSize(int $runSize): self { - $this->memoryLimit = $memoryLimit; + $this->runSize = $runSize; return $this; } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php index 06eb4e686a..70d7a99239 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/ExternalBuckets.php @@ -4,8 +4,6 @@ namespace Flow\ETL\GroupBy; -use Flow\ETL\Dataset\Memory\Consumption; -use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\InvalidArgumentException; use Generator; @@ -21,7 +19,9 @@ */ final class ExternalBuckets implements Buckets { - private Consumption $consumption; + public const int DEFAULT_BUCKETS_COUNT = 10; + + public const int DEFAULT_RUN_SIZE = 10_000; /** * @var array @@ -34,20 +34,25 @@ final class ExternalBuckets implements Buckets private array $runIds = []; /** + * @param int<1, max> $runSize * @param int<1, max> $bucketsCount */ public function __construct( private readonly BucketRunCache $cache, - private readonly Unit $memoryLimit, - private readonly int $bucketsCount = 10, + private readonly int $runSize = self::DEFAULT_RUN_SIZE, + private readonly int $bucketsCount = self::DEFAULT_BUCKETS_COUNT, ) { + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->runSize < 1) { + throw new InvalidArgumentException('Run size must be greater than 0, given: ' . $this->runSize); + } + // @mago-ignore analysis:invalid-operand // @mago-ignore analysis:impossible-condition,redundant-comparison if ($this->bucketsCount < 1) { throw new InvalidArgumentException('Buckets count must be greater than 0, given: ' . $this->bucketsCount); } - - $this->consumption = new Consumption(false); } public function all(): iterable @@ -116,7 +121,7 @@ public function merge(Bucket $bucket): void $this->hot[$bucket->hash] = $bucket; } - if ($this->consumption->currentDiff()->isGreaterThan($this->memoryLimit)) { + if (count($this->hot) >= $this->runSize) { $this->spillRun(); } } @@ -175,6 +180,5 @@ private function spillRun(): void $this->cache->set($runId, $this->hot); $this->runIds[] = $runId; $this->hot = []; - $this->consumption = new Consumption(false); } } diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php index c2c6b3a583..1f7d6dc0ad 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByProcessor.php @@ -75,7 +75,7 @@ public function process(Generator $rows, FlowContext $context): Generator $context->config->serializer(), cacheDir: $this->spillDir($context), ), - $config->memoryLimit, + $config->runSize, $config->bucketsCount, ), GroupingBackend::Memory => new InMemoryBuckets(), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php index 31a299b8cc..90b4bd9337 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByAggregationTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use Flow\ETL\Config\ConfigBuilder; -use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -46,7 +45,7 @@ public function test_memory_and_filesystem_backends_produce_identical_results(): ); $memory = $pipeline(config_builder()); - $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingRunSize(1)); static::assertSame($memory, $filesystem); static::assertCount(3, $memory); @@ -76,7 +75,7 @@ public function test_multi_column_key_does_not_collide(): void ]; $result = iterator_to_array( - data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + data_frame(config_builder()->groupingFilesystem('file')->groupingRunSize(1)) ->read(from_array($input)) ->groupBy(ref('a'), ref('b')) ->aggregate(sum(ref('v'))) @@ -108,7 +107,7 @@ public function test_chained_filesystem_group_by_stages_do_not_corrupt_each_othe ]; $result = iterator_to_array( - data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + data_frame(config_builder()->groupingFilesystem('file')->groupingRunSize(1)) ->read(from_array($input)) ->groupBy(ref('seller'), ref('region')) ->aggregate(count(ref('seller'))) @@ -136,7 +135,7 @@ public function test_first_and_last_survive_the_filesystem_round_trip(): void ]; $result = iterator_to_array( - data_frame(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))) + data_frame(config_builder()->groupingFilesystem('file')->groupingRunSize(1)) ->read(from_array($input)) ->groupBy(ref('k')) ->aggregate(first(ref('v')), last(ref('v'))) @@ -169,7 +168,7 @@ public function test_collect_matches_between_backends_through_spill(): void ); $memory = $pipeline(config_builder()); - $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingRunSize(1)); static::assertSame($memory, $filesystem); @@ -213,7 +212,7 @@ public function test_average_matches_between_backends_through_spill(): void ); $memory = $pipeline(config_builder()); - $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingRunSize(1)); static::assertSame($memory, $filesystem); @@ -246,7 +245,7 @@ public function test_backends_match_on_a_realistic_dataset(): void static::assertSame( $pipeline(config_builder()), - $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromBytes(1))), + $pipeline(config_builder()->groupingFilesystem('file')->groupingRunSize(1)), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php index d3e6015dbc..790d0ffaa9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use Flow\ETL\Config\ConfigBuilder; -use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -43,7 +42,7 @@ public function test_memory_and_filesystem_aggregation_match_at_100k(): void $memory = $pipeline(config_builder()); $start = microtime(true); - $filesystem = $pipeline(config_builder()->groupingFilesystem('file')->groupingMemoryLimit(Unit::fromMb(4))); + $filesystem = $pipeline(config_builder()->groupingFilesystem('file')); $elapsed = microtime(true) - $start; static::assertSame($memory, $filesystem); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php index 449ed8892e..624a9410b7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/GroupBy/ExternalBucketsTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Integration\GroupBy; use FilesystemIterator; -use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\GroupBy\Aggregators; use Flow\ETL\GroupBy\Bucket; use Flow\ETL\GroupBy\BucketRunCache; @@ -33,8 +32,8 @@ public function test_parity_with_in_memory_across_forced_spills(): void $memory = new InMemoryBuckets(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromBytes(1), - 2, + runSize: 1, + bucketsCount: 2, ); for ($round = 0; $round < 3; $round++) { @@ -79,8 +78,8 @@ public function test_parity_with_multi_entry_runs(): void $memory = new InMemoryBuckets(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromKb(64), - 2, + runSize: 100, + bucketsCount: 2, ); for ($round = 0; $round < 2; $round++) { @@ -124,8 +123,8 @@ public function test_clear_removes_intermediate_runs_after_abandoned_all(): void $context = flow_context(); $external = new ExternalBuckets( new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), - Unit::fromBytes(1), - 2, + runSize: 1, + bucketsCount: 2, ); for ($g = 0; $g < 20; $g++) { @@ -158,7 +157,7 @@ public function test_clear_removes_intermediate_runs_after_abandoned_all(): void public function test_empty_yields_nothing(): void { - $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); + $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir)); static::assertSame([], iterator_to_array($external->all(), false)); } @@ -166,7 +165,7 @@ public function test_empty_yields_nothing(): void public function test_single_group_fast_path(): void { $context = flow_context(); - $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir), Unit::fromMb(64)); + $external = new ExternalBuckets(new BucketRunCache($this->fs(), cacheDir: $this->cacheDir)); $bucket = new Bucket('7', new GroupKey(['k' => 7]), new Aggregators(sum(ref('v')))); $bucket->aggregators->first()->aggregate(row(int_entry('v', 7)), $context); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php index 856f51e553..eba502c589 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/GroupingConfigBuilderTest.php @@ -6,7 +6,6 @@ use Flow\ETL\Config\Grouping\GroupingBackend; use Flow\ETL\Config\Grouping\GroupingConfigBuilder; -use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\FlowTestCase; final class GroupingConfigBuilderTest extends FlowTestCase @@ -16,8 +15,9 @@ public function test_default_backend_is_memory(): void $config = (new GroupingConfigBuilder())->build(); static::assertSame(GroupingBackend::Memory, $config->backend); - static::assertSame('file', $config->filesystemProtocol); + static::assertSame(10_000, $config->runSize); static::assertSame(10, $config->bucketsCount); + static::assertSame('file', $config->filesystemProtocol); } public function test_filesystem_protocol_flips_backend(): void @@ -30,14 +30,14 @@ public function test_filesystem_protocol_flips_backend(): void static::assertSame('local', $config->filesystemProtocol); } - public function test_memory_limit_and_buckets_count_are_set(): void + public function test_run_size_and_buckets_count_are_set(): void { $config = (new GroupingConfigBuilder()) - ->memoryLimit(Unit::fromMb(16)) + ->runSize(500) ->bucketsCount(4) ->build(); - static::assertSame(Unit::fromMb(16)->inBytes(), $config->memoryLimit->inBytes()); + static::assertSame(500, $config->runSize); static::assertSame(4, $config->bucketsCount); } } From 9293fee5e49da971482c9e5d30d9cdf7c1d7054b Mon Sep 17 00:00:00 2001 From: Aleksander Kowalski Date: Wed, 8 Jul 2026 23:07:23 +0200 Subject: [PATCH 13/13] fix(flow-php/etl): regenerate api docs and shrink the 100k scale test footprint The scale test materialized full fake orders (nested address/items/notes) only to project three columns, inflating the PHPUnit process past the memory limits that Monitoring\Memory\ConfigurationTest sets later in the same run. Project the columns straight off the generator instead. api.json regenerated via just docs for the new grouping API. --- .../Integration/DataFrame/GroupByScaleTest.php | 17 +++++++++-------- web/landing/resources/api.json | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php index 790d0ffaa9..942747550d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByScaleTest.php @@ -8,7 +8,6 @@ use Flow\ETL\Tests\Double\FakeRandomOrdersExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; -use function array_map; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\data_frame; @@ -22,13 +21,15 @@ final class GroupByScaleTest extends FlowIntegrationTestCase { public function test_memory_and_filesystem_aggregation_match_at_100k(): void { - $raw = iterator_to_array((new FakeRandomOrdersExtractor(100_000))->rawData(), false); - - $data = array_map(static fn(array $order): array => [ - 'seller_id' => $order['seller_id'], - 'email' => $order['email'], - 'discount' => $order['discount'], - ], $raw); + $data = []; + + foreach ((new FakeRandomOrdersExtractor(100_000))->rawData() as $order) { + $data[] = [ + 'seller_id' => $order['seller_id'], + 'email' => $order['email'], + 'discount' => $order['discount'], + ]; + } $pipeline = static fn(ConfigBuilder $config): array => iterator_to_array( data_frame($config) diff --git a/web/landing/resources/api.json b/web/landing/resources/api.json index 3c599818cb..d6913c05e9 100644 --- a/web/landing/resources/api.json +++ b/web/landing/resources/api.json @@ -1 +1 @@ -[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":25,"slug":"and","name":"and","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":30,"slug":"andnot","name":"andNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":35,"slug":"append","name":"append","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Append","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":48,"slug":"arrayfilter","name":"arrayFilter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayFilter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IHJlbW92aW5nIGFsbCBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgIC0ganNvbgogICAgICogICAgLSBsaXN0CiAgICAgKiAgICAtIG1hcAogICAgICogICAgLSBzdHJ1Y3R1cmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":53,"slug":"arrayget","name":"arrayGet","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":61,"slug":"arraygetcollection","name":"arrayGetCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGtleXMKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":66,"slug":"arraygetcollectionfirst","name":"arrayGetCollectionFirst","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":79,"slug":"arraykeep","name":"arrayKeep","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeep","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IGtlZXBpbmcgb25seSBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgLSBqc29uCiAgICAgKiAgIC0gbGlzdAogICAgICogICAtIG1hcAogICAgICogICAtIHN0cnVjdHVyZS4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":92,"slug":"arraykeys","name":"arrayKeys","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayKeys","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCBrZXlzIGZyb20gYW4gYXJyYXksIGlnbm9yaW5nIHRoZSB2YWx1ZXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":100,"slug":"arraymerge","name":"arrayMerge","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHJlZgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":105,"slug":"arraymergecollection","name":"arrayMergeCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":110,"slug":"arraypathexists","name":"arrayPathExists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":115,"slug":"arrayreverse","name":"arrayReverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"preserveKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":120,"slug":"arraysort","name":"arraySort","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"sortFunction","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":136,"slug":"arrayvalues","name":"arrayValues","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayValues","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCB2YWx1ZXMgZnJvbSBhbiBhcnJheSwgaWdub3JpbmcgdGhlIGtleXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":141,"slug":"ascii","name":"ascii","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Ascii","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":151,"slug":"between","name":"between","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"lowerBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upperBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gbWl4ZWR8U2NhbGFyRnVuY3Rpb24gJGxvd2VyQm91bmRSZWYKICAgICAqIEBwYXJhbSBtaXhlZHxTY2FsYXJGdW5jdGlvbiAkdXBwZXJCb3VuZFJlZgogICAgICogQHBhcmFtIEJvdW5kYXJ5fFNjYWxhckZ1bmN0aW9uICRib3VuZGFyeQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":159,"slug":"binarylength","name":"binaryLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"BinaryLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":168,"slug":"call","name":"call","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"callable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arguments","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"refAlias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"returnType","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGFyZ3VtZW50cwogICAgICogQHBhcmFtIFR5cGU8bWl4ZWQ+ICRyZXR1cm5UeXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":177,"slug":"capitalize","name":"capitalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":185,"slug":"cast","name":"cast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":190,"slug":"chunk","name":"chunk","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"size","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Chunk","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":195,"slug":"coalesce","name":"coalesce","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":200,"slug":"codepointlength","name":"codePointLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CodePointLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":205,"slug":"collapsewhitespace","name":"collapseWhitespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CollapseWhitespace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":210,"slug":"concat","name":"concat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":215,"slug":"concatwithseparator","name":"concatWithSeparator","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":222,"slug":"contains","name":"contains","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Contains","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":227,"slug":"dateformat","name":"dateFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":232,"slug":"datetimeformat","name":"dateTimeFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":237,"slug":"divide","name":"divide","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"rounding","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Divide","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":245,"slug":"domelementattributescount","name":"domElementAttributesCount","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementAttributesCount","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":250,"slug":"domelementattributevalue","name":"domElementAttributeValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DOMElementAttributeValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":255,"slug":"domelementnextsibling","name":"domElementNextSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allowOnlyElement","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"DOMElementNextSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":260,"slug":"domelementparent","name":"domElementParent","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementParent","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":265,"slug":"domelementprevioussibling","name":"domElementPreviousSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allowOnlyElement","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"DOMElementPreviousSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":270,"slug":"domelementvalue","name":"domElementValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":275,"slug":"endswith","name":"endsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EndsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":280,"slug":"ensureend","name":"ensureEnd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureEnd","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":285,"slug":"ensurestart","name":"ensureStart","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureStart","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":290,"slug":"equals","name":"equals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":295,"slug":"exists","name":"exists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":320,"slug":"expand","name":"expand","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeHBhbmRzIGVhY2ggdmFsdWUgaW50byBlbnRyeSwgaWYgdGhlcmUgYXJlIG1vcmUgdGhhbiBvbmUgdmFsdWUsIG11bHRpcGxlIHJvd3Mgd2lsbCBiZSBjcmVhdGVkLgogICAgICogQXJyYXkga2V5cyBhcmUgaWdub3JlZCwgb25seSB2YWx1ZXMgYXJlIHVzZWQgdG8gY3JlYXRlIG5ldyByb3dzLgogICAgICoKICAgICAqIEJlZm9yZToKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqICAgfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAgICAgKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogICAgICoKICAgICAqIEFmdGVyOgogICAgICogICArLS0rLS0tLS0tLS0rCiAgICAgKiAgIHxpZHxleHBhbmRlZHwKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICogICB8IDF8ICAgICAgIDF8CiAgICAgKiAgIHwgMXwgICAgICAgMnwKICAgICAqICAgfCAxfCAgICAgICAzfAogICAgICogICArLS0rLS0tLS0tLS0rCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":325,"slug":"greaterthan","name":"greaterThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":330,"slug":"greaterthanequal","name":"greaterThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":335,"slug":"hash","name":"hash","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":340,"slug":"htmlqueryselector","name":"htmlQuerySelector","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelector","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":345,"slug":"htmlqueryselectorall","name":"htmlQuerySelectorAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelectorAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":353,"slug":"indexof","name":"indexOf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBpbmRleCBvZiBnaXZlbiAkbmVlZGxlIGluIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":364,"slug":"indexoflast","name":"indexOfLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOfLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBsYXN0IGluZGV4IG9mIGdpdmVuICRuZWVkbGUgaW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":372,"slug":"isempty","name":"isEmpty","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsEmpty","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":377,"slug":"iseven","name":"isEven","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":382,"slug":"isfalse","name":"isFalse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":390,"slug":"isin","name":"isIn","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"haystack","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsIn","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGhheXN0YWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":395,"slug":"isnotnull","name":"isNotNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":400,"slug":"isnotnumeric","name":"isNotNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":405,"slug":"isnull","name":"isNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":410,"slug":"isnumeric","name":"isNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":415,"slug":"isodd","name":"isOdd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":420,"slug":"istrue","name":"isTrue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":428,"slug":"istype","name":"isType","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IsType","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":440,"slug":"isutf8","name":"isUtf8","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsUtf8","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGVjayBzdHJpbmcgaXMgdXRmOCBhbmQgcmV0dXJucyB0cnVlIG9yIGZhbHNlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":445,"slug":"jsondecode","name":"jsonDecode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonDecode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":450,"slug":"jsonencode","name":"jsonEncode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonEncode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":455,"slug":"lessthan","name":"lessThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":460,"slug":"lessthanequal","name":"lessThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":465,"slug":"literal","name":"literal","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":470,"slug":"lower","name":"lower","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":475,"slug":"minus","name":"minus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Minus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":480,"slug":"mod","name":"mod","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mod","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":485,"slug":"modifydatetime","name":"modifyDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"modifier","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ModifyDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":490,"slug":"multiply","name":"multiply","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Multiply","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":495,"slug":"notequals","name":"notEquals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":500,"slug":"notsame","name":"notSame","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotSame","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":505,"slug":"numberformat","name":"numberFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimalSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousandsSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":519,"slug":"oneach","name":"onEach","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"OnEach","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeGVjdXRlIGEgc2NhbGFyIGZ1bmN0aW9uIG9uIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheS9saXN0L21hcC9zdHJ1Y3R1cmUgZW50cnkuCiAgICAgKiBJbiBvcmRlciB0byB1c2UgdGhpcyBmdW5jdGlvbiwgeW91IG5lZWQgdG8gcHJvdmlkZSBhIHJlZmVyZW5jZSB0byB0aGUgImVsZW1lbnQiIHRoYXQgd2lsbCBiZSB1c2VkIGluIHRoZSBmdW5jdGlvbi4KICAgICAqCiAgICAgKiBFeGFtcGxlOiAkZGYtPndpdGhFbnRyeSgnYXJyYXknLCByZWYoJ2FycmF5JyktPm9uRWFjaChyZWYoJ2VsZW1lbnQnKS0+Y2FzdCh0eXBlX3N0cmluZygpKSkpCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":524,"slug":"or","name":"or","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":529,"slug":"ornot","name":"orNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":534,"slug":"plus","name":"plus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Plus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":539,"slug":"power","name":"power","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Power","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":544,"slug":"prepend","name":"prepend","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Prepend","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":549,"slug":"regex","name":"regex","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":557,"slug":"regexall","name":"regexAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":565,"slug":"regexmatch","name":"regexMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":573,"slug":"regexmatchall","name":"regexMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":581,"slug":"regexreplace","name":"regexReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":589,"slug":"repeat","name":"repeat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"times","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Repeat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":594,"slug":"reverse","name":"reverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Reverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":599,"slug":"round","name":"round","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":604,"slug":"same","name":"same","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":609,"slug":"sanitize","name":"sanitize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":616,"slug":"size","name":"size","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":624,"slug":"slug","name":"slug","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'-'"},{"name":"locale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"symbolsMap","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Slug","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258bnVsbHxhcnJheTxhcnJheS1rZXksIG1peGVkPiAkc3ltYm9sc01hcAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":632,"slug":"split","name":"split","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":637,"slug":"sprintf","name":"sprintf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":642,"slug":"startswith","name":"startsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StartsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":650,"slug":"stringafter","name":"stringAfter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgZmlyc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":658,"slug":"stringafterlast","name":"stringAfterLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfterLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgbGFzdCBvY2N1cnJlbmNlIG9mIHRoZSBnaXZlbiBzdHJpbmcuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":668,"slug":"stringbefore","name":"stringBefore","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBefore","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGZpcnN0IG9jY3VycmVuY2Ugb2YgdGhlIGdpdmVuIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":678,"slug":"stringbeforelast","name":"stringBeforeLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBeforeLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGxhc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":688,"slug":"stringcontainsany","name":"stringContainsAny","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needles","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringContainsAny","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbiAkbmVlZGxlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":693,"slug":"stringequalsto","name":"stringEqualsTo","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringEqualsTo","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":701,"slug":"stringfold","name":"stringFold","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringFold","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGEgc3RyaW5nIHRoYXQgeW91IGNhbiB1c2UgaW4gY2FzZS1pbnNlbnNpdGl2ZSBjb21wYXJpc29ucy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":706,"slug":"stringmatch","name":"stringMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":711,"slug":"stringmatchall","name":"stringMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":716,"slug":"stringnormalize","name":"stringNormalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"form","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"16"}],"return_type":[{"name":"StringNormalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":725,"slug":"stringstyle","name":"stringStyle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"style","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringStyle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDb3ZlcnQgc3RyaW5nIHRvIGEgc3R5bGUgZnJvbSBlbnVtIGxpc3QsIHBhc3NlZCBpbiBwYXJhbWV0ZXIuCiAgICAgKiBDYW4gYmUgc3RyaW5nICJ1cHBlciIgb3IgU3RyaW5nU3R5bGVzOjpVUFBFUiBmb3IgVXBwZXIgKGV4YW1wbGUpLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":733,"slug":"stringtitle","name":"stringTitle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allWords","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringTitle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGFuZ2VzIGFsbCBncmFwaGVtZXMvY29kZSBwb2ludHMgdG8gInRpdGxlIGNhc2UiLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":738,"slug":"stringwidth","name":"stringWidth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringWidth","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":743,"slug":"strpad","name":"strPad","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"},{"name":"type","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":748,"slug":"strpadboth","name":"strPadBoth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":753,"slug":"strpadleft","name":"strPadLeft","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":758,"slug":"strpadright","name":"strPadRight","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":767,"slug":"strreplace","name":"strReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"search","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StrReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbnxzdHJpbmcgJHNlYXJjaAogICAgICogQHBhcmFtIGFycmF5PHN0cmluZz58U2NhbGFyRnVuY3Rpb258c3RyaW5nICRyZXBsYWNlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":775,"slug":"todate","name":"toDate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":786,"slug":"todatetime","name":"toDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqIEBwYXJhbSBcRGF0ZVRpbWVab25lfFNjYWxhckZ1bmN0aW9uICR0aW1lWm9uZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":793,"slug":"trim","name":"trim","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\ETL\\Function\\Trim","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Trim\\Type::..."},{"name":"characters","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' \\t\\n\\r\\0\u000b'"}],"return_type":[{"name":"Trim","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":798,"slug":"truncate","name":"truncate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ellipsis","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'...'"}],"return_type":[{"name":"Truncate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":803,"slug":"unicodelength","name":"unicodeLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"UnicodeLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":830,"slug":"unpack","name":"unpack","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"skipKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"entryPrefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHNraXBLZXlzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":837,"slug":"upper","name":"upper","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":842,"slug":"wordwrap","name":"wordwrap","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"width","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"break","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"cut","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Wordwrap","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":850,"slug":"xpath","name":"xpath","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"XPath","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":23,"slug":"setup","name":"setUp","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"config","type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":28,"slug":"extract","name":"extract","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":33,"slug":"from","name":"from","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":38,"slug":"process","name":"process","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":46,"slug":"read","name":"read","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRmxvdzo6ZXh0cmFjdCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":89,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":99,"slug":"autocast","name":"autoCast","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":121,"slug":"batchby","name":"batchBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"minSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBidXQga2VlcCB0aG9zZSB3aXRoIGNvbW1vbiB2YWx1ZSBpbiBnaXZlbiBjb2x1bW4gdG9nZXRoZXIuCiAgICAgKiBUaGlzIHdvcmtzIHByb3Blcmx5IG9ubHkgb24gc29ydGVkIGRhdGFzZXRzLgogICAgICoKICAgICAqIFdoZW4gbWluU2l6ZSBpcyBub3QgcHJvdmlkZWQsIGJhdGNoZXMgd2lsbCBiZSBjcmVhdGVkIG9ubHkgd2hlbiB0aGVyZSBpcyBhIGNoYW5nZSBpbiB2YWx1ZSBvZiB0aGUgY29sdW1uLgogICAgICogV2hlbiBtaW5TaXplIGlzIHByb3ZpZGVkLCBiYXRjaGVzIHdpbGwgYmUgY3JlYXRlZCBvbmx5IHdoZW4gdGhlcmUgaXMgYSBjaGFuZ2UgaW4gdmFsdWUgb2YgdGhlIGNvbHVtbiBvcgogICAgICogd2hlbiB0aGVyZSBhcmUgYXQgbGVhc3QgbWluU2l6ZSByb3dzIGluIHRoZSBiYXRjaC4KICAgICAqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAkY29sdW1uIC0gY29sdW1uIHRvIGdyb3VwIGJ5IChhbGwgcm93cyB3aXRoIHNhbWUgdmFsdWUgc3RheSB0b2dldGhlcikKICAgICAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5TaXplIC0gb3B0aW9uYWwgbWluaW11bSByb3dzIHBlciBiYXRjaCBmb3IgZWZmaWNpZW5jeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":141,"slug":"batchsize","name":"batchSize","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplLgogICAgICogRm9yIGV4YW1wbGUsIHdoZW4gRXh0cmFjdG9yIGlzIHlpZWxkaW5nIG9uZSByb3cgYXQgdGltZSwgdGhpcyBtZXRob2Qgd2lsbCBtZXJnZSB0aGVtIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplCiAgICAgKiBiZWZvcmUgcGFzc2luZyB0aGVtIHRvIHRoZSBuZXh0IHBpcGVsaW5lIGVsZW1lbnQuCiAgICAgKiBTaW1pbGFybHkgd2hlbiBFeHRyYWN0b3IgaXMgeWllbGRpbmcgYmF0Y2hlcyBvZiByb3dzLCB0aGlzIG1ldGhvZCB3aWxsIHNwbGl0IHRoZW0gaW50byBzbWFsbGVyIGJhdGNoZXMgb2YgZ2l2ZW4KICAgICAqIHNpemUuCiAgICAgKgogICAgICogSW4gb3JkZXIgdG8gbWVyZ2UgYWxsIFJvd3MgaW50byBhIHNpbmdsZSBiYXRjaCB1c2UgRGF0YUZyYW1lOjpjb2xsZWN0KCkgbWV0aG9kIG9yIHNldCBzaXplIHRvIC0xIG9yIDAuCiAgICAgKgogICAgICogQHBhcmFtIGludDwtMSwgbWF4PiAkc2l6ZQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":172,"slug":"cache","name":"cache","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheBatchSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTdGFydCBwcm9jZXNzaW5nIHJvd3MgdXAgdG8gdGhpcyBtb21lbnQgYW5kIHB1dCBlYWNoIGluc3RhbmNlIG9mIFJvd3MKICAgICAqIGludG8gcHJldmlvdXNseSBkZWZpbmVkIGNhY2hlLgogICAgICogQ2FjaGUgdHlwZSBjYW4gYmUgc2V0IHRocm91Z2ggQ29uZmlnQnVpbGRlci4KICAgICAqIEJ5IGRlZmF1bHQgZXZlcnl0aGluZyBpcyBjYWNoZWQgaW4gc3lzdGVtIHRtcCBkaXIuCiAgICAgKgogICAgICogSW1wb3J0YW50OiBjYWNoZSBiYXRjaCBzaXplIG1pZ2h0IHNpZ25pZmljYW50bHkgaW1wcm92ZSBwZXJmb3JtYW5jZSB3aGVuIHByb2Nlc3NpbmcgbGFyZ2UgYW1vdW50IG9mIHJvd3MuCiAgICAgKiBMYXJnZXIgYmF0Y2ggc2l6ZSB3aWxsIGluY3JlYXNlIG1lbW9yeSBjb25zdW1wdGlvbiBidXQgd2lsbCByZWR1Y2UgbnVtYmVyIG9mIElPIG9wZXJhdGlvbnMuCiAgICAgKiBXaGVuIG5vdCBzZXQsIHRoZSBiYXRjaCBzaXplIGlzIHRha2VuIGZyb20gdGhlIGxhc3QgRGF0YUZyYW1lOjpiYXRjaFNpemUoKSBjYWxsLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHBhcmFtIG51bGx8c3RyaW5nICRpZAogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":193,"slug":"collect","name":"collect","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZWZvcmUgdHJhbnNmb3JtaW5nIHJvd3MsIGNvbGxlY3QgdGhlbSBhbmQgbWVyZ2UgaW50byBzaW5nbGUgUm93cyBpbnN0YW5jZS4KICAgICAqIFRoaXMgbWlnaHQgbGVhZCB0byBtZW1vcnkgaXNzdWVzIHdoZW4gcHJvY2Vzc2luZyBsYXJnZSBhbW91bnQgb2Ygcm93cywgdXNlIHdpdGggY2F1dGlvbi4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":212,"slug":"collectrefs","name":"collectRefs","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBhbGxvd3MgdG8gY29sbGVjdCByZWZlcmVuY2VzIHRvIGFsbCBlbnRyaWVzIHVzZWQgaW4gdGhpcyBwaXBlbGluZS4KICAgICAqCiAgICAgKiBgYGBwaHAKICAgICAqIChuZXcgRmxvdygpKQogICAgICogICAtPnJlYWQoRnJvbTo6Y2hhaW4oKSkKICAgICAqICAgLT5jb2xsZWN0UmVmcygkcmVmcyA9IHJlZnMoKSkKICAgICAqICAgLT5ydW4oKTsKICAgICAqIGBgYAogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":225,"slug":"constrain","name":"constrain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"constraint","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"constraints","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":238,"slug":"count","name":"count","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICogUmV0dXJuIHRvdGFsIGNvdW50IG9mIHJvd3MgcHJvY2Vzc2VkIGJ5IHRoaXMgcGlwZWxpbmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":259,"slug":"crossjoin","name":"crossJoin","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":276,"slug":"display","name":"display","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gaW50ICRsaW1pdCBtYXhpbXVtIG51bWJlcnMgb2Ygcm93cyB0byBkaXNwbGF5CiAgICAgKiBAcGFyYW0gYm9vbHxpbnQgJHRydW5jYXRlIGZhbHNlIG9yIGlmIHNldCB0byAwIGNvbHVtbnMgYXJlIG5vdCB0cnVuY2F0ZWQsIG90aGVyd2lzZSBkZWZhdWx0IHRydW5jYXRlIHRvIDIwCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgIGNoYXJhY3RlcnMKICAgICAqIEBwYXJhbSBGb3JtYXR0ZXIgJGZvcm1hdHRlcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":304,"slug":"drop","name":"drop","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGdpdmVuIGVudHJpZXMuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":318,"slug":"dropduplicates","name":"dropDuplicates","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAuLi4kZW50cmllcwogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHJldHVybiAkdGhpcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":331,"slug":"droppartitions","name":"dropPartitions","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dropPartitionColumns","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGFsbCBwYXJ0aXRpb25zIGZyb20gUm93cywgYWRkaXRpb25hbGx5IHdoZW4gJGRyb3BQYXJ0aXRpb25Db2x1bW5zIGlzIHNldCB0byB0cnVlLCBwYXJ0aXRpb24gY29sdW1ucyBhcmUKICAgICAqIGFsc28gcmVtb3ZlZC4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":338,"slug":"duplicaterow","name":"duplicateRow","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":358,"slug":"fetch","name":"fetch","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZSBhd2FyZSB0aGF0IGZldGNoIGlzIG5vdCBtZW1vcnkgc2FmZSBhbmQgd2lsbCBsb2FkIGFsbCByb3dzIGludG8gbWVtb3J5LgogICAgICogSWYgeW91IHdhbnQgdG8gc2FmZWx5IGl0ZXJhdGUgb3ZlciBSb3dzIHVzZSBvZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6LgogICAgICoKICAgICAqIERhdGFGcmFtZTo6Z2V0KCkgOiBcR2VuZXJhdG9yCiAgICAgKiBEYXRhRnJhbWU6OmdldEFzQXJyYXkoKSA6IFxHZW5lcmF0b3IKICAgICAqIERhdGFGcmFtZTo6Z2V0RWFjaCgpIDogXEdlbmVyYXRvcgogICAgICogRGF0YUZyYW1lOjpnZXRFYWNoQXNBcnJheSgpIDogXEdlbmVyYXRvcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":383,"slug":"filter","name":"filter","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":395,"slug":"filterpartitions","name":"filterPartitions","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"filter","type":[{"name":"Filter","namespace":"Flow\\Filesystem\\Path","is_nullable":false,"is_variadic":false},{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgUnVudGltZUV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":423,"slug":"filters","name":"filters","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxTY2FsYXJGdW5jdGlvbj4gJGZ1bmN0aW9ucwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":437,"slug":"foreach","name":"forEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MpIDogdm9pZCAkY2FsbGJhY2sKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":449,"slug":"get","name":"get","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gaW5zdGFuY2Ugb2YgUm93cy4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxSb3dzPgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":470,"slug":"getasarray","name":"getAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gYXJyYXkuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8YXJyYXk8YXJyYXk8bWl4ZWQ+Pj4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":491,"slug":"geteach","name":"getEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBpbnN0YW5jZSBvZiBSb3cuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8Um93PgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":514,"slug":"geteachasarray","name":"getEachAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBhcnJheS4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxhcnJheTxtaXhlZD4+CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":533,"slug":"groupby","name":"groupBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GroupedDataFrame","namespace":"Flow\\ETL\\DataFrame","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":541,"slug":"join","name":"join","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":557,"slug":"joineach","name":"joinEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"factory","type":[{"name":"DataFrameFactory","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwc2FsbS1wYXJhbSBzdHJpbmd8Sm9pbiAkdHlwZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":580,"slug":"limit","name":"limit","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":594,"slug":"load","name":"load","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"loader","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":606,"slug":"map","name":"map","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBjYWxsYWJsZShSb3cgJHJvdykgOiBSb3cgJGNhbGxiYWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":618,"slug":"match","name":"match","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfFNjaGVtYVZhbGlkYXRvciAkdmFsaWRhdG9yIC0gd2hlbiBudWxsLCBTdHJpY3RWYWxpZGF0b3IgZ2V0cyBpbml0aWFsaXplZAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":636,"slug":"mode","name":"mode","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"mode","type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"ExecutionMode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBpcyB1c2VkIHRvIHNldCB0aGUgYmVoYXZpb3Igb2YgdGhlIERhdGFGcmFtZS4KICAgICAqCiAgICAgKiBBdmFpbGFibGUgbW9kZXM6CiAgICAgKiAtIFNhdmVNb2RlIGRlZmluZXMgaG93IEZsb3cgc2hvdWxkIGJlaGF2ZSB3aGVuIHdyaXRpbmcgdG8gYSBmaWxlL2ZpbGVzIHRoYXQgYWxyZWFkeSBleGlzdHMuCiAgICAgKiAtIEV4ZWN1dGlvbk1vZGUgLSBkZWZpbmVzIGhvdyBmdW5jdGlvbnMgc2hvdWxkIGJlaGF2ZSB3aGVuIHRoZXkgZW5jb3VudGVyIHVuZXhwZWN0ZWQgZGF0YSAoZS5nLiwgdHlwZSBtaXNtYXRjaGVzLCBtaXNzaW5nIHZhbHVlcykuCiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAcmV0dXJuICR0aGlzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":661,"slug":"offset","name":"offset","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTa2lwIGdpdmVuIG51bWJlciBvZiByb3dzIGZyb20gdGhlIGJlZ2lubmluZyBvZiB0aGUgZGF0YXNldC4KICAgICAqIFdoZW4gJG9mZnNldCBpcyBudWxsLCBub3RoaW5nIGhhcHBlbnMgKG5vIHJvd3MgYXJlIHNraXBwZWQpLgogICAgICoKICAgICAqIFBlcmZvcm1hbmNlIE5vdGU6IERhdGFGcmFtZSBtdXN0IGl0ZXJhdGUgdGhyb3VnaCBhbmQgcHJvY2VzcyBhbGwgc2tpcHBlZCByb3dzCiAgICAgKiB0byByZWFjaCB0aGUgb2Zmc2V0IHBvc2l0aW9uLiBGb3IgbGFyZ2Ugb2Zmc2V0cywgdGhpcyBjYW4gaW1wYWN0IHBlcmZvcm1hbmNlCiAgICAgKiBhcyB0aGUgZGF0YSBzb3VyY2Ugc3RpbGwgbmVlZHMgdG8gYmUgcmVhZCBhbmQgcHJvY2Vzc2VkIHVwIHRvIHRoZSBvZmZzZXQgcG9pbnQuCiAgICAgKgogICAgICogQHBhcmFtID9pbnQ8MCwgbWF4PiAkb2Zmc2V0CiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":675,"slug":"onerror","name":"onError","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"handler","type":[{"name":"ErrorHandler","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":685,"slug":"partitionby","name":"partitionBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":694,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":710,"slug":"printrows","name":"printRows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":727,"slug":"printschema","name":"printSchema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":740,"slug":"rename","name":"rename","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"from","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"to","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":747,"slug":"renameeach","name":"renameEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"strategies","type":[{"name":"RenameEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":754,"slug":"reorderentries","name":"reorderEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"comparator","type":[{"name":"Comparator","namespace":"Flow\\ETL\\Transformer\\OrderEntries","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Transformer\\OrderEntries\\TypeComparator::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":765,"slug":"rows","name":"rows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6dHJhbnNmb3JtIG1ldGhvZC4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":783,"slug":"run","name":"run","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"analyze","type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Report","namespace":"Flow\\ETL\\Dataset","is_nullable":true,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIFdoZW4gYW5hbHl6aW5nIHBpcGVsaW5lIGV4ZWN1dGlvbiB3ZSBjYW4gY2hvc2UgdG8gY29sbGVjdCB2YXJpb3VzIG1ldHJpY3MgdGhyb3VnaCBhbmFseXplKCktPndpdGgqKCkgbWV0aG9kCiAgICAgKgogICAgICogLSBjb2x1bW4gc3RhdGlzdGljcyAtIGFuYWx5emUoKS0+d2l0aENvbHVtblN0YXRpc3RpY3MoKQogICAgICogLSBzY2hlbWEgLSBhbmFseXplKCktPndpdGhTY2hlbWEoKQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MsIEZsb3dDb250ZXh0ICRjb250ZXh0KTogdm9pZCAkY2FsbGJhY2sKICAgICAqIEBwYXJhbSBBbmFseXplfGJvb2wgJGFuYWx5emUgLSB3aGVuIHNldCBydW4gd2lsbCByZXR1cm4gUmVwb3J0CiAgICAgKgogICAgICogQHJldHVybiAoJGFuYWx5emUgaXMgQW5hbHl6ZXx0cnVlID8gUmVwb3J0IDogbnVsbCkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":815,"slug":"savemode","name":"saveMode","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"mode","type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjptb2RlLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":825,"slug":"schema","name":"schema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gU2NoZW1hCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":847,"slug":"select","name":"select","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogS2VlcCBvbmx5IGdpdmVuIGVudHJpZXMuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":857,"slug":"sortby","name":"sortBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":869,"slug":"transform","name":"transform","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjp3aXRoKCkuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":880,"slug":"until","name":"until","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgZGlmZmVyZW5jZSBiZXR3ZWVuIGZpbHRlciBhbmQgdW50aWwgaXMgdGhhdCBmaWx0ZXIgd2lsbCBrZWVwIGZpbHRlcmluZyByb3dzIHVudGlsIGV4dHJhY3RvcnMgZmluaXNoIHlpZWxkaW5nCiAgICAgKiByb3dzLiBVbnRpbCB3aWxsIHNlbmQgYSBTVE9QIHNpZ25hbCB0byB0aGUgRXh0cmFjdG9yIHdoZW4gdGhlIGNvbmRpdGlvbiBpcyBub3QgbWV0LgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":894,"slug":"void","name":"void","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogVGhpcyBtZXRob2QgaXMgdXNlZnVsIG1vc3RseSBpbiBkZXZlbG9wbWVudCB3aGVuCiAgICAgKiB5b3Ugd2FudCB0byBwYXVzZSBwcm9jZXNzaW5nIGF0IGNlcnRhaW4gbW9tZW50IHdpdGhvdXQKICAgICAqIHJlbW92aW5nIGNvZGUuIEFsbCBvcGVyYXRpb25zIHdpbGwgZ2V0IHByb2Nlc3NlZCB1cCB0byB0aGlzIHBvaW50LAogICAgICogZnJvbSBoZXJlIG5vIHJvd3MgYXJlIHBhc3NlZCBmb3J3YXJkLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":904,"slug":"with","name":"with","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":932,"slug":"withentries","name":"withEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxpbnQsIFdpdGhFbnRyeT58YXJyYXk8c3RyaW5nLCBTY2FsYXJGdW5jdGlvbnxXaW5kb3dGdW5jdGlvbnxXaXRoRW50cnk+ICRyZWZlcmVuY2VzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":950,"slug":"withentry","name":"withEntry","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"reference","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"WindowFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gRGVmaW5pdGlvbjxtaXhlZD58c3RyaW5nICRlbnRyeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":977,"slug":"write","name":"write","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"loader","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6bG9hZCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":20,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":35,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null}] \ No newline at end of file +[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":25,"slug":"and","name":"and","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":30,"slug":"andnot","name":"andNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":35,"slug":"append","name":"append","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Append","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":48,"slug":"arrayfilter","name":"arrayFilter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayFilter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IHJlbW92aW5nIGFsbCBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgIC0ganNvbgogICAgICogICAgLSBsaXN0CiAgICAgKiAgICAtIG1hcAogICAgICogICAgLSBzdHJ1Y3R1cmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":53,"slug":"arrayget","name":"arrayGet","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":61,"slug":"arraygetcollection","name":"arrayGetCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGtleXMKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":66,"slug":"arraygetcollectionfirst","name":"arrayGetCollectionFirst","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":79,"slug":"arraykeep","name":"arrayKeep","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeep","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IGtlZXBpbmcgb25seSBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgLSBqc29uCiAgICAgKiAgIC0gbGlzdAogICAgICogICAtIG1hcAogICAgICogICAtIHN0cnVjdHVyZS4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":92,"slug":"arraykeys","name":"arrayKeys","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayKeys","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCBrZXlzIGZyb20gYW4gYXJyYXksIGlnbm9yaW5nIHRoZSB2YWx1ZXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":100,"slug":"arraymerge","name":"arrayMerge","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHJlZgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":105,"slug":"arraymergecollection","name":"arrayMergeCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":110,"slug":"arraypathexists","name":"arrayPathExists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":115,"slug":"arrayreverse","name":"arrayReverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"preserveKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":120,"slug":"arraysort","name":"arraySort","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"sortFunction","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":136,"slug":"arrayvalues","name":"arrayValues","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayValues","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCB2YWx1ZXMgZnJvbSBhbiBhcnJheSwgaWdub3JpbmcgdGhlIGtleXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":141,"slug":"ascii","name":"ascii","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Ascii","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":151,"slug":"between","name":"between","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"lowerBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upperBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gbWl4ZWR8U2NhbGFyRnVuY3Rpb24gJGxvd2VyQm91bmRSZWYKICAgICAqIEBwYXJhbSBtaXhlZHxTY2FsYXJGdW5jdGlvbiAkdXBwZXJCb3VuZFJlZgogICAgICogQHBhcmFtIEJvdW5kYXJ5fFNjYWxhckZ1bmN0aW9uICRib3VuZGFyeQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":159,"slug":"binarylength","name":"binaryLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"BinaryLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":168,"slug":"call","name":"call","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"callable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arguments","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"refAlias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"returnType","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGFyZ3VtZW50cwogICAgICogQHBhcmFtIFR5cGU8bWl4ZWQ+ICRyZXR1cm5UeXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":177,"slug":"capitalize","name":"capitalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":185,"slug":"cast","name":"cast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":190,"slug":"chunk","name":"chunk","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"size","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Chunk","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":195,"slug":"coalesce","name":"coalesce","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":200,"slug":"codepointlength","name":"codePointLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CodePointLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":205,"slug":"collapsewhitespace","name":"collapseWhitespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CollapseWhitespace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":210,"slug":"concat","name":"concat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":215,"slug":"concatwithseparator","name":"concatWithSeparator","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":222,"slug":"contains","name":"contains","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Contains","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":227,"slug":"dateformat","name":"dateFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":232,"slug":"datetimeformat","name":"dateTimeFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":237,"slug":"divide","name":"divide","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"rounding","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Divide","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":245,"slug":"domelementattributescount","name":"domElementAttributesCount","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementAttributesCount","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":250,"slug":"domelementattributevalue","name":"domElementAttributeValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DOMElementAttributeValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":255,"slug":"domelementnextsibling","name":"domElementNextSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allowOnlyElement","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"DOMElementNextSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":260,"slug":"domelementparent","name":"domElementParent","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementParent","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":265,"slug":"domelementprevioussibling","name":"domElementPreviousSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allowOnlyElement","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"DOMElementPreviousSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":270,"slug":"domelementvalue","name":"domElementValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":275,"slug":"endswith","name":"endsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EndsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":280,"slug":"ensureend","name":"ensureEnd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureEnd","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":285,"slug":"ensurestart","name":"ensureStart","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureStart","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":290,"slug":"equals","name":"equals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":295,"slug":"exists","name":"exists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":320,"slug":"expand","name":"expand","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeHBhbmRzIGVhY2ggdmFsdWUgaW50byBlbnRyeSwgaWYgdGhlcmUgYXJlIG1vcmUgdGhhbiBvbmUgdmFsdWUsIG11bHRpcGxlIHJvd3Mgd2lsbCBiZSBjcmVhdGVkLgogICAgICogQXJyYXkga2V5cyBhcmUgaWdub3JlZCwgb25seSB2YWx1ZXMgYXJlIHVzZWQgdG8gY3JlYXRlIG5ldyByb3dzLgogICAgICoKICAgICAqIEJlZm9yZToKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqICAgfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAgICAgKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogICAgICoKICAgICAqIEFmdGVyOgogICAgICogICArLS0rLS0tLS0tLS0rCiAgICAgKiAgIHxpZHxleHBhbmRlZHwKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICogICB8IDF8ICAgICAgIDF8CiAgICAgKiAgIHwgMXwgICAgICAgMnwKICAgICAqICAgfCAxfCAgICAgICAzfAogICAgICogICArLS0rLS0tLS0tLS0rCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":325,"slug":"greaterthan","name":"greaterThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":330,"slug":"greaterthanequal","name":"greaterThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":335,"slug":"hash","name":"hash","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":340,"slug":"htmlqueryselector","name":"htmlQuerySelector","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelector","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":345,"slug":"htmlqueryselectorall","name":"htmlQuerySelectorAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelectorAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":353,"slug":"indexof","name":"indexOf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBpbmRleCBvZiBnaXZlbiAkbmVlZGxlIGluIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":364,"slug":"indexoflast","name":"indexOfLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOfLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBsYXN0IGluZGV4IG9mIGdpdmVuICRuZWVkbGUgaW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":372,"slug":"isempty","name":"isEmpty","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsEmpty","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":377,"slug":"iseven","name":"isEven","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":382,"slug":"isfalse","name":"isFalse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":390,"slug":"isin","name":"isIn","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"haystack","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsIn","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGhheXN0YWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":395,"slug":"isnotnull","name":"isNotNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":400,"slug":"isnotnumeric","name":"isNotNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":405,"slug":"isnull","name":"isNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":410,"slug":"isnumeric","name":"isNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":415,"slug":"isodd","name":"isOdd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":420,"slug":"istrue","name":"isTrue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":428,"slug":"istype","name":"isType","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IsType","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":440,"slug":"isutf8","name":"isUtf8","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsUtf8","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGVjayBzdHJpbmcgaXMgdXRmOCBhbmQgcmV0dXJucyB0cnVlIG9yIGZhbHNlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":445,"slug":"jsondecode","name":"jsonDecode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonDecode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":450,"slug":"jsonencode","name":"jsonEncode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonEncode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":455,"slug":"lessthan","name":"lessThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":460,"slug":"lessthanequal","name":"lessThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":465,"slug":"literal","name":"literal","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":470,"slug":"lower","name":"lower","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":475,"slug":"minus","name":"minus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Minus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":480,"slug":"mod","name":"mod","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mod","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":485,"slug":"modifydatetime","name":"modifyDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"modifier","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ModifyDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":490,"slug":"multiply","name":"multiply","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Multiply","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":495,"slug":"notequals","name":"notEquals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":500,"slug":"notsame","name":"notSame","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotSame","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":505,"slug":"numberformat","name":"numberFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimalSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousandsSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":519,"slug":"oneach","name":"onEach","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"OnEach","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeGVjdXRlIGEgc2NhbGFyIGZ1bmN0aW9uIG9uIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheS9saXN0L21hcC9zdHJ1Y3R1cmUgZW50cnkuCiAgICAgKiBJbiBvcmRlciB0byB1c2UgdGhpcyBmdW5jdGlvbiwgeW91IG5lZWQgdG8gcHJvdmlkZSBhIHJlZmVyZW5jZSB0byB0aGUgImVsZW1lbnQiIHRoYXQgd2lsbCBiZSB1c2VkIGluIHRoZSBmdW5jdGlvbi4KICAgICAqCiAgICAgKiBFeGFtcGxlOiAkZGYtPndpdGhFbnRyeSgnYXJyYXknLCByZWYoJ2FycmF5JyktPm9uRWFjaChyZWYoJ2VsZW1lbnQnKS0+Y2FzdCh0eXBlX3N0cmluZygpKSkpCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":524,"slug":"or","name":"or","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":529,"slug":"ornot","name":"orNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":534,"slug":"plus","name":"plus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Plus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":539,"slug":"power","name":"power","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Power","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":544,"slug":"prepend","name":"prepend","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Prepend","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":549,"slug":"regex","name":"regex","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":557,"slug":"regexall","name":"regexAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":565,"slug":"regexmatch","name":"regexMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":573,"slug":"regexmatchall","name":"regexMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":581,"slug":"regexreplace","name":"regexReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":589,"slug":"repeat","name":"repeat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"times","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Repeat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":594,"slug":"reverse","name":"reverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Reverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":599,"slug":"round","name":"round","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":604,"slug":"same","name":"same","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":609,"slug":"sanitize","name":"sanitize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":616,"slug":"size","name":"size","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":624,"slug":"slug","name":"slug","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'-'"},{"name":"locale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"symbolsMap","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Slug","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258bnVsbHxhcnJheTxhcnJheS1rZXksIG1peGVkPiAkc3ltYm9sc01hcAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":632,"slug":"split","name":"split","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":637,"slug":"sprintf","name":"sprintf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":642,"slug":"startswith","name":"startsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StartsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":650,"slug":"stringafter","name":"stringAfter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgZmlyc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":658,"slug":"stringafterlast","name":"stringAfterLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfterLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgbGFzdCBvY2N1cnJlbmNlIG9mIHRoZSBnaXZlbiBzdHJpbmcuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":668,"slug":"stringbefore","name":"stringBefore","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBefore","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGZpcnN0IG9jY3VycmVuY2Ugb2YgdGhlIGdpdmVuIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":678,"slug":"stringbeforelast","name":"stringBeforeLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBeforeLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGxhc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":688,"slug":"stringcontainsany","name":"stringContainsAny","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needles","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringContainsAny","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbiAkbmVlZGxlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":693,"slug":"stringequalsto","name":"stringEqualsTo","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringEqualsTo","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":701,"slug":"stringfold","name":"stringFold","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringFold","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGEgc3RyaW5nIHRoYXQgeW91IGNhbiB1c2UgaW4gY2FzZS1pbnNlbnNpdGl2ZSBjb21wYXJpc29ucy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":706,"slug":"stringmatch","name":"stringMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":711,"slug":"stringmatchall","name":"stringMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":716,"slug":"stringnormalize","name":"stringNormalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"form","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"16"}],"return_type":[{"name":"StringNormalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":725,"slug":"stringstyle","name":"stringStyle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"style","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringStyle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDb3ZlcnQgc3RyaW5nIHRvIGEgc3R5bGUgZnJvbSBlbnVtIGxpc3QsIHBhc3NlZCBpbiBwYXJhbWV0ZXIuCiAgICAgKiBDYW4gYmUgc3RyaW5nICJ1cHBlciIgb3IgU3RyaW5nU3R5bGVzOjpVUFBFUiBmb3IgVXBwZXIgKGV4YW1wbGUpLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":733,"slug":"stringtitle","name":"stringTitle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allWords","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringTitle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGFuZ2VzIGFsbCBncmFwaGVtZXMvY29kZSBwb2ludHMgdG8gInRpdGxlIGNhc2UiLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":738,"slug":"stringwidth","name":"stringWidth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringWidth","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":743,"slug":"strpad","name":"strPad","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"},{"name":"type","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":748,"slug":"strpadboth","name":"strPadBoth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":753,"slug":"strpadleft","name":"strPadLeft","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":758,"slug":"strpadright","name":"strPadRight","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":767,"slug":"strreplace","name":"strReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"search","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StrReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbnxzdHJpbmcgJHNlYXJjaAogICAgICogQHBhcmFtIGFycmF5PHN0cmluZz58U2NhbGFyRnVuY3Rpb258c3RyaW5nICRyZXBsYWNlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":775,"slug":"todate","name":"toDate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":786,"slug":"todatetime","name":"toDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqIEBwYXJhbSBcRGF0ZVRpbWVab25lfFNjYWxhckZ1bmN0aW9uICR0aW1lWm9uZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":793,"slug":"trim","name":"trim","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\ETL\\Function\\Trim","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Trim\\Type::..."},{"name":"characters","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' \\t\\n\\r\\0\u000b'"}],"return_type":[{"name":"Trim","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":798,"slug":"truncate","name":"truncate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ellipsis","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'...'"}],"return_type":[{"name":"Truncate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":803,"slug":"unicodelength","name":"unicodeLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"UnicodeLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":830,"slug":"unpack","name":"unpack","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"skipKeys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"entryPrefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHNraXBLZXlzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":837,"slug":"upper","name":"upper","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":842,"slug":"wordwrap","name":"wordwrap","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"width","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"break","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"cut","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Wordwrap","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":850,"slug":"xpath","name":"xpath","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"XPath","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":23,"slug":"setup","name":"setUp","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"config","type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":28,"slug":"extract","name":"extract","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":33,"slug":"from","name":"from","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":38,"slug":"process","name":"process","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":46,"slug":"read","name":"read","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRmxvdzo6ZXh0cmFjdCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":89,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":99,"slug":"autocast","name":"autoCast","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":121,"slug":"batchby","name":"batchBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"minSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBidXQga2VlcCB0aG9zZSB3aXRoIGNvbW1vbiB2YWx1ZSBpbiBnaXZlbiBjb2x1bW4gdG9nZXRoZXIuCiAgICAgKiBUaGlzIHdvcmtzIHByb3Blcmx5IG9ubHkgb24gc29ydGVkIGRhdGFzZXRzLgogICAgICoKICAgICAqIFdoZW4gbWluU2l6ZSBpcyBub3QgcHJvdmlkZWQsIGJhdGNoZXMgd2lsbCBiZSBjcmVhdGVkIG9ubHkgd2hlbiB0aGVyZSBpcyBhIGNoYW5nZSBpbiB2YWx1ZSBvZiB0aGUgY29sdW1uLgogICAgICogV2hlbiBtaW5TaXplIGlzIHByb3ZpZGVkLCBiYXRjaGVzIHdpbGwgYmUgY3JlYXRlZCBvbmx5IHdoZW4gdGhlcmUgaXMgYSBjaGFuZ2UgaW4gdmFsdWUgb2YgdGhlIGNvbHVtbiBvcgogICAgICogd2hlbiB0aGVyZSBhcmUgYXQgbGVhc3QgbWluU2l6ZSByb3dzIGluIHRoZSBiYXRjaC4KICAgICAqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAkY29sdW1uIC0gY29sdW1uIHRvIGdyb3VwIGJ5IChhbGwgcm93cyB3aXRoIHNhbWUgdmFsdWUgc3RheSB0b2dldGhlcikKICAgICAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5TaXplIC0gb3B0aW9uYWwgbWluaW11bSByb3dzIHBlciBiYXRjaCBmb3IgZWZmaWNpZW5jeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":141,"slug":"batchsize","name":"batchSize","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplLgogICAgICogRm9yIGV4YW1wbGUsIHdoZW4gRXh0cmFjdG9yIGlzIHlpZWxkaW5nIG9uZSByb3cgYXQgdGltZSwgdGhpcyBtZXRob2Qgd2lsbCBtZXJnZSB0aGVtIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplCiAgICAgKiBiZWZvcmUgcGFzc2luZyB0aGVtIHRvIHRoZSBuZXh0IHBpcGVsaW5lIGVsZW1lbnQuCiAgICAgKiBTaW1pbGFybHkgd2hlbiBFeHRyYWN0b3IgaXMgeWllbGRpbmcgYmF0Y2hlcyBvZiByb3dzLCB0aGlzIG1ldGhvZCB3aWxsIHNwbGl0IHRoZW0gaW50byBzbWFsbGVyIGJhdGNoZXMgb2YgZ2l2ZW4KICAgICAqIHNpemUuCiAgICAgKgogICAgICogSW4gb3JkZXIgdG8gbWVyZ2UgYWxsIFJvd3MgaW50byBhIHNpbmdsZSBiYXRjaCB1c2UgRGF0YUZyYW1lOjpjb2xsZWN0KCkgbWV0aG9kIG9yIHNldCBzaXplIHRvIC0xIG9yIDAuCiAgICAgKgogICAgICogQHBhcmFtIGludDwtMSwgbWF4PiAkc2l6ZQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":172,"slug":"cache","name":"cache","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheBatchSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTdGFydCBwcm9jZXNzaW5nIHJvd3MgdXAgdG8gdGhpcyBtb21lbnQgYW5kIHB1dCBlYWNoIGluc3RhbmNlIG9mIFJvd3MKICAgICAqIGludG8gcHJldmlvdXNseSBkZWZpbmVkIGNhY2hlLgogICAgICogQ2FjaGUgdHlwZSBjYW4gYmUgc2V0IHRocm91Z2ggQ29uZmlnQnVpbGRlci4KICAgICAqIEJ5IGRlZmF1bHQgZXZlcnl0aGluZyBpcyBjYWNoZWQgaW4gc3lzdGVtIHRtcCBkaXIuCiAgICAgKgogICAgICogSW1wb3J0YW50OiBjYWNoZSBiYXRjaCBzaXplIG1pZ2h0IHNpZ25pZmljYW50bHkgaW1wcm92ZSBwZXJmb3JtYW5jZSB3aGVuIHByb2Nlc3NpbmcgbGFyZ2UgYW1vdW50IG9mIHJvd3MuCiAgICAgKiBMYXJnZXIgYmF0Y2ggc2l6ZSB3aWxsIGluY3JlYXNlIG1lbW9yeSBjb25zdW1wdGlvbiBidXQgd2lsbCByZWR1Y2UgbnVtYmVyIG9mIElPIG9wZXJhdGlvbnMuCiAgICAgKiBXaGVuIG5vdCBzZXQsIHRoZSBiYXRjaCBzaXplIGlzIHRha2VuIGZyb20gdGhlIGxhc3QgRGF0YUZyYW1lOjpiYXRjaFNpemUoKSBjYWxsLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHBhcmFtIG51bGx8c3RyaW5nICRpZAogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":193,"slug":"collect","name":"collect","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZWZvcmUgdHJhbnNmb3JtaW5nIHJvd3MsIGNvbGxlY3QgdGhlbSBhbmQgbWVyZ2UgaW50byBzaW5nbGUgUm93cyBpbnN0YW5jZS4KICAgICAqIFRoaXMgbWlnaHQgbGVhZCB0byBtZW1vcnkgaXNzdWVzIHdoZW4gcHJvY2Vzc2luZyBsYXJnZSBhbW91bnQgb2Ygcm93cywgdXNlIHdpdGggY2F1dGlvbi4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":212,"slug":"collectrefs","name":"collectRefs","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBhbGxvd3MgdG8gY29sbGVjdCByZWZlcmVuY2VzIHRvIGFsbCBlbnRyaWVzIHVzZWQgaW4gdGhpcyBwaXBlbGluZS4KICAgICAqCiAgICAgKiBgYGBwaHAKICAgICAqIChuZXcgRmxvdygpKQogICAgICogICAtPnJlYWQoRnJvbTo6Y2hhaW4oKSkKICAgICAqICAgLT5jb2xsZWN0UmVmcygkcmVmcyA9IHJlZnMoKSkKICAgICAqICAgLT5ydW4oKTsKICAgICAqIGBgYAogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":225,"slug":"constrain","name":"constrain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"constraint","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"constraints","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":238,"slug":"count","name":"count","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICogUmV0dXJuIHRvdGFsIGNvdW50IG9mIHJvd3MgcHJvY2Vzc2VkIGJ5IHRoaXMgcGlwZWxpbmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":259,"slug":"crossjoin","name":"crossJoin","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":276,"slug":"display","name":"display","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gaW50ICRsaW1pdCBtYXhpbXVtIG51bWJlcnMgb2Ygcm93cyB0byBkaXNwbGF5CiAgICAgKiBAcGFyYW0gYm9vbHxpbnQgJHRydW5jYXRlIGZhbHNlIG9yIGlmIHNldCB0byAwIGNvbHVtbnMgYXJlIG5vdCB0cnVuY2F0ZWQsIG90aGVyd2lzZSBkZWZhdWx0IHRydW5jYXRlIHRvIDIwCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgIGNoYXJhY3RlcnMKICAgICAqIEBwYXJhbSBGb3JtYXR0ZXIgJGZvcm1hdHRlcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":304,"slug":"drop","name":"drop","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGdpdmVuIGVudHJpZXMuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":318,"slug":"dropduplicates","name":"dropDuplicates","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAuLi4kZW50cmllcwogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHJldHVybiAkdGhpcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":331,"slug":"droppartitions","name":"dropPartitions","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dropPartitionColumns","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGFsbCBwYXJ0aXRpb25zIGZyb20gUm93cywgYWRkaXRpb25hbGx5IHdoZW4gJGRyb3BQYXJ0aXRpb25Db2x1bW5zIGlzIHNldCB0byB0cnVlLCBwYXJ0aXRpb24gY29sdW1ucyBhcmUKICAgICAqIGFsc28gcmVtb3ZlZC4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":338,"slug":"duplicaterow","name":"duplicateRow","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":358,"slug":"fetch","name":"fetch","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZSBhd2FyZSB0aGF0IGZldGNoIGlzIG5vdCBtZW1vcnkgc2FmZSBhbmQgd2lsbCBsb2FkIGFsbCByb3dzIGludG8gbWVtb3J5LgogICAgICogSWYgeW91IHdhbnQgdG8gc2FmZWx5IGl0ZXJhdGUgb3ZlciBSb3dzIHVzZSBvZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6LgogICAgICoKICAgICAqIERhdGFGcmFtZTo6Z2V0KCkgOiBcR2VuZXJhdG9yCiAgICAgKiBEYXRhRnJhbWU6OmdldEFzQXJyYXkoKSA6IFxHZW5lcmF0b3IKICAgICAqIERhdGFGcmFtZTo6Z2V0RWFjaCgpIDogXEdlbmVyYXRvcgogICAgICogRGF0YUZyYW1lOjpnZXRFYWNoQXNBcnJheSgpIDogXEdlbmVyYXRvcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":383,"slug":"filter","name":"filter","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":395,"slug":"filterpartitions","name":"filterPartitions","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"filter","type":[{"name":"Filter","namespace":"Flow\\Filesystem\\Path","is_nullable":false,"is_variadic":false},{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgUnVudGltZUV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":423,"slug":"filters","name":"filters","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxTY2FsYXJGdW5jdGlvbj4gJGZ1bmN0aW9ucwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":437,"slug":"foreach","name":"forEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MpIDogdm9pZCAkY2FsbGJhY2sKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":449,"slug":"get","name":"get","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gaW5zdGFuY2Ugb2YgUm93cy4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxSb3dzPgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":470,"slug":"getasarray","name":"getAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gYXJyYXkuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8YXJyYXk8YXJyYXk8bWl4ZWQ+Pj4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":491,"slug":"geteach","name":"getEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBpbnN0YW5jZSBvZiBSb3cuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8Um93PgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":514,"slug":"geteachasarray","name":"getEachAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBhcnJheS4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxhcnJheTxtaXhlZD4+CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":533,"slug":"groupby","name":"groupBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GroupedDataFrame","namespace":"Flow\\ETL\\DataFrame","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":541,"slug":"join","name":"join","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":557,"slug":"joineach","name":"joinEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"factory","type":[{"name":"DataFrameFactory","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwc2FsbS1wYXJhbSBzdHJpbmd8Sm9pbiAkdHlwZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":580,"slug":"limit","name":"limit","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":594,"slug":"load","name":"load","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"loader","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":606,"slug":"map","name":"map","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBjYWxsYWJsZShSb3cgJHJvdykgOiBSb3cgJGNhbGxiYWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":618,"slug":"match","name":"match","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfFNjaGVtYVZhbGlkYXRvciAkdmFsaWRhdG9yIC0gd2hlbiBudWxsLCBTdHJpY3RWYWxpZGF0b3IgZ2V0cyBpbml0aWFsaXplZAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":636,"slug":"mode","name":"mode","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"mode","type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"ExecutionMode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBpcyB1c2VkIHRvIHNldCB0aGUgYmVoYXZpb3Igb2YgdGhlIERhdGFGcmFtZS4KICAgICAqCiAgICAgKiBBdmFpbGFibGUgbW9kZXM6CiAgICAgKiAtIFNhdmVNb2RlIGRlZmluZXMgaG93IEZsb3cgc2hvdWxkIGJlaGF2ZSB3aGVuIHdyaXRpbmcgdG8gYSBmaWxlL2ZpbGVzIHRoYXQgYWxyZWFkeSBleGlzdHMuCiAgICAgKiAtIEV4ZWN1dGlvbk1vZGUgLSBkZWZpbmVzIGhvdyBmdW5jdGlvbnMgc2hvdWxkIGJlaGF2ZSB3aGVuIHRoZXkgZW5jb3VudGVyIHVuZXhwZWN0ZWQgZGF0YSAoZS5nLiwgdHlwZSBtaXNtYXRjaGVzLCBtaXNzaW5nIHZhbHVlcykuCiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAcmV0dXJuICR0aGlzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":661,"slug":"offset","name":"offset","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTa2lwIGdpdmVuIG51bWJlciBvZiByb3dzIGZyb20gdGhlIGJlZ2lubmluZyBvZiB0aGUgZGF0YXNldC4KICAgICAqIFdoZW4gJG9mZnNldCBpcyBudWxsLCBub3RoaW5nIGhhcHBlbnMgKG5vIHJvd3MgYXJlIHNraXBwZWQpLgogICAgICoKICAgICAqIFBlcmZvcm1hbmNlIE5vdGU6IERhdGFGcmFtZSBtdXN0IGl0ZXJhdGUgdGhyb3VnaCBhbmQgcHJvY2VzcyBhbGwgc2tpcHBlZCByb3dzCiAgICAgKiB0byByZWFjaCB0aGUgb2Zmc2V0IHBvc2l0aW9uLiBGb3IgbGFyZ2Ugb2Zmc2V0cywgdGhpcyBjYW4gaW1wYWN0IHBlcmZvcm1hbmNlCiAgICAgKiBhcyB0aGUgZGF0YSBzb3VyY2Ugc3RpbGwgbmVlZHMgdG8gYmUgcmVhZCBhbmQgcHJvY2Vzc2VkIHVwIHRvIHRoZSBvZmZzZXQgcG9pbnQuCiAgICAgKgogICAgICogQHBhcmFtID9pbnQ8MCwgbWF4PiAkb2Zmc2V0CiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":675,"slug":"onerror","name":"onError","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"handler","type":[{"name":"ErrorHandler","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":685,"slug":"partitionby","name":"partitionBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":694,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":710,"slug":"printrows","name":"printRows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":727,"slug":"printschema","name":"printSchema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":740,"slug":"rename","name":"rename","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"from","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"to","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":747,"slug":"renameeach","name":"renameEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"strategies","type":[{"name":"RenameEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":754,"slug":"reorderentries","name":"reorderEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"comparator","type":[{"name":"Comparator","namespace":"Flow\\ETL\\Transformer\\OrderEntries","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Transformer\\OrderEntries\\TypeComparator::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":765,"slug":"rows","name":"rows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6dHJhbnNmb3JtIG1ldGhvZC4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":783,"slug":"run","name":"run","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"analyze","type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Report","namespace":"Flow\\ETL\\Dataset","is_nullable":true,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIFdoZW4gYW5hbHl6aW5nIHBpcGVsaW5lIGV4ZWN1dGlvbiB3ZSBjYW4gY2hvc2UgdG8gY29sbGVjdCB2YXJpb3VzIG1ldHJpY3MgdGhyb3VnaCBhbmFseXplKCktPndpdGgqKCkgbWV0aG9kCiAgICAgKgogICAgICogLSBjb2x1bW4gc3RhdGlzdGljcyAtIGFuYWx5emUoKS0+d2l0aENvbHVtblN0YXRpc3RpY3MoKQogICAgICogLSBzY2hlbWEgLSBhbmFseXplKCktPndpdGhTY2hlbWEoKQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MsIEZsb3dDb250ZXh0ICRjb250ZXh0KTogdm9pZCAkY2FsbGJhY2sKICAgICAqIEBwYXJhbSBBbmFseXplfGJvb2wgJGFuYWx5emUgLSB3aGVuIHNldCBydW4gd2lsbCByZXR1cm4gUmVwb3J0CiAgICAgKgogICAgICogQHJldHVybiAoJGFuYWx5emUgaXMgQW5hbHl6ZXx0cnVlID8gUmVwb3J0IDogbnVsbCkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":815,"slug":"savemode","name":"saveMode","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"mode","type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjptb2RlLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":825,"slug":"schema","name":"schema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gU2NoZW1hCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":847,"slug":"select","name":"select","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogS2VlcCBvbmx5IGdpdmVuIGVudHJpZXMuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":857,"slug":"sortby","name":"sortBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":869,"slug":"transform","name":"transform","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjp3aXRoKCkuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":880,"slug":"until","name":"until","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgZGlmZmVyZW5jZSBiZXR3ZWVuIGZpbHRlciBhbmQgdW50aWwgaXMgdGhhdCBmaWx0ZXIgd2lsbCBrZWVwIGZpbHRlcmluZyByb3dzIHVudGlsIGV4dHJhY3RvcnMgZmluaXNoIHlpZWxkaW5nCiAgICAgKiByb3dzLiBVbnRpbCB3aWxsIHNlbmQgYSBTVE9QIHNpZ25hbCB0byB0aGUgRXh0cmFjdG9yIHdoZW4gdGhlIGNvbmRpdGlvbiBpcyBub3QgbWV0LgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":894,"slug":"void","name":"void","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogVGhpcyBtZXRob2QgaXMgdXNlZnVsIG1vc3RseSBpbiBkZXZlbG9wbWVudCB3aGVuCiAgICAgKiB5b3Ugd2FudCB0byBwYXVzZSBwcm9jZXNzaW5nIGF0IGNlcnRhaW4gbW9tZW50IHdpdGhvdXQKICAgICAqIHJlbW92aW5nIGNvZGUuIEFsbCBvcGVyYXRpb25zIHdpbGwgZ2V0IHByb2Nlc3NlZCB1cCB0byB0aGlzIHBvaW50LAogICAgICogZnJvbSBoZXJlIG5vIHJvd3MgYXJlIHBhc3NlZCBmb3J3YXJkLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":904,"slug":"with","name":"with","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":932,"slug":"withentries","name":"withEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxpbnQsIFdpdGhFbnRyeT58YXJyYXk8c3RyaW5nLCBTY2FsYXJGdW5jdGlvbnxXaW5kb3dGdW5jdGlvbnxXaXRoRW50cnk+ICRyZWZlcmVuY2VzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":950,"slug":"withentry","name":"withEntry","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"reference","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"WindowFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gRGVmaW5pdGlvbjxtaXhlZD58c3RyaW5nICRlbnRyeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":977,"slug":"write","name":"write","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"loader","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6bG9hZCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":20,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":30,"slug":"getgroups","name":"getGroups","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gbnVsbHxpbnQ8MSwgbWF4PiAkc2l6ZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":35,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null}] \ No newline at end of file