Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"ext-zlib": "*",
"composer-runtime-api": "^2.2",
"async-aws/s3": "^2.6 || ^3.0",
"brick/math": "^0.12 || ^0.13 || ^0.14 || ^0.15 || ^0.16 || ^0.17",
"brick/math": "^0.12 || ^0.13 || ^0.14 || ^0.15 || ^0.16 || ^0.17 || ^0.18",
"cmsig/seal": "^0.12",
"coduo/php-humanizer": "^5.0",
"cuyz/valinor": "^2.4",
Expand Down
14 changes: 7 additions & 7 deletions composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 23 additions & 10 deletions documentation/upgrading.md
Original file line number Diff line number Diff line change
Expand Up @@ -226,25 +226,38 @@ Applies to the HttpKernel server span and to the `http_client`/`psr18_client` cl

### 15) `flow-php/symfony-telemetry-bundle` - `runtime_mode` removed; terminate flushes, process end shuts down

| Before | After |
|------------------------------------------------------------------------------|---------------------------------------------|
| `flow_telemetry.runtime_mode: auto`/`classic`/`worker` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\RuntimeModeResolver` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\RuntimeMode` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\WorkerModeDetector` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\EnvironmentWorkerModeDetector` | removed |
| shutdown on `kernel.terminate`/`console.terminate` (classic mode) | flush on terminate; shutdown at process end |
| Before | After |
|-----------------------------------------------------------------------------|---------------------------------------------|
| `flow_telemetry.runtime_mode: auto`/`classic`/`worker` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\RuntimeModeResolver` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\RuntimeMode` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\WorkerModeDetector` | removed |
| `Flow\Bridge\Symfony\TelemetryBundle\Runtime\EnvironmentWorkerModeDetector` | removed |
| shutdown on `kernel.terminate`/`console.terminate` (classic mode) | flush on terminate; shutdown at process end |

Drop the `runtime_mode` key from `flow_telemetry` config and remove any `WorkerModeDetector` service overrides.

### 16) `flow-php/telemetry` - `Telemetry::registerShutdownFunction()` holds a weak reference

| Before | After |
|--------------------------------------------------|----------------------------------------------------------------|
| Before | After |
|--------------------------------------------------|---------------------------------------------------------------|
| strong reference; instance kept alive until exit | weak reference; garbage-collected instances are not shut down |

Keep the registered `Telemetry` instance referenced for as long as it should be shut down at process end.

### 17) `flow-php/etl` - `SchemaValidator::isValid()` replaced by `validate(): ValidationContext`

| Before | After |
|-----------------------------------------------|------------------------------------------------------------------|
| `SchemaValidator::isValid(...): bool` | `SchemaValidator::validate(...): ValidationContext` |
| `$validator->isValid($expected, $given)` | `$validator->validate($expected, $given)->isValid()` |
| `schema_validate(...): bool` | `schema_validate(...): ValidationContext` |
| `new SchemaValidationException($exp, $given)` | `new SchemaValidationException($exp, $given, ValidationContext)` |
| — | `SchemaValidationException::context(): ValidationContext` |

Custom `SchemaValidator` implementations must return a `Flow\ETL\Schema\Validator\ValidationContext`
built from the missing, mismatched (`MismatchedDefinition`), and unexpected definitions they reject.

---

## Upgrading from 0.40.x to 0.41.x
Expand Down
2 changes: 1 addition & 1 deletion src/core/etl/composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"php": "~8.3.0 || ~8.4.0 || ~8.5.0",
"ext-json": "*",
"psr/clock": "^1.0",
"brick/math": "^0.12 || ^0.13 || ^0.14 || ^0.15 || ^0.16 || ^0.17",
"brick/math": "^0.12 || ^0.13 || ^0.14 || ^0.15 || ^0.16 || ^0.17 || ^0.18",
"composer-runtime-api": "^2.2",
"flow-php/telemetry": "self.version",
"flow-php/types": "self.version",
Expand Down
10 changes: 7 additions & 3 deletions src/core/etl/src/Flow/ETL/DSL/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@
use Flow\ETL\Schema\Validator\EvolvingValidator;
use Flow\ETL\Schema\Validator\SelectiveValidator;
use Flow\ETL\Schema\Validator\StrictValidator;
use Flow\ETL\Schema\Validator\ValidationContext;
use Flow\ETL\SchemaValidator;
use Flow\ETL\String\StringStyles;
use Flow\ETL\Time\Duration;
Expand Down Expand Up @@ -1900,9 +1901,12 @@ function schema_to_ascii(Schema $schema, ?SchemaFormatter $formatter = null): st
* @param Schema $given
*/
#[DocumentationDSL(module: Module::CORE, type: DSLType::HELPER)]
function schema_validate(Schema $expected, Schema $given, SchemaValidator $validator = new StrictValidator()): bool
{
return $validator->isValid($expected, $given);
function schema_validate(
Schema $expected,
Schema $given,
SchemaValidator $validator = new StrictValidator(),
): ValidationContext {
return $validator->validate($expected, $given);
}

#[DocumentationDSL(module: Module::CORE, type: DSLType::HELPER)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,98 +5,21 @@
namespace Flow\ETL\Exception;

use Flow\ETL\Schema;

use function count;
use Flow\ETL\Schema\Validator\ValidationContext;

final class SchemaValidationException extends RuntimeException
{
public function __construct(
private readonly Schema $expected,
private readonly Schema $given,
private readonly ValidationContext $context,
) {
/**
* @var array<string> $missingDefinitions
*/
$missingDefinitions = [];

/**
* @var array<string> $mismatchedDefinitions
*/
$mismatchedDefinitions = [];

/**
* @var array<string> $unexpectedDefinitions
*/
$unexpectedDefinitions = [];

foreach ($this->expected->definitions() as $expectedDefinition) {
$givenDefinition = $this->given->findDefinition($expectedDefinition->entry());

if ($givenDefinition === null) {
$missingDefinitions[] =
$expectedDefinition->entry()->name()
. '<'
. ($expectedDefinition->isNullable() ? '?' : '')
. $expectedDefinition->type()->toString()
. '>';

continue;
}

if (!$expectedDefinition->isCompatible($givenDefinition)) {
$mismatchedDefinitions[] =
'expected: '
. $expectedDefinition->entry()->name()
. '<'
. $expectedDefinition->type()->toString()
. '>, '
. 'given: '
. $givenDefinition->entry()->name()
. '<'
. ($givenDefinition->isNullable() ? '?' : '')
. $givenDefinition->type()->toString()
. '>';
}
}

foreach ($this->given->definitions() as $givenDefinition) {
if ($this->expected->findDefinition($givenDefinition->entry()) === null) {
$unexpectedDefinitions[] =
$givenDefinition->entry()->name()
. '<'
. ($givenDefinition->isNullable() ? '?' : '')
. $givenDefinition->type()->toString()
. '>';
}
}

$message = '';

if (count($missingDefinitions)) {
$message .= " Missing Definitions: \n";

foreach ($missingDefinitions as $missingDefinition) {
$message .= ' |-- ' . $missingDefinition . "\n";
}
}

if (count($mismatchedDefinitions)) {
$message .= " Mismatched Definitions: \n";

foreach ($mismatchedDefinitions as $mismatchedDefinition) {
$message .= ' |-- ' . $mismatchedDefinition . "\n";
}
}

if (count($unexpectedDefinitions)) {
$message .= " Unexpected Definitions: \n";

foreach ($unexpectedDefinitions as $unexpectedDefinition) {
$message .= ' |-- ' . $unexpectedDefinition . "\n";
}
}
parent::__construct("Schema validation failed: \n" . $this->context->toString());
}

parent::__construct("Schema validation failed: \n" . $message);
public function context(): ValidationContext
{
return $this->context;
}

/**
Expand Down
6 changes: 4 additions & 2 deletions src/core/etl/src/Flow/ETL/Loader/SchemaValidationLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,10 @@ public function load(Rows $rows, FlowContext $context): void
try {
$given = $rows->schema();

if (!$this->validator->isValid($this->expected, $given)) {
throw new SchemaValidationException($this->expected, $given);
$validation = $this->validator->validate($this->expected, $given);

if (!$validation->isValid()) {
throw new SchemaValidationException($this->expected, $given, $validation);
}

$context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]);
Expand Down
35 changes: 14 additions & 21 deletions src/core/etl/src/Flow/ETL/Schema/Validator/EvolvingValidator.php
Original file line number Diff line number Diff line change
Expand Up @@ -20,38 +20,31 @@
*/
final class EvolvingValidator implements SchemaValidator
{
/**
* @param Schema $expected
* @param Schema $given
*/
public function isValid(Schema $expected, Schema $given): bool
public function validate(Schema $expected, Schema $given): ValidationContext
{
if ($given->count() < $expected->count()) {
return false;
}
$missingDefinitions = [];
$mismatchedDefinitions = [];

foreach ($expected->definitions() as $definition) {
if ($given->findDefinition($definition->entry()) === null) {
return false;
}
}
foreach ($expected->definitions() as $expectedDefinition) {
$givenDefinition = $given->findDefinition($expectedDefinition->entry());

foreach ($given->definitions() as $rightDefinition) {
$leftDefinition = $expected->findDefinition($rightDefinition->entry());
if ($givenDefinition === null) {
$missingDefinitions[] = $expectedDefinition;

if ($leftDefinition === null) {
continue;
}

if (!$rightDefinition->isNullable() && $leftDefinition->isNullable()) {
return false;
if (!$givenDefinition->isNullable() && $expectedDefinition->isNullable()) {
$mismatchedDefinitions[] = new MismatchedDefinition($expectedDefinition, $givenDefinition);

continue;
}

if (!type_equals($rightDefinition->type(), $leftDefinition->type())) {
return false;
if (!type_equals($givenDefinition->type(), $expectedDefinition->type())) {
$mismatchedDefinitions[] = new MismatchedDefinition($expectedDefinition, $givenDefinition);
}
}

return true;
return new ValidationContext($missingDefinitions, $mismatchedDefinitions);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?php

declare(strict_types=1);

namespace Flow\ETL\Schema\Validator;

use Flow\ETL\Schema\Definition;

final readonly class MismatchedDefinition
{
/**
* @param Definition<mixed> $expected
* @param Definition<mixed> $given
*/
public function __construct(
private Definition $expected,
private Definition $given,
) {}

/**
* @return Definition<mixed>
*/
public function expected(): Definition
{
return $this->expected;
}

/**
* @return Definition<mixed>
*/
public function given(): Definition
{
return $this->given;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,18 @@
*/
final class SelectiveValidator implements SchemaValidator
{
public function isValid(Schema $expected, Schema $given): bool
public function validate(Schema $expected, Schema $given): ValidationContext
{
$missingDefinitions = [];
$mismatchedDefinitions = [];

foreach ($expected->definitions() as $expectedDefinition) {
$givenDefinition = $given->findDefinition($expectedDefinition->entry());

if ($givenDefinition === null) {
return false;
$missingDefinitions[] = $expectedDefinition;

continue;
}

if (
Expand All @@ -34,10 +39,10 @@ public function isValid(Schema $expected, Schema $given): bool
}

if (!$expectedDefinition->isCompatible($givenDefinition)) {
return false;
$mismatchedDefinitions[] = new MismatchedDefinition($expectedDefinition, $givenDefinition);
}
}

return true;
return new ValidationContext($missingDefinitions, $mismatchedDefinitions);
}
}
Loading
Loading