From 5e1c65c00102c63df396f948a8f5a87366a21946 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 12:36:19 +0100 Subject: [PATCH 01/31] Add new query parsers / compilers / runners --- src/Query/Query.php | 38 ++-- src/Toolkit/Query/AST/ArgumentList.php | 9 + src/Toolkit/Query/AST/ArrayList.php | 9 + src/Toolkit/Query/AST/Closure.php | 12 ++ src/Toolkit/Query/AST/Coalesce.php | 10 ++ src/Toolkit/Query/AST/GlobalFunction.php | 10 ++ src/Toolkit/Query/AST/Literal.php | 9 + src/Toolkit/Query/AST/MemberAccess.php | 12 ++ src/Toolkit/Query/AST/Node.php | 11 ++ src/Toolkit/Query/AST/Ternary.php | 13 ++ src/Toolkit/Query/AST/Variable.php | 9 + src/Toolkit/Query/BaseParser.php | 83 +++++++++ src/Toolkit/Query/Parser.php | 168 ++++++++++++++++++ src/Toolkit/Query/Runner.php | 27 +++ src/Toolkit/Query/Runners/Interpreted.php | 31 ++++ src/Toolkit/Query/Runners/Transpiled.php | 86 +++++++++ .../Query/Runners/Visitors/CodeGen.php | 151 ++++++++++++++++ .../Query/Runners/Visitors/Interpreter.php | 103 +++++++++++ src/Toolkit/Query/Runtime.php | 28 +++ src/Toolkit/Query/Token.php | 11 ++ src/Toolkit/Query/TokenType.php | 26 +++ src/Toolkit/Query/Tokenizer.php | 93 ++++++++++ src/Toolkit/Query/Visitor.php | 29 +++ tests/Query/QueryTest.php | 16 ++ 24 files changed, 983 insertions(+), 11 deletions(-) create mode 100644 src/Toolkit/Query/AST/ArgumentList.php create mode 100644 src/Toolkit/Query/AST/ArrayList.php create mode 100644 src/Toolkit/Query/AST/Closure.php create mode 100644 src/Toolkit/Query/AST/Coalesce.php create mode 100644 src/Toolkit/Query/AST/GlobalFunction.php create mode 100644 src/Toolkit/Query/AST/Literal.php create mode 100644 src/Toolkit/Query/AST/MemberAccess.php create mode 100644 src/Toolkit/Query/AST/Node.php create mode 100644 src/Toolkit/Query/AST/Ternary.php create mode 100644 src/Toolkit/Query/AST/Variable.php create mode 100644 src/Toolkit/Query/BaseParser.php create mode 100644 src/Toolkit/Query/Parser.php create mode 100644 src/Toolkit/Query/Runner.php create mode 100644 src/Toolkit/Query/Runners/Interpreted.php create mode 100644 src/Toolkit/Query/Runners/Transpiled.php create mode 100644 src/Toolkit/Query/Runners/Visitors/CodeGen.php create mode 100644 src/Toolkit/Query/Runners/Visitors/Interpreter.php create mode 100644 src/Toolkit/Query/Runtime.php create mode 100644 src/Toolkit/Query/Token.php create mode 100644 src/Toolkit/Query/TokenType.php create mode 100644 src/Toolkit/Query/Tokenizer.php create mode 100644 src/Toolkit/Query/Visitor.php diff --git a/src/Query/Query.php b/src/Query/Query.php index dc6fca8f4d..f799a70e78 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -11,6 +11,10 @@ use Kirby\Cms\User; use Kirby\Image\QrCode; use Kirby\Toolkit\I18n; +use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Interpreted; +use Kirby\Toolkit\Query\Runners\Transpiled; +use Kirby\Toolkit\Query\Visitor; /** * The Query class can be used to query arrays and objects, @@ -60,24 +64,16 @@ public static function factory(string|null $query): static /** * Method to help classes that extend Query * to intercept a segment's result. + * + * @deprecated 5.0.0 Will be removed in 6.0.0 */ public function intercept(mixed $result): mixed { return $result; } - /** - * Returns the query result if anything - * can be found, otherwise returns null - * - * @throws \Kirby\Exception\BadMethodCallException If an invalid method is accessed by the query - */ - public function resolve(array|object $data = []): mixed + private function resolve_legacy(array|object $data = []): mixed { - if (empty($this->query) === true) { - return $data; - } - // merge data with default entries if (is_array($data) === true) { $data = [...static::$entries, ...$data]; @@ -99,6 +95,26 @@ public function resolve(array|object $data = []): mixed // loop through all segments to resolve query return Expression::factory($this->query, $this)->resolve($data); + + } + + /** + * Returns the query result if anything + * can be found, otherwise returns null + * + * @throws \Kirby\Exception\BadMethodCallException If an invalid method is accessed by the query + */ + public function resolve(array|object $data = []): mixed + { + if (empty($this->query) === true) { + return $data; + } + + return match(option('query.runner', 'interpreted')) { + 'transpiled' => (new Transpiled(static::$entries))->run($this->query, $data), + 'interpreted' => (new Interpreted(static::$entries))->run($this->query, $data), + default => $this->resolve_legacy($data) + }; } } diff --git a/src/Toolkit/Query/AST/ArgumentList.php b/src/Toolkit/Query/AST/ArgumentList.php new file mode 100644 index 0000000000..4976433bd9 --- /dev/null +++ b/src/Toolkit/Query/AST/ArgumentList.php @@ -0,0 +1,9 @@ +visitNode($this); + } +} diff --git a/src/Toolkit/Query/AST/Ternary.php b/src/Toolkit/Query/AST/Ternary.php new file mode 100644 index 0000000000..df18106606 --- /dev/null +++ b/src/Toolkit/Query/AST/Ternary.php @@ -0,0 +1,13 @@ + + */ + protected Iterator $tokens; + + + public function __construct( + Tokenizer|Iterator $source, + ) { + if($source instanceof Tokenizer) { + $this->tokens = $source->tokenize(); + } else { + $this->tokens = $source; + } + + $first = $this->tokens->current(); + + if ($first === null) { + throw new \Exception('No tokens found.'); + } + + $this->current = $first; + } + + protected function consume(TokenType $type, string $message): Token { + if ($this->check($type)) { + return $this->advance(); + } + + throw new \Exception($message); + } + + protected function check(TokenType $type): bool { + if ($this->isAtEnd()) { + return false; + } + + return $this->current->type === $type; + } + + protected function advance(): ?Token { + if (!$this->isAtEnd()) { + $this->previous = $this->current; + $this->tokens->next(); + $this->current = $this->tokens->current(); + } + + return $this->previous; + } + + protected function isAtEnd(): bool { + return $this->current->type === TokenType::EOF; + } + + + protected function match(TokenType $type): Token|false { + if ($this->check($type)) { + return $this->advance(); + } + + return false; + } + + protected function matchAny(array $types): Token|false { + foreach ($types as $type) { + if ($this->check($type)) { + return $this->advance(); + } + } + + return false; + } +} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php new file mode 100644 index 0000000000..7f2e757b02 --- /dev/null +++ b/src/Toolkit/Query/Parser.php @@ -0,0 +1,168 @@ +expression(); + + // ensure that we consumed all tokens + if(!$this->isAtEnd()) + $this->consume(TokenType::EOF, 'Expect end of expression.'); + + return $expression; + } + + private function expression(): Node { + return $this->coalesce(); + } + + private function coalesce(): Node { + $left = $this->ternary(); + + while ($this->match(TokenType::COALESCE)) { + $operator = $this->previous; + $right = $this->ternary(); + $left = new Coalesce($left, $right); + } + + return $left; + } + + private function ternary(): Node { + $left = $this->memberAccess(); + + if ($tok = $this->matchAny([TokenType::QUESTION_MARK, TokenType::TERNARY_DEFAULT])) { + if($tok->type === TokenType::TERNARY_DEFAULT) { + $trueIsDefault = true; + $trueBranch = null; + $falseBranch = $this->expression(); + } else { + $trueIsDefault = false; + $trueBranch = $this->expression(); + $this->consume(TokenType::COLON, 'Expect ":" after true branch.'); + $falseBranch = $this->expression(); + } + + return new Ternary($left, $trueBranch, $falseBranch, $trueIsDefault); + } + + return $left; + } + + private function memberAccess(): Node { + $left = $this->atomic(); + + while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { + $nullSafe = $tok->type === TokenType::NULLSAFE; + + $right = $this->consume(TokenType::IDENTIFIER, 'Expect property name after ".".'); + + if($this->match(TokenType::OPEN_PAREN)) { + $arguments = $this->argumentList(); + $left = new MemberAccess($left, $right->lexeme, $arguments, $nullSafe); + } else { + $left = new MemberAccess($left, $right->lexeme, null, $nullSafe); + } + } + + return $left; + } + + private function listUntil(TokenType $until): array { + $elements = []; + + while (!$this->isAtEnd() && !$this->check($until)) { + $elements[] = $this->expression(); + + if (!$this->match(TokenType::COMMA)) { + break; + } + } + + // consume the closing token + $this->consume($until, 'Expect closing bracket after list.'); + + return $elements; + } + + private function argumentList(): Node { + $list = $this->listUntil(TokenType::CLOSE_PAREN); + + return new ArgumentList($list); + } + + private function atomic(): Node { + // primitives + if ($token = $this->matchAny([ + TokenType::TRUE, + TokenType::FALSE, + TokenType::NULL, + TokenType::STRING, + TokenType::NUMBER, + ])) { + return new Literal($token->literal); + } + + // array literals + if ($token = $this->match(TokenType::OPEN_BRACKET)) { + $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); + + return new ArrayList($arrayItems); + } + + // global functions and variables + if ($token = $this->match(TokenType::IDENTIFIER)) { + if($this->match(TokenType::OPEN_PAREN)) { + $arguments = $this->argumentList(); + return new GlobalFunction($token->lexeme, $arguments); + } + + return new Variable($token->lexeme); + } + + // grouping and closure argument lists + if ($token = $this->match(TokenType::OPEN_PAREN)) { + $list = $this->listUntil(TokenType::CLOSE_PAREN); + + if($this->match(TokenType::ARROW)) { + $expression = $this->expression(); + // check if all elements are variables + foreach($list as $element) { + if(!$element instanceof Variable) { + throw new \Exception('Expecting only variables in closure argument list.'); + } + } + $arguments = new ArgumentList($list); + return new Closure($arguments, $expression); + } else { + if(count($list) > 1) { + throw new \Exception('Expecting \"=>\" after closure argument list.'); + } else { + // this is just a grouping + return $list[0]; + } + } + } + + throw new \Exception('Expect expression.'); + } +} diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php new file mode 100644 index 0000000000..b5294a0840 --- /dev/null +++ b/src/Toolkit/Query/Runner.php @@ -0,0 +1,27 @@ +parse(); + + return self::$cache[$query] = fn(array $binding) => $node->accept(new Visitors\Interpreter($this->allowedFunctions, $binding)); + } + + public function run(string $query, array $bindings = []): mixed { + $resolver = $this->getResolver($query); + return $resolver($bindings); + } +} diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php new file mode 100644 index 0000000000..628343d903 --- /dev/null +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -0,0 +1,86 @@ +parse(); + $codeGen = new Visitors\CodeGen($this->allowedFunctions); + + $functionBody = $node->accept($codeGen); + + $mappings = join("\n", array_map(fn($k, $v) => "$$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); + + $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; + $function = "getResolver($query); + if(!is_callable($function)) { + throw new \Exception("Query is not valid"); + } + return $function($context, $this->allowedFunctions); + } +} diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php new file mode 100644 index 0000000000..a6b42ed8a0 --- /dev/null +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -0,0 +1,151 @@ + $argument->accept($this), $node->arguments); + return join(', ', $arguments); + } + + /** + * Generates code like `[element1, element2, element3]` from an array list node. + */ + public function visitArrayList(ArrayList $node): string { + $elements = array_map(fn($element) => $element->accept($this), $node->elements); + return '[' . join(', ', $elements) . ']'; + } + + /** + * Generates code like `$left ?? $right` from a coalesce node. + */ + public function visitCoalesce(Coalesce $node): string { + $left = $node->left->accept($this); + $right = $node->right->accept($this); + return "($left ?? $right)"; + } + + /** + * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. + */ + public function visitLiteral(Literal $node): string { + return var_export($node->value, true); + } + + /** + * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. + */ + public function visitMemberAccess(MemberAccess $node): string { + $object = $node->object->accept($this); + $member = $node->member; + + $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $memberStr = var_export($member, true); + $nullSafe = $node->nullSafe ? 'true' : 'false'; + + if($node->arguments) { + $arguments = $node->arguments->accept($this); + $member = var_export($member, true); + + return "Runtime::access($object, $memberStr, $nullSafe, $arguments)"; + } + + return "Runtime::access($object, $memberStr, $nullSafe)"; + } + + /** + * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. + */ + public function visitTernary(Ternary $node): string { + $left = $node->condition->accept($this); + $falseBranch = $node->falseBranch->accept($this); + + if($node->trueBranchIsDefault) { + return "($left ?: $falseBranch)"; + } else { + $trueBranch = $node->trueBranch->accept($this); + return "($left ? $trueBranch : $falseBranch)"; + } + } + + public function visitVariable(Variable $node): string { + $name = $node->name; + $namestr = var_export($name, true); + + $key = "_" . crc32($name); + if(isset($this->directAccessFor[$name])) { + return "$$key"; + } + + if(!isset($this->mappings[$key])) { + $this->mappings[$key] = "(match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null })"; + } + + return "\$$key"; + } + + /** + * Generates code like `$functions['function']($arguments)` from a global function node. + */ + public function visitGlobalFunction(GlobalFunction $node): string { + $name = $node->name; + if(!isset($this->validGlobalFunctions[$name])) { + throw new \Exception("Invalid global function $name"); + } + + $arguments = $node->arguments->accept($this); + $name = var_export($name, true); + + return "\$functions[$name]($arguments)"; + } + + public function visitClosure(Closure $node): mixed { + $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + + $names = array_map(fn($n) => $n->name, $node->arguments->arguments); + $args = array_map(fn(string $n) => '$_' . crc32($n), $names); + $args = join(', ', $args); + + $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); + + return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); + } +} diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php new file mode 100644 index 0000000000..034ba33da0 --- /dev/null +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -0,0 +1,103 @@ + $argument->accept($this), $node->arguments); + } + + public function visitArrayList(ArrayList $node): mixed { + return array_map(fn($element) => $element->accept($this), $node->elements); + } + + public function visitCoalesce(Coalesce $node): mixed { + return $node->left->accept($this) ?? $node->right->accept($this); + } + + public function visitLiteral(Literal $node): mixed { + return $node->value; + } + + public function visitMemberAccess(MemberAccess $node): mixed { + $left = $node->object->accept($this); + if($node->arguments !== null) { + return Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + } + return Runtime::access($left, $node->member, $node->nullSafe); + } + + public function visitTernary(Ternary $node): mixed { + if($node->trueBranchIsDefault) { + return $node->condition->accept($this) ?: $node->trueBranch->accept($this); + } else { + return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + } + } + + public function visitVariable(Variable $node): mixed { + // what looks like a variable might actually be a global function + // but if there is a variable with the same name, the variable takes precedence + + if(isset($this->context[$node->name])) { + return $this->context[$node->name]; + } + + if(isset($this->validGlobalFunctions[$node->name])) { + return $this->validGlobalFunctions[$node->name](); + } + + return null; + } + + public function visitGlobalFunction(GlobalFunction $node): mixed { + if(!isset($this->validGlobalFunctions[$node->name])) { + throw new Exception("Invalid global function $node->name"); + } + return $this->validGlobalFunctions[$node->name](...$node->arguments->accept($this)); + } + + public function visitClosure(Closure $node): mixed { + $self = $this; + + return function(...$params) use ($self, $node) { + $context = $self->context; + $functions = $self->validGlobalFunctions; + + $arguments = array_combine( + array_map(fn($param) => $param->name, $node->arguments->arguments), + $params + ); + + $visitor = new self($functions, [...$context, ...$arguments]); + + return $node->body->accept($visitor); + }; + } +} diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php new file mode 100644 index 0000000000..f1bf41e691 --- /dev/null +++ b/src/Toolkit/Query/Runtime.php @@ -0,0 +1,28 @@ +$key(...$arguments); + } + return $object->$key ?? null; + } else { + throw new \Exception("Cannot access \"$key\" on " . gettype($object)); + } + } +} diff --git a/src/Toolkit/Query/Token.php b/src/Toolkit/Query/Token.php new file mode 100644 index 0000000000..0a03c914ad --- /dev/null +++ b/src/Toolkit/Query/Token.php @@ -0,0 +1,11 @@ +length = mb_strlen($source); + } + + /** + * Tokenizes the source string and returns a generator of tokens. + * @return Generator + */ + public function tokenize(): Generator { + $current = 0; + + while ($current < $this->length) { + $t = self::scanToken($this->source, $current); + // don't yield whitespace tokens (ignore them) + if($t->type !== TokenType::WHITESPACE) { + yield $t; + } + $current += mb_strlen($t->lexeme); + } + + yield new Token(TokenType::EOF, '', null); + } + + protected static function scanToken(string $source, int $current): Token { + $l = ''; + $c = $source[$current]; + + return match(true) { + // single character tokens + $c === '.' => new Token(TokenType::DOT, '.'), + $c === '(' => new Token(TokenType::OPEN_PAREN, '('), + $c === ')' => new Token(TokenType::CLOSE_PAREN, ')'), + $c === '[' => new Token(TokenType::OPEN_BRACKET, '['), + $c === ']' => new Token(TokenType::CLOSE_BRACKET, ']'), + $c === ',' => new Token(TokenType::COMMA, ','), + $c === ':' => new Token(TokenType::COLON, ':'), + + // two character tokens + self::match($source, $current, '\\?\\?', $l) => new Token(TokenType::COALESCE, $l), + self::match($source, $current, '\\?\\s*\\.', $l) => new Token(TokenType::NULLSAFE, $l), + self::match($source, $current, '\\?\\s*:', $l) => new Token(TokenType::TERNARY_DEFAULT, $l), + self::match($source, $current, '=>', $l) => new Token(TokenType::ARROW, $l), + + // make sure this check comes after the two above that check for '?' in the beginning + $c === '?' => new Token(TokenType::QUESTION_MARK, '?'), + + // multi character tokens + self::match($source, $current, '\\s+', $l) => new Token(TokenType::WHITESPACE, $l), + self::match($source, $current, 'true', $l, true) => new Token(TokenType::TRUE, $l, true), + self::match($source, $current, 'false', $l, true) => new Token(TokenType::FALSE, $l, false), + self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), + self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '[0-9]+\\.[0-9]+', $l) => new Token(TokenType::NUMBER, $l, floatval($l)), + self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::NUMBER, $l, intval($l)), + self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), + + + // unknown token + default => throw new \Exception("Unexpected character: {$source[$current]}"), + }; + } + + /** + * Checks if a given regex matches the current position in the source. Returns the matched string or false. Advances the current position when a match is found. + * @param string $regex + * @return string|false + */ + protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { + $regex = '/\G' . $regex . '/u'; + if($caseIgnore) { + $regex .= 'i'; + } + $matches = []; + preg_match($regex, $source, $matches, 0, $current); + if (empty($matches[0])) { + return false; + } + $lexeme = $matches[0]; + return true; + } +} diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php new file mode 100644 index 0000000000..f1cedbcc45 --- /dev/null +++ b/src/Toolkit/Query/Visitor.php @@ -0,0 +1,29 @@ +getShortName(); + + $method = 'visit' . $shortName; + if(method_exists($this, $method)) { + return $this->$method($node); + } + + throw new Exception("No visitor method for " . $node::class); + } + + abstract function visitArgumentList(AST\ArgumentList $node): mixed; + abstract function visitArrayList(AST\ArrayList $node): mixed; + abstract function visitCoalesce(AST\Coalesce $node): mixed; + abstract function visitLiteral(AST\Literal $node): mixed; + abstract function visitMemberAccess(AST\MemberAccess $node): mixed; + abstract function visitTernary(AST\Ternary $node): mixed; + abstract function visitVariable(AST\Variable $node): mixed; + abstract function visitGlobalFunction(AST\GlobalFunction $node): mixed; + abstract function visitClosure(AST\Closure $node): mixed; +} diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index c51d961f48..b23f1025bf 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -89,7 +89,23 @@ public function testResolveWithClosureArgument() $bar = $query->resolve($data); $this->assertInstanceOf(Closure::class, $bar); + $bar = $bar(); $this->assertSame('simpson', $bar); } + + /** + * @covers ::resolve + */ + public function testResolveWithClosureWithArgument() + { + $query = new Query('(foo) => foo.homer'); + $data = []; + + $bar = $query->resolve($data); + $this->assertInstanceOf(Closure::class, $bar); + + $bar = $bar(['homer' => 'simpson']); + $this->assertSame('simpson', $bar); + } } From f23a29aafe15356677a80556ac990f513b3e1e0b Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 13:37:55 +0100 Subject: [PATCH 02/31] Fix missing imports of Exception --- src/Toolkit/Query/BaseParser.php | 5 +++-- src/Toolkit/Query/Parser.php | 7 ++++--- src/Toolkit/Query/Runner.php | 3 +-- src/Toolkit/Query/Runners/Visitors/CodeGen.php | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index a2baca5baa..5c4f1a96d8 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; abstract class BaseParser { @@ -26,7 +27,7 @@ public function __construct( $first = $this->tokens->current(); if ($first === null) { - throw new \Exception('No tokens found.'); + throw new Exception('No tokens found.'); } $this->current = $first; @@ -37,7 +38,7 @@ protected function consume(TokenType $type, string $message): Token { return $this->advance(); } - throw new \Exception($message); + throw new Exception($message); } protected function check(TokenType $type): bool { diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 7f2e757b02..8f35b2ecda 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; @@ -148,14 +149,14 @@ private function atomic(): Node { // check if all elements are variables foreach($list as $element) { if(!$element instanceof Variable) { - throw new \Exception('Expecting only variables in closure argument list.'); + throw new Exception('Expecting only variables in closure argument list.'); } } $arguments = new ArgumentList($list); return new Closure($arguments, $expression); } else { if(count($list) > 1) { - throw new \Exception('Expecting \"=>\" after closure argument list.'); + throw new Exception('Expecting \"=>\" after closure argument list.'); } else { // this is just a grouping return $list[0]; @@ -163,6 +164,6 @@ private function atomic(): Node { } } - throw new \Exception('Expect expression.'); + throw new Exception('Expect expression.'); } } diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index b5294a0840..00c9bacf46 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -2,7 +2,6 @@ namespace Kirby\Toolkit\Query; -use Closure; use Exception; abstract class Runner { @@ -21,7 +20,7 @@ public function __construct( * @param string $query The query string to be executed. * @param array $context An optional array of context variables to be passed to the query executor. * @return mixed The result of the executed query. - * @throws \Exception If the query is not valid or the executor is not callable. + * @throws Exception If the query is not valid or the executor is not callable. */ abstract public function run(string $query, array $context = []): mixed; } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index a6b42ed8a0..cebf569498 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Exception; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; use Kirby\Toolkit\Query\AST\Closure; @@ -128,7 +129,7 @@ public function visitVariable(Variable $node): string { public function visitGlobalFunction(GlobalFunction $node): string { $name = $node->name; if(!isset($this->validGlobalFunctions[$name])) { - throw new \Exception("Invalid global function $name"); + throw new Exception("Invalid global function $name"); } $arguments = $node->arguments->accept($this); From ea6025c6b2feeef87167552d8bec6d27b2c0cbfe Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 13:37:55 +0100 Subject: [PATCH 03/31] Fix missing imports of Exception --- src/Toolkit/Query/BaseParser.php | 5 +++-- src/Toolkit/Query/Parser.php | 7 ++++--- src/Toolkit/Query/Runner.php | 3 +-- src/Toolkit/Query/Runners/Transpiled.php | 4 ++-- src/Toolkit/Query/Runners/Visitors/CodeGen.php | 3 ++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index a2baca5baa..5c4f1a96d8 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; abstract class BaseParser { @@ -26,7 +27,7 @@ public function __construct( $first = $this->tokens->current(); if ($first === null) { - throw new \Exception('No tokens found.'); + throw new Exception('No tokens found.'); } $this->current = $first; @@ -37,7 +38,7 @@ protected function consume(TokenType $type, string $message): Token { return $this->advance(); } - throw new \Exception($message); + throw new Exception($message); } protected function check(TokenType $type): bool { diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 7f2e757b02..8f35b2ecda 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; @@ -148,14 +149,14 @@ private function atomic(): Node { // check if all elements are variables foreach($list as $element) { if(!$element instanceof Variable) { - throw new \Exception('Expecting only variables in closure argument list.'); + throw new Exception('Expecting only variables in closure argument list.'); } } $arguments = new ArgumentList($list); return new Closure($arguments, $expression); } else { if(count($list) > 1) { - throw new \Exception('Expecting \"=>\" after closure argument list.'); + throw new Exception('Expecting \"=>\" after closure argument list.'); } else { // this is just a grouping return $list[0]; @@ -163,6 +164,6 @@ private function atomic(): Node { } } - throw new \Exception('Expect expression.'); + throw new Exception('Expect expression.'); } } diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index b5294a0840..00c9bacf46 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -2,7 +2,6 @@ namespace Kirby\Toolkit\Query; -use Closure; use Exception; abstract class Runner { @@ -21,7 +20,7 @@ public function __construct( * @param string $query The query string to be executed. * @param array $context An optional array of context variables to be passed to the query executor. * @return mixed The result of the executed query. - * @throws \Exception If the query is not valid or the executor is not callable. + * @throws Exception If the query is not valid or the executor is not callable. */ abstract public function run(string $query, array $context = []): mixed; } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 628343d903..1800fac7ee 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -74,12 +74,12 @@ protected function getResolver(string $query): Closure { * @param string $query The query string to be executed. * @param array $context An optional array of context variables to be passed to the query executor. * @return mixed The result of the executed query. - * @throws \Exception If the query is not valid or the executor is not callable. + * @throws Exception If the query is not valid or the executor is not callable. */ public function run(string $query, array $context = []): mixed { $function = $this->getResolver($query); if(!is_callable($function)) { - throw new \Exception("Query is not valid"); + throw new Exception("Query is not valid"); } return $function($context, $this->allowedFunctions); } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index a6b42ed8a0..cebf569498 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Exception; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; use Kirby\Toolkit\Query\AST\Closure; @@ -128,7 +129,7 @@ public function visitVariable(Variable $node): string { public function visitGlobalFunction(GlobalFunction $node): string { $name = $node->name; if(!isset($this->validGlobalFunctions[$name])) { - throw new \Exception("Invalid global function $name"); + throw new Exception("Invalid global function $name"); } $arguments = $node->arguments->accept($this); From 54f66b45dbe8d5fd6d9ea73320b9a8fc1d973c41 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 14:30:37 +0100 Subject: [PATCH 04/31] query parser fixes - cast data objects to arrays before passing to query runner - allow integers as identifiers of array keys when used after a dot - don't emit warnings when an array key is missing --- src/Query/Query.php | 4 ++-- src/Toolkit/Query/AST/MemberAccess.php | 2 +- src/Toolkit/Query/Parser.php | 26 +++++++++++++++++++++----- src/Toolkit/Query/Runtime.php | 26 ++++++++++++++++++-------- src/Toolkit/Query/TokenType.php | 2 +- src/Toolkit/Query/Tokenizer.php | 7 +++---- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index f799a70e78..e23d369835 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -111,8 +111,8 @@ public function resolve(array|object $data = []): mixed } return match(option('query.runner', 'interpreted')) { - 'transpiled' => (new Transpiled(static::$entries))->run($this->query, $data), - 'interpreted' => (new Interpreted(static::$entries))->run($this->query, $data), + 'transpiled' => (new Transpiled(static::$entries))->run($this->query, (array)$data), + 'interpreted' => (new Interpreted(static::$entries))->run($this->query, (array)$data), default => $this->resolve_legacy($data) }; } diff --git a/src/Toolkit/Query/AST/MemberAccess.php b/src/Toolkit/Query/AST/MemberAccess.php index c1f21476f4..dce29e4666 100644 --- a/src/Toolkit/Query/AST/MemberAccess.php +++ b/src/Toolkit/Query/AST/MemberAccess.php @@ -5,7 +5,7 @@ class MemberAccess extends Node { public function __construct( public Node $object, - public string $member, + public string|int $member, public ?ArgumentList $arguments = null, public bool $nullSafe = false, ) {} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 8f35b2ecda..73beb7a3b8 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -40,7 +40,6 @@ private function coalesce(): Node { $left = $this->ternary(); while ($this->match(TokenType::COALESCE)) { - $operator = $this->previous; $right = $this->ternary(); $left = new Coalesce($left, $right); } @@ -75,13 +74,19 @@ private function memberAccess(): Node { while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { $nullSafe = $tok->type === TokenType::NULLSAFE; - $right = $this->consume(TokenType::IDENTIFIER, 'Expect property name after ".".'); + if($right = $this->match(TokenType::IDENTIFIER)) { + $right = $right->lexeme; + } else if($right = $this->match(TokenType::INTEGER)) { + $right = $right->literal; + } else { + throw new Exception('Expect property name after ".".'); + } if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccess($left, $right->lexeme, $arguments, $nullSafe); + $left = new MemberAccess($left, $right, $arguments, $nullSafe); } else { - $left = new MemberAccess($left, $right->lexeme, null, $nullSafe); + $left = new MemberAccess($left, $right, null, $nullSafe); } } @@ -111,14 +116,25 @@ private function argumentList(): Node { return new ArgumentList($list); } + + private function atomic(): Node { + + // float numbers + if ($integer = $this->match(TokenType::INTEGER)) { + if($this->match(TokenType::DOT)) { + $fractional = $this->match(TokenType::INTEGER); + return new Literal(floatval($integer->literal . '.' . $fractional->literal)); + } + return new Literal($integer->literal); + } + // primitives if ($token = $this->matchAny([ TokenType::TRUE, TokenType::FALSE, TokenType::NULL, TokenType::STRING, - TokenType::NUMBER, ])) { return new Literal($token->literal); } diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index f1bf41e691..07eadc61d5 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -2,27 +2,37 @@ namespace Kirby\Toolkit\Query; +use Exception; + class Runtime { - static function access($object, $key, bool $nullSafe = false, ...$arguments): mixed { + static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed { if($nullSafe && $object === null) { return null; } if(is_array($object)) { - if($arguments) { - return $object[$key](...$arguments); - } - if($object[$key] instanceof \Closure) { - return $object[$key](); + $item = ($object[$key] ?? $object[(string)$key] ?? null); + + if($item) { + if($arguments) { + return $item(...$arguments); + } + if($item instanceof \Closure) { + return $item(); + } } - return $object[$key] ?? null; + + return $item; } else if(is_object($object)) { + if(is_int($key)) { + $key = (string)$key; + } if(method_exists($object, $key) || method_exists($object, '__call')) { return $object->$key(...$arguments); } return $object->$key ?? null; } else { - throw new \Exception("Cannot access \"$key\" on " . gettype($object)); + throw new Exception("Cannot access \"$key\" on " . gettype($object)); } } } diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index 519b952a74..dcfe904596 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -4,7 +4,7 @@ enum TokenType { case STRING; - case NUMBER; + case INTEGER; case WHITESPACE; case IDENTIFIER; case DOT; diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 48b38d7192..430499931d 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Generator; class Tokenizer { @@ -62,13 +63,11 @@ protected static function scanToken(string $source, int $current): Token { self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '[0-9]+\\.[0-9]+', $l) => new Token(TokenType::NUMBER, $l, floatval($l)), - self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::NUMBER, $l, intval($l)), + self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::INTEGER, $l, intval($l)), self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), - // unknown token - default => throw new \Exception("Unexpected character: {$source[$current]}"), + default => throw new Exception("Unexpected character: {$source[$current]}"), }; } From fa2ed1966e16946133a4af83fed9155e9f586aef Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 16:58:04 +0100 Subject: [PATCH 05/31] Implement `intercept`, rename AST classes with a "Node" suffix - Implements the intercept mechanism for both runners - Renames the AST Node classes with a "Node" suffix to avoid confusion with some PHP internal classes (like `Closure` -> `ClosureNode`) --- src/Query/Query.php | 18 ++-- ...{ArgumentList.php => ArgumentListNode.php} | 2 +- .../AST/{ArrayList.php => ArrayListNode.php} | 2 +- .../AST/{Closure.php => ClosureNode.php} | 4 +- .../AST/{Coalesce.php => CoalesceNode.php} | 2 +- ...balFunction.php => GlobalFunctionNode.php} | 4 +- .../AST/{Literal.php => LiteralNode.php} | 2 +- ...{MemberAccess.php => MemberAccessNode.php} | 4 +- .../AST/{Ternary.php => TernaryNode.php} | 2 +- .../AST/{Variable.php => VariableNode.php} | 2 +- src/Toolkit/Query/Parser.php | 49 +++++----- src/Toolkit/Query/Runner.php | 3 + src/Toolkit/Query/Runners/Interpreted.php | 15 +++- src/Toolkit/Query/Runners/Transpiled.php | 5 +- .../Query/Runners/Visitors/CodeGen.php | 61 +++++++------ .../Query/Runners/Visitors/Interpreter.php | 89 +++++++++++++------ src/Toolkit/Query/Visitor.php | 31 +++++-- tests/Query/QueryTest.php | 32 +++++++ 18 files changed, 216 insertions(+), 111 deletions(-) rename src/Toolkit/Query/AST/{ArgumentList.php => ArgumentListNode.php} (73%) rename src/Toolkit/Query/AST/{ArrayList.php => ArrayListNode.php} (75%) rename src/Toolkit/Query/AST/{Closure.php => ClosureNode.php} (60%) rename src/Toolkit/Query/AST/{Coalesce.php => CoalesceNode.php} (78%) rename src/Toolkit/Query/AST/{GlobalFunction.php => GlobalFunctionNode.php} (57%) rename src/Toolkit/Query/AST/{Literal.php => LiteralNode.php} (75%) rename src/Toolkit/Query/AST/{MemberAccess.php => MemberAccessNode.php} (66%) rename src/Toolkit/Query/AST/{Ternary.php => TernaryNode.php} (86%) rename src/Toolkit/Query/AST/{Variable.php => VariableNode.php} (75%) diff --git a/src/Query/Query.php b/src/Query/Query.php index e23d369835..9f1455c8c7 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -3,6 +3,7 @@ namespace Kirby\Query; use Closure; +use Exception; use Kirby\Cms\App; use Kirby\Cms\Collection; use Kirby\Cms\File; @@ -64,8 +65,6 @@ public static function factory(string|null $query): static /** * Method to help classes that extend Query * to intercept a segment's result. - * - * @deprecated 5.0.0 Will be removed in 6.0.0 */ public function intercept(mixed $result): mixed { @@ -110,11 +109,18 @@ public function resolve(array|object $data = []): mixed return $data; } - return match(option('query.runner', 'interpreted')) { - 'transpiled' => (new Transpiled(static::$entries))->run($this->query, (array)$data), - 'interpreted' => (new Interpreted(static::$entries))->run($this->query, (array)$data), - default => $this->resolve_legacy($data) + if(option('query.runner', 'interpreted') == 'legacy') { + return $this->resolve_legacy($data); + } + + $runnerClass = match(option('query.runner', 'interpreted')) { + 'transpiled' => Transpiled::class, + 'interpreted' => Interpreted::class, + default => throw new Exception('Invalid query runner') }; + + $runner = new $runnerClass(static::$entries, $this->intercept(...)); + return $runner->run($this->query, (array)$data); } } diff --git a/src/Toolkit/Query/AST/ArgumentList.php b/src/Toolkit/Query/AST/ArgumentListNode.php similarity index 73% rename from src/Toolkit/Query/AST/ArgumentList.php rename to src/Toolkit/Query/AST/ArgumentListNode.php index 4976433bd9..1980223203 100644 --- a/src/Toolkit/Query/AST/ArgumentList.php +++ b/src/Toolkit/Query/AST/ArgumentListNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class ArgumentList extends Node { +class ArgumentListNode extends Node { public function __construct( public array $arguments, ) {} diff --git a/src/Toolkit/Query/AST/ArrayList.php b/src/Toolkit/Query/AST/ArrayListNode.php similarity index 75% rename from src/Toolkit/Query/AST/ArrayList.php rename to src/Toolkit/Query/AST/ArrayListNode.php index b5ae063d8a..0b092d7cb2 100644 --- a/src/Toolkit/Query/AST/ArrayList.php +++ b/src/Toolkit/Query/AST/ArrayListNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class ArrayList extends Node { +class ArrayListNode extends Node { public function __construct( public array $elements, ) {} diff --git a/src/Toolkit/Query/AST/Closure.php b/src/Toolkit/Query/AST/ClosureNode.php similarity index 60% rename from src/Toolkit/Query/AST/Closure.php rename to src/Toolkit/Query/AST/ClosureNode.php index edda31b30f..83a26c68de 100644 --- a/src/Toolkit/Query/AST/Closure.php +++ b/src/Toolkit/Query/AST/ClosureNode.php @@ -2,10 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class Closure extends Node +class ClosureNode extends Node { public function __construct( - public ArgumentList $arguments, + public ArgumentListNode $arguments, public Node $body, ) { } diff --git a/src/Toolkit/Query/AST/Coalesce.php b/src/Toolkit/Query/AST/CoalesceNode.php similarity index 78% rename from src/Toolkit/Query/AST/Coalesce.php rename to src/Toolkit/Query/AST/CoalesceNode.php index 63b7d9d445..1873347123 100644 --- a/src/Toolkit/Query/AST/Coalesce.php +++ b/src/Toolkit/Query/AST/CoalesceNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Coalesce extends Node { +class CoalesceNode extends Node { public function __construct( public Node $left, public Node $right, diff --git a/src/Toolkit/Query/AST/GlobalFunction.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php similarity index 57% rename from src/Toolkit/Query/AST/GlobalFunction.php rename to src/Toolkit/Query/AST/GlobalFunctionNode.php index 57a8e037be..ab59bee65e 100644 --- a/src/Toolkit/Query/AST/GlobalFunction.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,9 +2,9 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunction extends Node { +class GlobalFunctionNode extends Node { public function __construct( public string $name, - public ArgumentList $arguments, + public ArgumentListNode $arguments, ) {} } diff --git a/src/Toolkit/Query/AST/Literal.php b/src/Toolkit/Query/AST/LiteralNode.php similarity index 75% rename from src/Toolkit/Query/AST/Literal.php rename to src/Toolkit/Query/AST/LiteralNode.php index bdfe68602d..fc52cfc9eb 100644 --- a/src/Toolkit/Query/AST/Literal.php +++ b/src/Toolkit/Query/AST/LiteralNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Literal extends Node { +class LiteralNode extends Node { public function __construct( public mixed $value, ) {} diff --git a/src/Toolkit/Query/AST/MemberAccess.php b/src/Toolkit/Query/AST/MemberAccessNode.php similarity index 66% rename from src/Toolkit/Query/AST/MemberAccess.php rename to src/Toolkit/Query/AST/MemberAccessNode.php index dce29e4666..7280940abb 100644 --- a/src/Toolkit/Query/AST/MemberAccess.php +++ b/src/Toolkit/Query/AST/MemberAccessNode.php @@ -2,11 +2,11 @@ namespace Kirby\Toolkit\Query\AST; -class MemberAccess extends Node { +class MemberAccessNode extends Node { public function __construct( public Node $object, public string|int $member, - public ?ArgumentList $arguments = null, + public ?ArgumentListNode $arguments = null, public bool $nullSafe = false, ) {} } diff --git a/src/Toolkit/Query/AST/Ternary.php b/src/Toolkit/Query/AST/TernaryNode.php similarity index 86% rename from src/Toolkit/Query/AST/Ternary.php rename to src/Toolkit/Query/AST/TernaryNode.php index df18106606..38e662ba03 100644 --- a/src/Toolkit/Query/AST/Ternary.php +++ b/src/Toolkit/Query/AST/TernaryNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Ternary extends Node { +class TernaryNode extends Node { public function __construct( public Node $condition, public ?Node $trueBranch, diff --git a/src/Toolkit/Query/AST/Variable.php b/src/Toolkit/Query/AST/VariableNode.php similarity index 75% rename from src/Toolkit/Query/AST/Variable.php rename to src/Toolkit/Query/AST/VariableNode.php index 567f15d913..2088743eec 100644 --- a/src/Toolkit/Query/AST/Variable.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Variable extends Node { +class VariableNode extends Node { public function __construct( public string $name, ) {} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 73beb7a3b8..bde8c4d078 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -4,16 +4,16 @@ use Exception; use Iterator; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\GlobalFunction; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\Node; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; class Parser extends BaseParser { public function __construct( @@ -41,7 +41,7 @@ private function coalesce(): Node { while ($this->match(TokenType::COALESCE)) { $right = $this->ternary(); - $left = new Coalesce($left, $right); + $left = new CoalesceNode($left, $right); } return $left; @@ -62,7 +62,7 @@ private function ternary(): Node { $falseBranch = $this->expression(); } - return new Ternary($left, $trueBranch, $falseBranch, $trueIsDefault); + return new TernaryNode($left, $trueBranch, $falseBranch, $trueIsDefault); } return $left; @@ -84,9 +84,9 @@ private function memberAccess(): Node { if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccess($left, $right, $arguments, $nullSafe); + $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); } else { - $left = new MemberAccess($left, $right, null, $nullSafe); + $left = new MemberAccessNode($left, $right, null, $nullSafe); } } @@ -113,20 +113,17 @@ private function listUntil(TokenType $until): array { private function argumentList(): Node { $list = $this->listUntil(TokenType::CLOSE_PAREN); - return new ArgumentList($list); + return new ArgumentListNode($list); } - - private function atomic(): Node { - // float numbers if ($integer = $this->match(TokenType::INTEGER)) { if($this->match(TokenType::DOT)) { $fractional = $this->match(TokenType::INTEGER); - return new Literal(floatval($integer->literal . '.' . $fractional->literal)); + return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); } - return new Literal($integer->literal); + return new LiteralNode($integer->literal); } // primitives @@ -136,24 +133,24 @@ private function atomic(): Node { TokenType::NULL, TokenType::STRING, ])) { - return new Literal($token->literal); + return new LiteralNode($token->literal); } // array literals if ($token = $this->match(TokenType::OPEN_BRACKET)) { $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); - return new ArrayList($arrayItems); + return new ArrayListNode($arrayItems); } // global functions and variables if ($token = $this->match(TokenType::IDENTIFIER)) { if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - return new GlobalFunction($token->lexeme, $arguments); + return new GlobalFunctionNode($token->lexeme, $arguments); } - return new Variable($token->lexeme); + return new VariableNode($token->lexeme); } // grouping and closure argument lists @@ -164,12 +161,12 @@ private function atomic(): Node { $expression = $this->expression(); // check if all elements are variables foreach($list as $element) { - if(!$element instanceof Variable) { + if(!$element instanceof VariableNode) { throw new Exception('Expecting only variables in closure argument list.'); } } - $arguments = new ArgumentList($list); - return new Closure($arguments, $expression); + $arguments = new ArgumentListNode($list); + return new ClosureNode($arguments, $expression); } else { if(count($list) > 1) { throw new Exception('Expecting \"=>\" after closure argument list.'); diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index 00c9bacf46..b327194d44 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; abstract class Runner { @@ -12,6 +13,7 @@ abstract class Runner { */ public function __construct( public array $allowedFunctions = [], + protected Closure|null $interceptor = null, ) {} /** @@ -23,4 +25,5 @@ public function __construct( * @throws Exception If the query is not valid or the executor is not callable. */ abstract public function run(string $query, array $context = []): mixed; + } diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 768a9caeba..055b74b10d 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -10,6 +10,11 @@ class Interpreted extends Runner { private static array $cache = []; + public function __construct( + public array $allowedFunctions = [], + protected Closure|null $interceptor = null, + ) {} + protected function getResolver(string $query): Closure { // load closure from process cache if(isset(self::$cache[$query])) { @@ -21,7 +26,15 @@ protected function getResolver(string $query): Closure { $parser = new Parser($t); $node = $parser->parse(); - return self::$cache[$query] = fn(array $binding) => $node->accept(new Visitors\Interpreter($this->allowedFunctions, $binding)); + $self = $this; + + return self::$cache[$query] = function(array $binding) use ($node, $self) { + $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); + if($self->interceptor !== null) { + $interpreter->setInterceptor($self->interceptor); + } + return $node->accept($interpreter); + }; } public function run(string $query, array $bindings = []): mixed { diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 1800fac7ee..9ca16ffcae 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -19,6 +19,7 @@ class Transpiled extends Runner { */ public function __construct( public array $allowedFunctions = [], + public Closure|null $interceptor = null, ) {} @@ -54,7 +55,7 @@ protected function getResolver(string $query): Closure { $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; - $function = "allowedFunctions); + return $function($context, $this->allowedFunctions, $this->interceptor ?? fn($v) => $v); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index cebf569498..77972c553c 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -3,15 +3,15 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; use Exception; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\GlobalFunction; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; use Kirby\Toolkit\Query\Visitor; @@ -29,20 +29,27 @@ class CodeGen extends Visitor { */ public array $uses = []; + /** + * @var array{string:string} + */ public array $mappings = []; /** * CodeGen constructor. * - * @param array{string:Closure} $validGlobalFunctions An array of valid global function closures. + * @param array{string:PHPClosure} $validGlobalFunctions An array of valid global function closures. */ public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} + private function intercept(string $value): string { + return "(\$intercept($value))"; + } + /** * Generates code like `arg1, arg2, arg3` from an argument list node. */ - public function visitArgumentList(ArgumentList $node): string { + public function visitArgumentList(ArgumentListNode $node): string { $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } @@ -50,7 +57,7 @@ public function visitArgumentList(ArgumentList $node): string { /** * Generates code like `[element1, element2, element3]` from an array list node. */ - public function visitArrayList(ArrayList $node): string { + public function visitArrayList(ArrayListNode $node): string { $elements = array_map(fn($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } @@ -58,7 +65,7 @@ public function visitArrayList(ArrayList $node): string { /** * Generates code like `$left ?? $right` from a coalesce node. */ - public function visitCoalesce(Coalesce $node): string { + public function visitCoalesce(CoalesceNode $node): string { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; @@ -67,14 +74,14 @@ public function visitCoalesce(Coalesce $node): string { /** * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. */ - public function visitLiteral(Literal $node): string { - return var_export($node->value, true); + public function visitLiteral(LiteralNode $node): string { + return '$intercept(' . var_export($node->value, true) . ')'; } /** * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. */ - public function visitMemberAccess(MemberAccess $node): string { + public function visitMemberAccess(MemberAccessNode $node): string { $object = $node->object->accept($this); $member = $node->member; @@ -86,16 +93,16 @@ public function visitMemberAccess(MemberAccess $node): string { $arguments = $node->arguments->accept($this); $member = var_export($member, true); - return "Runtime::access($object, $memberStr, $nullSafe, $arguments)"; + return $this->intercept("Runtime::access($object, $memberStr, $nullSafe, $arguments)"); } - return "Runtime::access($object, $memberStr, $nullSafe)"; + return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } /** * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. */ - public function visitTernary(Ternary $node): string { + public function visitTernary(TernaryNode $node): string { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); @@ -107,17 +114,17 @@ public function visitTernary(Ternary $node): string { } } - public function visitVariable(Variable $node): string { + public function visitVariable(VariableNode $node): string { $name = $node->name; $namestr = var_export($name, true); - $key = "_" . crc32($name); + $key = "_$name"; if(isset($this->directAccessFor[$name])) { - return "$$key"; + return $this->intercept("$$key"); } if(!isset($this->mappings[$key])) { - $this->mappings[$key] = "(match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null })"; + $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } return "\$$key"; @@ -126,7 +133,7 @@ public function visitVariable(Variable $node): string { /** * Generates code like `$functions['function']($arguments)` from a global function node. */ - public function visitGlobalFunction(GlobalFunction $node): string { + public function visitGlobalFunction(GlobalFunctionNode $node): string { $name = $node->name; if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); @@ -135,14 +142,14 @@ public function visitGlobalFunction(GlobalFunction $node): string { $arguments = $node->arguments->accept($this); $name = var_export($name, true); - return "\$functions[$name]($arguments)"; + return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } - public function visitClosure(Closure $node): mixed { + public function visitClosure(ClosureNode $node): mixed { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; $names = array_map(fn($n) => $n->name, $node->arguments->arguments); - $args = array_map(fn(string $n) => '$_' . crc32($n), $names); + $args = array_map(fn(string $n) => "\$_$n", $names); $args = join(', ', $args); $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 034ba33da0..5c72961029 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -2,16 +2,17 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Closure; use Exception; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; -use Kirby\Toolkit\Query\AST\GlobalFunction; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; @@ -29,31 +30,46 @@ public function __construct( public array $context = [] ) {} - public function visitArgumentList(ArgumentList $node): array { + public function visitArgumentList(ArgumentListNode $node): array { return array_map(fn($argument) => $argument->accept($this), $node->arguments); } - public function visitArrayList(ArrayList $node): mixed { + public function visitArrayList(ArrayListNode $node): mixed { return array_map(fn($element) => $element->accept($this), $node->elements); } - public function visitCoalesce(Coalesce $node): mixed { + public function visitCoalesce(CoalesceNode $node): mixed { return $node->left->accept($this) ?? $node->right->accept($this); } - public function visitLiteral(Literal $node): mixed { - return $node->value; + public function visitLiteral(LiteralNode $node): mixed { + $val = $node->value; + + if($this->interceptor !== null) { + $val = ($this->interceptor)($val); + } + + return $val; } - public function visitMemberAccess(MemberAccess $node): mixed { + public function visitMemberAccess(MemberAccessNode $node): mixed { $left = $node->object->accept($this); + + $item = null; if($node->arguments !== null) { - return Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + $item = Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + } else { + $item = Runtime::access($left, $node->member, $node->nullSafe); + } + + if($this->interceptor !== null) { + $item = ($this->interceptor)($item); } - return Runtime::access($left, $node->member, $node->nullSafe); + + return $item; } - public function visitTernary(Ternary $node): mixed { + public function visitTernary(TernaryNode $node): mixed { if($node->trueBranchIsDefault) { return $node->condition->accept($this) ?: $node->trueBranch->accept($this); } else { @@ -61,29 +77,43 @@ public function visitTernary(Ternary $node): mixed { } } - public function visitVariable(Variable $node): mixed { + public function visitVariable(VariableNode $node): mixed { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence - if(isset($this->context[$node->name])) { - return $this->context[$node->name]; - } + $item = match (true) { + isset($this->context[$node->name]) => $this->context[$node->name], + isset($this->validGlobalFunctions[$node->name]) => $this->validGlobalFunctions[$node->name](), + default => null, + }; - if(isset($this->validGlobalFunctions[$node->name])) { - return $this->validGlobalFunctions[$node->name](); + if($this->interceptor !== null) { + $item = ($this->interceptor)($item); } - return null; + return $item; } - public function visitGlobalFunction(GlobalFunction $node): mixed { + public function visitGlobalFunction(GlobalFunctionNode $node): mixed { if(!isset($this->validGlobalFunctions[$node->name])) { throw new Exception("Invalid global function $node->name"); } - return $this->validGlobalFunctions[$node->name](...$node->arguments->accept($this)); + + $function = $this->validGlobalFunctions[$node->name]; + if($this->interceptor !== null) { + $function = ($this->interceptor)($function); + } + + $result = $function(...$node->arguments->accept($this)); + + if($this->interceptor !== null) { + $result = ($this->interceptor)($result); + } + + return $result; } - public function visitClosure(Closure $node): mixed { + public function visitClosure(ClosureNode $node): mixed { $self = $this; return function(...$params) use ($self, $node) { @@ -96,6 +126,9 @@ public function visitClosure(Closure $node): mixed { ); $visitor = new self($functions, [...$context, ...$arguments]); + if($self->interceptor !== null) { + $visitor->setInterceptor($self->interceptor); + } return $node->body->accept($visitor); }; diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index f1cedbcc45..2f24a75a52 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -2,13 +2,19 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; use ReflectionClass; abstract class Visitor { + protected Closure|null $interceptor = null; + function visitNode(AST\Node $node): mixed { $shortName = (new ReflectionClass($node))->getShortName(); + // remove the "Node" suffix + $shortName = substr($shortName, 0, -4); + $method = 'visit' . $shortName; if(method_exists($this, $method)) { return $this->$method($node); @@ -17,13 +23,20 @@ function visitNode(AST\Node $node): mixed { throw new Exception("No visitor method for " . $node::class); } - abstract function visitArgumentList(AST\ArgumentList $node): mixed; - abstract function visitArrayList(AST\ArrayList $node): mixed; - abstract function visitCoalesce(AST\Coalesce $node): mixed; - abstract function visitLiteral(AST\Literal $node): mixed; - abstract function visitMemberAccess(AST\MemberAccess $node): mixed; - abstract function visitTernary(AST\Ternary $node): mixed; - abstract function visitVariable(AST\Variable $node): mixed; - abstract function visitGlobalFunction(AST\GlobalFunction $node): mixed; - abstract function visitClosure(AST\Closure $node): mixed; + abstract function visitArgumentList(AST\ArgumentListNode $node): mixed; + abstract function visitArrayList(AST\ArrayListNode $node): mixed; + abstract function visitCoalesce(AST\CoalesceNode $node): mixed; + abstract function visitLiteral(AST\LiteralNode $node): mixed; + abstract function visitMemberAccess(AST\MemberAccessNode $node): mixed; + abstract function visitTernary(AST\TernaryNode $node): mixed; + abstract function visitVariable(AST\VariableNode $node): mixed; + abstract function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; + abstract function visitClosure(AST\ClosureNode $node): mixed; + + /** + * Sets and activates an interceptor closure that is called for each resolved value. + */ + public function setInterceptor(Closure $interceptor): void { + $this->interceptor = $interceptor; + } } diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index b23f1025bf..e9813ae9dd 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -108,4 +108,36 @@ public function testResolveWithClosureWithArgument() $bar = $bar(['homer' => 'simpson']); $this->assertSame('simpson', $bar); } + + /** + * @covers ::intercept + */ + public function testResolveWithInterceptor() + { + $query = new class extends Query { + public function __construct() + { + parent::__construct('foo.getObj.name'); + } + + public function intercept($result): mixed + { + if(is_object($result) === true) { + $result = clone $result; + $result->name .= ' simpson'; + } + + return $result; + } + }; + + $data = [ + 'foo' => [ + 'getObj' => fn () => (object)['name' => 'homer'] + ] + ]; + + $bar = $query->resolve($data); + $this->assertSame('homer simpson', $bar); + } } From d1dba29ee5fb81200c70e9daa3722fef3de0af1a Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 17:18:11 +0100 Subject: [PATCH 06/31] fix type annotations for arrays in CodeGen and Interpreter classes --- src/Toolkit/Query/Runners/Visitors/CodeGen.php | 7 ++++--- src/Toolkit/Query/Runners/Visitors/Interpreter.php | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 77972c553c..06ba3a606a 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Closure; use Exception; use Kirby\Toolkit\Query\AST\ArgumentListNode; use Kirby\Toolkit\Query\AST\ArrayListNode; @@ -25,19 +26,19 @@ class CodeGen extends Visitor { /** * If we need something from a namespace, we'll add the namespace here into the array key - * @var array{string:true} + * @var array */ public array $uses = []; /** - * @var array{string:string} + * @var array */ public array $mappings = []; /** * CodeGen constructor. * - * @param array{string:PHPClosure} $validGlobalFunctions An array of valid global function closures. + * @param array $validGlobalFunctions An array of valid global function closures. */ public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 5c72961029..34f041d2f6 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -22,8 +22,8 @@ */ class Interpreter extends Visitor { /** - * @param array{string:Closure} $validGlobalFunctions An array of valid global function closures. - * @param array{string:mixed} $context The data bindings for the query. + * @param array $validGlobalFunctions An array of valid global function closures. + * @param array $context The data bindings for the query. */ public function __construct( public array $validGlobalFunctions = [], From dd6eb998d07f8ba96db0483b28d124accfa22447 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 17:20:30 +0100 Subject: [PATCH 07/31] rename parameter in run method from bindings to context to be fully compatible with parent --- src/Toolkit/Query/Runners/Interpreted.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 055b74b10d..455fcb1bec 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -37,8 +37,8 @@ protected function getResolver(string $query): Closure { }; } - public function run(string $query, array $bindings = []): mixed { + public function run(string $query, array $context = []): mixed { $resolver = $this->getResolver($query); - return $resolver($bindings); + return $resolver($context); } } From d56e4d685083f2a70febe7fa74d34f986214ba88 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:44:33 +0100 Subject: [PATCH 08/31] don't use option helper --- src/Query/Query.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 9f1455c8c7..38383f7a77 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -109,11 +109,13 @@ public function resolve(array|object $data = []): mixed return $data; } - if(option('query.runner', 'interpreted') == 'legacy') { + $mode = App::instance()->option('query.runner', 'transpiled'); + + if($mode === 'legacy') { return $this->resolve_legacy($data); } - $runnerClass = match(option('query.runner', 'interpreted')) { + $runnerClass = match($mode) { 'transpiled' => Transpiled::class, 'interpreted' => Interpreted::class, default => throw new Exception('Invalid query runner') From 67aa780b3ae845fbb14a79ecd6cb32fd0f418f22 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:49:23 +0100 Subject: [PATCH 09/31] Update token type constants to include "T_" prefix for safety --- src/Toolkit/Query/BaseParser.php | 2 +- src/Toolkit/Query/Parser.php | 52 ++++++++++---------- src/Toolkit/Query/TokenType.php | 40 ++++++++-------- src/Toolkit/Query/Tokenizer.php | 82 ++++++++++++++++++++++---------- 4 files changed, 104 insertions(+), 72 deletions(-) diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index 5c4f1a96d8..f7cce1e872 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -60,7 +60,7 @@ protected function advance(): ?Token { } protected function isAtEnd(): bool { - return $this->current->type === TokenType::EOF; + return $this->current->type === TokenType::T_EOF; } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index bde8c4d078..01a5e4ecac 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -27,7 +27,7 @@ public function parse(): Node { // ensure that we consumed all tokens if(!$this->isAtEnd()) - $this->consume(TokenType::EOF, 'Expect end of expression.'); + $this->consume(TokenType::T_EOF, 'Expect end of expression.'); return $expression; } @@ -39,7 +39,7 @@ private function expression(): Node { private function coalesce(): Node { $left = $this->ternary(); - while ($this->match(TokenType::COALESCE)) { + while ($this->match(TokenType::T_COALESCE)) { $right = $this->ternary(); $left = new CoalesceNode($left, $right); } @@ -50,15 +50,15 @@ private function coalesce(): Node { private function ternary(): Node { $left = $this->memberAccess(); - if ($tok = $this->matchAny([TokenType::QUESTION_MARK, TokenType::TERNARY_DEFAULT])) { - if($tok->type === TokenType::TERNARY_DEFAULT) { + if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { + if($tok->type === TokenType::T_TERNARY_DEFAULT) { $trueIsDefault = true; $trueBranch = null; $falseBranch = $this->expression(); } else { $trueIsDefault = false; $trueBranch = $this->expression(); - $this->consume(TokenType::COLON, 'Expect ":" after true branch.'); + $this->consume(TokenType::T_COLON, 'Expect ":" after true branch.'); $falseBranch = $this->expression(); } @@ -71,18 +71,18 @@ private function ternary(): Node { private function memberAccess(): Node { $left = $this->atomic(); - while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { - $nullSafe = $tok->type === TokenType::NULLSAFE; + while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { + $nullSafe = $tok->type === TokenType::T_NULLSAFE; - if($right = $this->match(TokenType::IDENTIFIER)) { + if($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } else if($right = $this->match(TokenType::INTEGER)) { + } else if($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); } - if($this->match(TokenType::OPEN_PAREN)) { + if($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); } else { @@ -99,7 +99,7 @@ private function listUntil(TokenType $until): array { while (!$this->isAtEnd() && !$this->check($until)) { $elements[] = $this->expression(); - if (!$this->match(TokenType::COMMA)) { + if (!$this->match(TokenType::T_COMMA)) { break; } } @@ -111,16 +111,16 @@ private function listUntil(TokenType $until): array { } private function argumentList(): Node { - $list = $this->listUntil(TokenType::CLOSE_PAREN); + $list = $this->listUntil(TokenType::T_CLOSE_PAREN); return new ArgumentListNode($list); } private function atomic(): Node { // float numbers - if ($integer = $this->match(TokenType::INTEGER)) { - if($this->match(TokenType::DOT)) { - $fractional = $this->match(TokenType::INTEGER); + if ($integer = $this->match(TokenType::T_INTEGER)) { + if($this->match(TokenType::T_DOT)) { + $fractional = $this->match(TokenType::T_INTEGER); return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); } return new LiteralNode($integer->literal); @@ -128,24 +128,24 @@ private function atomic(): Node { // primitives if ($token = $this->matchAny([ - TokenType::TRUE, - TokenType::FALSE, - TokenType::NULL, - TokenType::STRING, + TokenType::T_TRUE, + TokenType::T_FALSE, + TokenType::T_NULL, + TokenType::T_STRING, ])) { return new LiteralNode($token->literal); } // array literals - if ($token = $this->match(TokenType::OPEN_BRACKET)) { - $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); + if ($token = $this->match(TokenType::T_OPEN_BRACKET)) { + $arrayItems = $this->listUntil(TokenType::T_CLOSE_BRACKET); return new ArrayListNode($arrayItems); } // global functions and variables - if ($token = $this->match(TokenType::IDENTIFIER)) { - if($this->match(TokenType::OPEN_PAREN)) { + if ($token = $this->match(TokenType::T_IDENTIFIER)) { + if($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); return new GlobalFunctionNode($token->lexeme, $arguments); } @@ -154,10 +154,10 @@ private function atomic(): Node { } // grouping and closure argument lists - if ($token = $this->match(TokenType::OPEN_PAREN)) { - $list = $this->listUntil(TokenType::CLOSE_PAREN); + if ($token = $this->match(TokenType::T_OPEN_PAREN)) { + $list = $this->listUntil(TokenType::T_CLOSE_PAREN); - if($this->match(TokenType::ARROW)) { + if($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); // check if all elements are variables foreach($list as $element) { diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index dcfe904596..2cdc14c93a 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -3,24 +3,24 @@ namespace Kirby\Toolkit\Query; enum TokenType { - case STRING; - case INTEGER; - case WHITESPACE; - case IDENTIFIER; - case DOT; - case OPEN_PAREN; - case CLOSE_PAREN; - case OPEN_BRACKET; - case CLOSE_BRACKET; - case QUESTION_MARK; - case TERNARY_DEFAULT; // ?: - case NULLSAFE; // ?. - case COLON; - case COALESCE; // ?? - case COMMA; - case EOF; - case TRUE; - case FALSE; - case NULL; - case ARROW; + case T_STRING; + case T_INTEGER; + case T_WHITESPACE; + case T_IDENTIFIER; + case T_DOT; + case T_OPEN_PAREN; + case T_CLOSE_PAREN; + case T_OPEN_BRACKET; + case T_CLOSE_BRACKET; + case T_QUESTION_MARK; + case T_TERNARY_DEFAULT; // ?: + case T_NULLSAFE; // ?. + case T_COLON; + case T_COALESCE; // ?? + case T_COMMA; + case T_EOF; + case T_TRUE; + case T_FALSE; + case T_NULL; + case T_ARROW; } diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 430499931d..a79f49051b 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -8,6 +8,24 @@ class Tokenizer { private int $length = 0; + /** + * The more complex regexes are written here in nowdoc format so we don't need to double or triple escape backslashes (that becomes ridiculous rather fast). + */ + + // Identifiers can contain letters, numbers, underscores and escaped dots. They can't start with a number. + // to match an array key like "foo.bar" we write the query as `foo\.bar`, to match an array key like "foo\.bar" we write the query as `foo\\.bar` + private const IDENTIFIER_REGEX = <<<'REGEX' + (?:[\p{L}\p{N}_]|\\\.|\\\\)* + REGEX; + + private const SINGLEQUOTE_STRING_REGEX = <<<'REGEX' + '([^'\\]*(?:\\.[^'\\]*)*)' + REGEX; + + private const DOUBLEQUOTE_STRING_REGEX = <<<'REGEX' + "([^"\\]*(?:\\.[^"\\]*)*)" + REGEX; + public function __construct( private readonly string $source, ) { @@ -24,47 +42,55 @@ public function tokenize(): Generator { while ($current < $this->length) { $t = self::scanToken($this->source, $current); // don't yield whitespace tokens (ignore them) - if($t->type !== TokenType::WHITESPACE) { + if($t->type !== TokenType::T_WHITESPACE) { yield $t; } $current += mb_strlen($t->lexeme); } - yield new Token(TokenType::EOF, '', null); + yield new Token(TokenType::T_EOF, '', null); } + /** + * Scans the source string for a token starting at the given position. + * @param string $source The source string + * @param int $current The current position in the source string + * + * @return Token The scanned token + * @throws Exception If an unexpected character is encountered + */ protected static function scanToken(string $source, int $current): Token { $l = ''; $c = $source[$current]; return match(true) { // single character tokens - $c === '.' => new Token(TokenType::DOT, '.'), - $c === '(' => new Token(TokenType::OPEN_PAREN, '('), - $c === ')' => new Token(TokenType::CLOSE_PAREN, ')'), - $c === '[' => new Token(TokenType::OPEN_BRACKET, '['), - $c === ']' => new Token(TokenType::CLOSE_BRACKET, ']'), - $c === ',' => new Token(TokenType::COMMA, ','), - $c === ':' => new Token(TokenType::COLON, ':'), + $c === '.' => new Token(TokenType::T_DOT, '.'), + $c === '(' => new Token(TokenType::T_OPEN_PAREN, '('), + $c === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), + $c === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), + $c === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), + $c === ',' => new Token(TokenType::T_COMMA, ','), + $c === ':' => new Token(TokenType::T_COLON, ':'), // two character tokens - self::match($source, $current, '\\?\\?', $l) => new Token(TokenType::COALESCE, $l), - self::match($source, $current, '\\?\\s*\\.', $l) => new Token(TokenType::NULLSAFE, $l), - self::match($source, $current, '\\?\\s*:', $l) => new Token(TokenType::TERNARY_DEFAULT, $l), - self::match($source, $current, '=>', $l) => new Token(TokenType::ARROW, $l), + self::match($source, $current, '\?\?', $l) => new Token(TokenType::T_COALESCE, $l), + self::match($source, $current, '\?\s*\.', $l) => new Token(TokenType::T_NULLSAFE, $l), + self::match($source, $current, '\?\s*:', $l) => new Token(TokenType::T_TERNARY_DEFAULT, $l), + self::match($source, $current, '=>', $l) => new Token(TokenType::T_ARROW, $l), // make sure this check comes after the two above that check for '?' in the beginning - $c === '?' => new Token(TokenType::QUESTION_MARK, '?'), + $c === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), // multi character tokens - self::match($source, $current, '\\s+', $l) => new Token(TokenType::WHITESPACE, $l), - self::match($source, $current, 'true', $l, true) => new Token(TokenType::TRUE, $l, true), - self::match($source, $current, 'false', $l, true) => new Token(TokenType::FALSE, $l, false), - self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), - self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::INTEGER, $l, intval($l)), - self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), + self::match($source, $current, '\s+', $l) => new Token(TokenType::T_WHITESPACE, $l), + self::match($source, $current, 'true', $l, true) => new Token(TokenType::T_TRUE, $l, true), + self::match($source, $current, 'false', $l, true) => new Token(TokenType::T_FALSE, $l, false), + self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), + self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, intval($l)), + self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), // unknown token default => throw new Exception("Unexpected character: {$source[$current]}"), @@ -72,9 +98,15 @@ protected static function scanToken(string $source, int $current): Token { } /** - * Checks if a given regex matches the current position in the source. Returns the matched string or false. Advances the current position when a match is found. - * @param string $regex - * @return string|false + * Matches a regex pattern at the current position in the source string. + * The matched lexeme will be stored in the $lexeme variable. + * + * @param string $source The source string + * @param int $current The current position in the source string (used as offset for the regex) + * @param string $regex The regex pattern to match (without delimiters / flags) + * @param string $lexeme The matched lexeme will be stored in this variable + * @param bool $caseIgnore Whether to ignore case while matching + * @return bool Whether the regex pattern was matched */ protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { $regex = '/\G' . $regex . '/u'; From f46ab0a131396a155758694ad52ef2e85d0d180b Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:50:20 +0100 Subject: [PATCH 10/31] allow identifiers to start with a number and contain (escaped) dots --- src/Toolkit/Query/AST/ClosureNode.php | 5 +- src/Toolkit/Query/AST/GlobalFunctionNode.php | 9 +++- src/Toolkit/Query/AST/IdentifierNode.php | 12 +++++ src/Toolkit/Query/AST/MemberAccessNode.php | 13 ++++- src/Toolkit/Query/AST/VariableNode.php | 9 +++- src/Toolkit/Query/Parser.php | 9 +++- src/Toolkit/Query/Runners/Transpiled.php | 2 +- .../Query/Runners/Visitors/CodeGen.php | 49 ++++++++----------- .../Query/Runners/Visitors/Interpreter.php | 17 ++++--- tests/Query/QueryTest.php | 17 ++++++- 10 files changed, 99 insertions(+), 43 deletions(-) create mode 100644 src/Toolkit/Query/AST/IdentifierNode.php diff --git a/src/Toolkit/Query/AST/ClosureNode.php b/src/Toolkit/Query/AST/ClosureNode.php index 83a26c68de..8eb1022845 100644 --- a/src/Toolkit/Query/AST/ClosureNode.php +++ b/src/Toolkit/Query/AST/ClosureNode.php @@ -4,8 +4,11 @@ class ClosureNode extends Node { + /** + * @param string[] $arguments The arguments names + */ public function __construct( - public ArgumentListNode $arguments, + public array $arguments, public Node $body, ) { } diff --git a/src/Toolkit/Query/AST/GlobalFunctionNode.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php index ab59bee65e..22674c8d49 100644 --- a/src/Toolkit/Query/AST/GlobalFunctionNode.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,9 +2,16 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunctionNode extends Node { +class GlobalFunctionNode extends IdentifierNode { public function __construct( public string $name, public ArgumentListNode $arguments, ) {} + + /** + * Replace escaped dots with real dots + */ + public function name(): string { + return str_replace('\.', '.', $this->name); + } } diff --git a/src/Toolkit/Query/AST/IdentifierNode.php b/src/Toolkit/Query/AST/IdentifierNode.php new file mode 100644 index 0000000000..e394576637 --- /dev/null +++ b/src/Toolkit/Query/AST/IdentifierNode.php @@ -0,0 +1,12 @@ +member)) { + return self::unescape($this->member); + } else { + return $this->member; + } + } } diff --git a/src/Toolkit/Query/AST/VariableNode.php b/src/Toolkit/Query/AST/VariableNode.php index 2088743eec..8ab46c2826 100644 --- a/src/Toolkit/Query/AST/VariableNode.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,8 +2,15 @@ namespace Kirby\Toolkit\Query\AST; -class VariableNode extends Node { +class VariableNode extends IdentifierNode { public function __construct( public string $name, ) {} + + /** + * Replaces escaped dots with real dots + */ + public function name(): string { + return self::unescape($this->name); + } } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 01a5e4ecac..7785c3424b 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -159,13 +159,18 @@ private function atomic(): Node { if($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); - // check if all elements are variables + + /** + * Assert that all elements are VariableNodes + * @var VariableNode[] $list + */ foreach($list as $element) { if(!$element instanceof VariableNode) { throw new Exception('Expecting only variables in closure argument list.'); } } - $arguments = new ArgumentListNode($list); + + $arguments = array_map(fn($element) => $element->name, $list); return new ClosureNode($arguments, $expression); } else { if(count($list) > 1) { diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 9ca16ffcae..0b67ef88f0 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -51,7 +51,7 @@ protected function getResolver(string $query): Closure { $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn($k, $v) => "$$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $mappings = join("\n", array_map(fn($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 06ba3a606a..13cc9f41d6 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -35,6 +35,19 @@ class CodeGen extends Visitor { */ public array $mappings = []; + + /** + * Variable names in Query Language are different from PHP variable names, + * they can start with a number and may contain escaped dots. + * + * This method returns a sanitized PHP variable name. + * + * @return string + */ + private static function phpName(string $name): string { + return '$_' . crc32($name); + } + /** * CodeGen constructor. * @@ -46,42 +59,26 @@ private function intercept(string $value): string { return "(\$intercept($value))"; } - - /** - * Generates code like `arg1, arg2, arg3` from an argument list node. - */ public function visitArgumentList(ArgumentListNode $node): string { $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } - /** - * Generates code like `[element1, element2, element3]` from an array list node. - */ public function visitArrayList(ArrayListNode $node): string { $elements = array_map(fn($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } - /** - * Generates code like `$left ?? $right` from a coalesce node. - */ public function visitCoalesce(CoalesceNode $node): string { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } - /** - * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. - */ public function visitLiteral(LiteralNode $node): string { return '$intercept(' . var_export($node->value, true) . ')'; } - /** - * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. - */ public function visitMemberAccess(MemberAccessNode $node): string { $object = $node->object->accept($this); $member = $node->member; @@ -100,9 +97,6 @@ public function visitMemberAccess(MemberAccessNode $node): string { return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } - /** - * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. - */ public function visitTernary(TernaryNode $node): string { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); @@ -116,26 +110,26 @@ public function visitTernary(TernaryNode $node): string { } public function visitVariable(VariableNode $node): string { - $name = $node->name; + $name = $node->name(); $namestr = var_export($name, true); - $key = "_$name"; + $key = self::phpName($name); if(isset($this->directAccessFor[$name])) { - return $this->intercept("$$key"); + return $this->intercept($key); } if(!isset($this->mappings[$key])) { - $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); + $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) && \$context[$namestr] instanceof Closure => \$context[$namestr](), isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } - return "\$$key"; + return $key; } /** * Generates code like `$functions['function']($arguments)` from a global function node. */ public function visitGlobalFunction(GlobalFunctionNode $node): string { - $name = $node->name; + $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); } @@ -149,11 +143,10 @@ public function visitGlobalFunction(GlobalFunctionNode $node): string { public function visitClosure(ClosureNode $node): mixed { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; - $names = array_map(fn($n) => $n->name, $node->arguments->arguments); - $args = array_map(fn(string $n) => "\$_$n", $names); + $args = array_map(self::phpName(...), $node->arguments); $args = join(', ', $args); - $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); + $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($node->arguments, true)); return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); } diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 34f041d2f6..c800b8ea3b 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -81,9 +81,11 @@ public function visitVariable(VariableNode $node): mixed { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence + $name = $node->name(); + $item = match (true) { - isset($this->context[$node->name]) => $this->context[$node->name], - isset($this->validGlobalFunctions[$node->name]) => $this->validGlobalFunctions[$node->name](), + isset($this->context[$name]) => $this->context[$name] instanceof Closure ? $this->context[$name]() : $this->context[$name], + isset($this->validGlobalFunctions[$name]) => $this->validGlobalFunctions[$name](), default => null, }; @@ -95,11 +97,13 @@ public function visitVariable(VariableNode $node): mixed { } public function visitGlobalFunction(GlobalFunctionNode $node): mixed { - if(!isset($this->validGlobalFunctions[$node->name])) { - throw new Exception("Invalid global function $node->name"); + $name = $node->name(); + + if(!isset($this->validGlobalFunctions[$name])) { + throw new Exception("Invalid global function $name"); } - $function = $this->validGlobalFunctions[$node->name]; + $function = $this->validGlobalFunctions[$name]; if($this->interceptor !== null) { $function = ($this->interceptor)($function); } @@ -120,8 +124,9 @@ public function visitClosure(ClosureNode $node): mixed { $context = $self->context; $functions = $self->validGlobalFunctions; + // [key1, key2] + [value1, value2] => [key1 => value1, key2 => value2] $arguments = array_combine( - array_map(fn($param) => $param->name, $node->arguments->arguments), + $node->arguments, $params ); diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index e9813ae9dd..5ba87109aa 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -67,11 +67,24 @@ public function testResolveWithExactArrayMatch() $query = new Query('user'); $this->assertSame('homer', $query->resolve(['user' => 'homer'])); - $query = new Query('user.username'); + $query = new Query('user\.username'); $this->assertSame('homer', $query->resolve(['user.username' => 'homer'])); - $query = new Query('user.callback'); + $query = new Query('user\.callback'); $this->assertSame('homer', $query->resolve(['user.callback' => fn () => 'homer'])); + + // in the query, the first slash escapes the second, the third escapes the dot + $query = <<<'TXT' + user\\\.username + TXT; + + // this is actually the array key + $key = <<<'TXT' + user\.username + TXT; + + $query = new Query($query); + $this->assertSame('homer', $query->resolve([$key => 'homer'])); } /** From c68b85f6c2c90e933a8285af56f759b60874ec8c Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 13:46:50 +0100 Subject: [PATCH 11/31] coding style --- src/Toolkit/Query/AST/ArgumentListNode.php | 6 ++- src/Toolkit/Query/AST/ArrayListNode.php | 6 ++- src/Toolkit/Query/AST/CoalesceNode.php | 6 ++- src/Toolkit/Query/AST/GlobalFunctionNode.php | 9 ++-- src/Toolkit/Query/AST/IdentifierNode.php | 6 ++- src/Toolkit/Query/AST/LiteralNode.php | 6 ++- src/Toolkit/Query/AST/MemberAccessNode.php | 15 +++--- src/Toolkit/Query/AST/Node.php | 6 ++- src/Toolkit/Query/AST/TernaryNode.php | 9 ++-- src/Toolkit/Query/AST/VariableNode.php | 9 ++-- src/Toolkit/Query/BaseParser.php | 23 +++++--- src/Toolkit/Query/Parser.php | 50 ++++++++++------- src/Toolkit/Query/Runner.php | 6 ++- src/Toolkit/Query/Runners/Interpreted.php | 14 +++-- src/Toolkit/Query/Runners/Transpiled.php | 22 ++++---- .../Query/Runners/Visitors/CodeGen.php | 54 +++++++++++-------- .../Query/Runners/Visitors/Interpreter.php | 46 +++++++++------- src/Toolkit/Query/Runtime.php | 13 +++-- src/Toolkit/Query/Token.php | 6 ++- src/Toolkit/Query/TokenType.php | 3 +- src/Toolkit/Query/Tokenizer.php | 14 +++-- src/Toolkit/Query/Visitor.php | 29 +++++----- tests/Query/QueryTest.php | 7 +-- 23 files changed, 221 insertions(+), 144 deletions(-) diff --git a/src/Toolkit/Query/AST/ArgumentListNode.php b/src/Toolkit/Query/AST/ArgumentListNode.php index 1980223203..b6d9d3bf83 100644 --- a/src/Toolkit/Query/AST/ArgumentListNode.php +++ b/src/Toolkit/Query/AST/ArgumentListNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class ArgumentListNode extends Node { +class ArgumentListNode extends Node +{ public function __construct( public array $arguments, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/ArrayListNode.php b/src/Toolkit/Query/AST/ArrayListNode.php index 0b092d7cb2..a60ce194fb 100644 --- a/src/Toolkit/Query/AST/ArrayListNode.php +++ b/src/Toolkit/Query/AST/ArrayListNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class ArrayListNode extends Node { +class ArrayListNode extends Node +{ public function __construct( public array $elements, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/CoalesceNode.php b/src/Toolkit/Query/AST/CoalesceNode.php index 1873347123..9a216ac648 100644 --- a/src/Toolkit/Query/AST/CoalesceNode.php +++ b/src/Toolkit/Query/AST/CoalesceNode.php @@ -2,9 +2,11 @@ namespace Kirby\Toolkit\Query\AST; -class CoalesceNode extends Node { +class CoalesceNode extends Node +{ public function __construct( public Node $left, public Node $right, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/GlobalFunctionNode.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php index 22674c8d49..2f0da7cd0b 100644 --- a/src/Toolkit/Query/AST/GlobalFunctionNode.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,16 +2,19 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunctionNode extends IdentifierNode { +class GlobalFunctionNode extends IdentifierNode +{ public function __construct( public string $name, public ArgumentListNode $arguments, - ) {} + ) { + } /** * Replace escaped dots with real dots */ - public function name(): string { + public function name(): string + { return str_replace('\.', '.', $this->name); } } diff --git a/src/Toolkit/Query/AST/IdentifierNode.php b/src/Toolkit/Query/AST/IdentifierNode.php index e394576637..5c07dcd325 100644 --- a/src/Toolkit/Query/AST/IdentifierNode.php +++ b/src/Toolkit/Query/AST/IdentifierNode.php @@ -2,11 +2,13 @@ namespace Kirby\Toolkit\Query\AST; -abstract class IdentifierNode extends Node { +abstract class IdentifierNode extends Node +{ /** * Replaces the escaped identifier with the actual identifier */ - static public function unescape(string $name): string { + public static function unescape(string $name): string + { return stripslashes($name); } } diff --git a/src/Toolkit/Query/AST/LiteralNode.php b/src/Toolkit/Query/AST/LiteralNode.php index fc52cfc9eb..7976f127b3 100644 --- a/src/Toolkit/Query/AST/LiteralNode.php +++ b/src/Toolkit/Query/AST/LiteralNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class LiteralNode extends Node { +class LiteralNode extends Node +{ public function __construct( public mixed $value, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/MemberAccessNode.php b/src/Toolkit/Query/AST/MemberAccessNode.php index deaa069931..7bcc5ec51e 100644 --- a/src/Toolkit/Query/AST/MemberAccessNode.php +++ b/src/Toolkit/Query/AST/MemberAccessNode.php @@ -2,22 +2,25 @@ namespace Kirby\Toolkit\Query\AST; -class MemberAccessNode extends IdentifierNode { +class MemberAccessNode extends IdentifierNode +{ public function __construct( public Node $object, public string|int $member, - public ?ArgumentListNode $arguments = null, + public ArgumentListNode|null $arguments = null, public bool $nullSafe = false, - ) {} + ) { + } /** * Returns the member name and replaces escaped dots with real dots if it's a string */ - public function member(): string|int { + public function member(): string|int + { if (is_string($this->member)) { return self::unescape($this->member); - } else { - return $this->member; } + return $this->member; + } } diff --git a/src/Toolkit/Query/AST/Node.php b/src/Toolkit/Query/AST/Node.php index c7507d6d99..3da71795da 100644 --- a/src/Toolkit/Query/AST/Node.php +++ b/src/Toolkit/Query/AST/Node.php @@ -4,8 +4,10 @@ use Kirby\Toolkit\Query\Visitor; -class Node { - public function accept(Visitor $visitor) { +class Node +{ + public function accept(Visitor $visitor) + { return $visitor->visitNode($this); } } diff --git a/src/Toolkit/Query/AST/TernaryNode.php b/src/Toolkit/Query/AST/TernaryNode.php index 38e662ba03..33aa203175 100644 --- a/src/Toolkit/Query/AST/TernaryNode.php +++ b/src/Toolkit/Query/AST/TernaryNode.php @@ -2,12 +2,13 @@ namespace Kirby\Toolkit\Query\AST; -class TernaryNode extends Node { +class TernaryNode extends Node +{ public function __construct( public Node $condition, - public ?Node $trueBranch, + public Node|null $trueBranch, public Node $falseBranch, - public bool $trueBranchIsDefault = false, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/VariableNode.php b/src/Toolkit/Query/AST/VariableNode.php index 8ab46c2826..495bd3979f 100644 --- a/src/Toolkit/Query/AST/VariableNode.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,15 +2,18 @@ namespace Kirby\Toolkit\Query\AST; -class VariableNode extends IdentifierNode { +class VariableNode extends IdentifierNode +{ public function __construct( public string $name, - ) {} + ) { + } /** * Replaces escaped dots with real dots */ - public function name(): string { + public function name(): string + { return self::unescape($this->name); } } diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index f7cce1e872..b597cf391a 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -5,8 +5,9 @@ use Exception; use Iterator; -abstract class BaseParser { - protected ?Token $previous; +abstract class BaseParser +{ + protected Token|null $previous; protected Token $current; /** @@ -33,7 +34,8 @@ public function __construct( $this->current = $first; } - protected function consume(TokenType $type, string $message): Token { + protected function consume(TokenType $type, string $message): Token + { if ($this->check($type)) { return $this->advance(); } @@ -41,7 +43,8 @@ protected function consume(TokenType $type, string $message): Token { throw new Exception($message); } - protected function check(TokenType $type): bool { + protected function check(TokenType $type): bool + { if ($this->isAtEnd()) { return false; } @@ -49,7 +52,8 @@ protected function check(TokenType $type): bool { return $this->current->type === $type; } - protected function advance(): ?Token { + protected function advance(): Token|null + { if (!$this->isAtEnd()) { $this->previous = $this->current; $this->tokens->next(); @@ -59,12 +63,14 @@ protected function advance(): ?Token { return $this->previous; } - protected function isAtEnd(): bool { + protected function isAtEnd(): bool + { return $this->current->type === TokenType::T_EOF; } - protected function match(TokenType $type): Token|false { + protected function match(TokenType $type): Token|false + { if ($this->check($type)) { return $this->advance(); } @@ -72,7 +78,8 @@ protected function match(TokenType $type): Token|false { return false; } - protected function matchAny(array $types): Token|false { + protected function matchAny(array $types): Token|false + { foreach ($types as $type) { if ($this->check($type)) { return $this->advance(); diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 7785c3424b..8be677f1bf 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -15,28 +15,33 @@ use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; -class Parser extends BaseParser { +class Parser extends BaseParser +{ public function __construct( Tokenizer|Iterator $source, ) { parent::__construct($source); } - public function parse(): Node { + public function parse(): Node + { $expression = $this->expression(); // ensure that we consumed all tokens - if(!$this->isAtEnd()) + if(!$this->isAtEnd()) { $this->consume(TokenType::T_EOF, 'Expect end of expression.'); + } return $expression; } - private function expression(): Node { + private function expression(): Node + { return $this->coalesce(); } - private function coalesce(): Node { + private function coalesce(): Node + { $left = $this->ternary(); while ($this->match(TokenType::T_COALESCE)) { @@ -47,7 +52,8 @@ private function coalesce(): Node { return $left; } - private function ternary(): Node { + private function ternary(): Node + { $left = $this->memberAccess(); if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { @@ -68,7 +74,8 @@ private function ternary(): Node { return $left; } - private function memberAccess(): Node { + private function memberAccess(): Node + { $left = $this->atomic(); while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { @@ -76,7 +83,7 @@ private function memberAccess(): Node { if($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } else if($right = $this->match(TokenType::T_INTEGER)) { + } elseif($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); @@ -93,7 +100,8 @@ private function memberAccess(): Node { return $left; } - private function listUntil(TokenType $until): array { + private function listUntil(TokenType $until): array + { $elements = []; while (!$this->isAtEnd() && !$this->check($until)) { @@ -110,18 +118,20 @@ private function listUntil(TokenType $until): array { return $elements; } - private function argumentList(): Node { + private function argumentList(): Node + { $list = $this->listUntil(TokenType::T_CLOSE_PAREN); return new ArgumentListNode($list); } - private function atomic(): Node { + private function atomic(): Node + { // float numbers if ($integer = $this->match(TokenType::T_INTEGER)) { if($this->match(TokenType::T_DOT)) { $fractional = $this->match(TokenType::T_INTEGER); - return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); + return new LiteralNode((float)($integer->literal . '.' . $fractional->literal)); } return new LiteralNode($integer->literal); } @@ -170,16 +180,16 @@ private function atomic(): Node { } } - $arguments = array_map(fn($element) => $element->name, $list); + $arguments = array_map(fn ($element) => $element->name, $list); return new ClosureNode($arguments, $expression); - } else { - if(count($list) > 1) { - throw new Exception('Expecting \"=>\" after closure argument list.'); - } else { - // this is just a grouping - return $list[0]; - } } + if(count($list) > 1) { + throw new Exception('Expecting \"=>\" after closure argument list.'); + } + // this is just a grouping + return $list[0]; + + } throw new Exception('Expect expression.'); diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index b327194d44..d01b295660 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -5,7 +5,8 @@ use Closure; use Exception; -abstract class Runner { +abstract class Runner +{ /** * Runner constructor. * @@ -14,7 +15,8 @@ abstract class Runner { public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, - ) {} + ) { + } /** * Executes a query within a given data context. diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 455fcb1bec..9ecf15aa03 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -7,15 +7,18 @@ use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Tokenizer; -class Interpreted extends Runner { +class Interpreted extends Runner +{ private static array $cache = []; public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, - ) {} + ) { + } - protected function getResolver(string $query): Closure { + protected function getResolver(string $query): Closure + { // load closure from process cache if(isset(self::$cache[$query])) { return self::$cache[$query]; @@ -28,7 +31,7 @@ protected function getResolver(string $query): Closure { $self = $this; - return self::$cache[$query] = function(array $binding) use ($node, $self) { + return self::$cache[$query] = function (array $binding) use ($node, $self) { $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); if($self->interceptor !== null) { $interpreter->setInterceptor($self->interceptor); @@ -37,7 +40,8 @@ protected function getResolver(string $query): Closure { }; } - public function run(string $query, array $context = []): mixed { + public function run(string $query, array $context = []): mixed + { $resolver = $this->getResolver($query); return $resolver($context); } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 0b67ef88f0..1d88e2042f 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -8,7 +8,8 @@ use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Tokenizer; -class Transpiled extends Runner { +class Transpiled extends Runner +{ private static array $cache = []; public static string $cacheFolder = '/tmp/query_cache'; @@ -20,7 +21,8 @@ class Transpiled extends Runner { public function __construct( public array $allowedFunctions = [], public Closure|null $interceptor = null, - ) {} + ) { + } /** @@ -30,7 +32,8 @@ public function __construct( * @param string $query The query string to be executed. * @return Closure The executor closure for the given query. */ - protected function getResolver(string $query): Closure { + protected function getResolver(string $query): Closure + { // load closure from process memory if(isset(self::$cache[$query])) { return self::$cache[$query]; @@ -51,10 +54,10 @@ protected function getResolver(string $query): Closure { $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; - $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); + $mappings = join("\n", array_map(fn ($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $comment = join("\n", array_map(fn ($l) => "// $l", explode("\n", $query))); - $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; + $uses = join("\n", array_map(fn ($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; $function = "getResolver($query); if(!is_callable($function)) { - throw new Exception("Query is not valid"); + throw new Exception('Query is not valid'); } - return $function($context, $this->allowedFunctions, $this->interceptor ?? fn($v) => $v); + return $function($context, $this->allowedFunctions, $this->interceptor ?? fn ($v) => $v); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 13cc9f41d6..8b8c44defe 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -15,15 +15,14 @@ use Kirby\Toolkit\Query\AST\VariableNode; use Kirby\Toolkit\Query\Visitor; - /** * Visitor that generates code representations from query structures. * * The `CodeGen` class traverses query nodes and generates corresponding PHP code. * It extends the base `Visitor` class, providing implementations specific to code generation. */ -class CodeGen extends Visitor { - +class CodeGen extends Visitor +{ /** * If we need something from a namespace, we'll add the namespace here into the array key * @var array @@ -41,10 +40,9 @@ class CodeGen extends Visitor { * they can start with a number and may contain escaped dots. * * This method returns a sanitized PHP variable name. - * - * @return string */ - private static function phpName(string $name): string { + private static function phpName(string $name): string + { return '$_' . crc32($name); } @@ -53,33 +51,41 @@ private static function phpName(string $name): string { * * @param array $validGlobalFunctions An array of valid global function closures. */ - public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} + public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor = []) + { + } - private function intercept(string $value): string { + private function intercept(string $value): string + { return "(\$intercept($value))"; } - public function visitArgumentList(ArgumentListNode $node): string { - $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); + public function visitArgumentList(ArgumentListNode $node): string + { + $arguments = array_map(fn ($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } - public function visitArrayList(ArrayListNode $node): string { - $elements = array_map(fn($element) => $element->accept($this), $node->elements); + public function visitArrayList(ArrayListNode $node): string + { + $elements = array_map(fn ($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } - public function visitCoalesce(CoalesceNode $node): string { + public function visitCoalesce(CoalesceNode $node): string + { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } - public function visitLiteral(LiteralNode $node): string { + public function visitLiteral(LiteralNode $node): string + { return '$intercept(' . var_export($node->value, true) . ')'; } - public function visitMemberAccess(MemberAccessNode $node): string { + public function visitMemberAccess(MemberAccessNode $node): string + { $object = $node->object->accept($this); $member = $node->member; @@ -97,19 +103,21 @@ public function visitMemberAccess(MemberAccessNode $node): string { return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } - public function visitTernary(TernaryNode $node): string { + public function visitTernary(TernaryNode $node): string + { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); if($node->trueBranchIsDefault) { return "($left ?: $falseBranch)"; - } else { - $trueBranch = $node->trueBranch->accept($this); - return "($left ? $trueBranch : $falseBranch)"; } + $trueBranch = $node->trueBranch->accept($this); + return "($left ? $trueBranch : $falseBranch)"; + } - public function visitVariable(VariableNode $node): string { + public function visitVariable(VariableNode $node): string + { $name = $node->name(); $namestr = var_export($name, true); @@ -128,7 +136,8 @@ public function visitVariable(VariableNode $node): string { /** * Generates code like `$functions['function']($arguments)` from a global function node. */ - public function visitGlobalFunction(GlobalFunctionNode $node): string { + public function visitGlobalFunction(GlobalFunctionNode $node): string + { $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); @@ -140,7 +149,8 @@ public function visitGlobalFunction(GlobalFunctionNode $node): string { return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } - public function visitClosure(ClosureNode $node): mixed { + public function visitClosure(ClosureNode $node): mixed + { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; $args = array_map(self::phpName(...), $node->arguments); diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index c800b8ea3b..484c1fa02e 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -8,19 +8,19 @@ use Kirby\Toolkit\Query\AST\ArrayListNode; use Kirby\Toolkit\Query\AST\ClosureNode; use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\AST\LiteralNode; use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; -use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; - /** * Visitor that interprets and directly executes a query AST. */ -class Interpreter extends Visitor { +class Interpreter extends Visitor +{ /** * @param array $validGlobalFunctions An array of valid global function closures. * @param array $context The data bindings for the query. @@ -28,21 +28,26 @@ class Interpreter extends Visitor { public function __construct( public array $validGlobalFunctions = [], public array $context = [] - ) {} + ) { + } - public function visitArgumentList(ArgumentListNode $node): array { - return array_map(fn($argument) => $argument->accept($this), $node->arguments); + public function visitArgumentList(ArgumentListNode $node): array + { + return array_map(fn ($argument) => $argument->accept($this), $node->arguments); } - public function visitArrayList(ArrayListNode $node): mixed { - return array_map(fn($element) => $element->accept($this), $node->elements); + public function visitArrayList(ArrayListNode $node): mixed + { + return array_map(fn ($element) => $element->accept($this), $node->elements); } - public function visitCoalesce(CoalesceNode $node): mixed { + public function visitCoalesce(CoalesceNode $node): mixed + { return $node->left->accept($this) ?? $node->right->accept($this); } - public function visitLiteral(LiteralNode $node): mixed { + public function visitLiteral(LiteralNode $node): mixed + { $val = $node->value; if($this->interceptor !== null) { @@ -52,7 +57,8 @@ public function visitLiteral(LiteralNode $node): mixed { return $val; } - public function visitMemberAccess(MemberAccessNode $node): mixed { + public function visitMemberAccess(MemberAccessNode $node): mixed + { $left = $node->object->accept($this); $item = null; @@ -69,15 +75,17 @@ public function visitMemberAccess(MemberAccessNode $node): mixed { return $item; } - public function visitTernary(TernaryNode $node): mixed { + public function visitTernary(TernaryNode $node): mixed + { if($node->trueBranchIsDefault) { return $node->condition->accept($this) ?: $node->trueBranch->accept($this); - } else { - return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); } + return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + } - public function visitVariable(VariableNode $node): mixed { + public function visitVariable(VariableNode $node): mixed + { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence @@ -96,7 +104,8 @@ public function visitVariable(VariableNode $node): mixed { return $item; } - public function visitGlobalFunction(GlobalFunctionNode $node): mixed { + public function visitGlobalFunction(GlobalFunctionNode $node): mixed + { $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { @@ -117,10 +126,11 @@ public function visitGlobalFunction(GlobalFunctionNode $node): mixed { return $result; } - public function visitClosure(ClosureNode $node): mixed { + public function visitClosure(ClosureNode $node): mixed + { $self = $this; - return function(...$params) use ($self, $node) { + return function (...$params) use ($self, $node) { $context = $self->context; $functions = $self->validGlobalFunctions; diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index 07eadc61d5..9c102b741f 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -4,8 +4,10 @@ use Exception; -class Runtime { - static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed { +class Runtime +{ + public static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed + { if($nullSafe && $object === null) { return null; } @@ -23,7 +25,8 @@ static function access(array|object|null $object, string|int $key, bool $nullSaf } return $item; - } else if(is_object($object)) { + } + if(is_object($object)) { if(is_int($key)) { $key = (string)$key; } @@ -31,8 +34,8 @@ static function access(array|object|null $object, string|int $key, bool $nullSaf return $object->$key(...$arguments); } return $object->$key ?? null; - } else { - throw new Exception("Cannot access \"$key\" on " . gettype($object)); } + throw new Exception("Cannot access \"$key\" on " . gettype($object)); + } } diff --git a/src/Toolkit/Query/Token.php b/src/Toolkit/Query/Token.php index 0a03c914ad..99c0a405a7 100644 --- a/src/Toolkit/Query/Token.php +++ b/src/Toolkit/Query/Token.php @@ -2,10 +2,12 @@ namespace Kirby\Toolkit\Query; -class Token { +class Token +{ public function __construct( public TokenType $type, public string $lexeme, public mixed $literal = null, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index 2cdc14c93a..eade2dcc28 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -2,7 +2,8 @@ namespace Kirby\Toolkit\Query; -enum TokenType { +enum TokenType +{ case T_STRING; case T_INTEGER; case T_WHITESPACE; diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index a79f49051b..993be68de0 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -5,7 +5,8 @@ use Exception; use Generator; -class Tokenizer { +class Tokenizer +{ private int $length = 0; /** @@ -36,7 +37,8 @@ public function __construct( * Tokenizes the source string and returns a generator of tokens. * @return Generator */ - public function tokenize(): Generator { + public function tokenize(): Generator + { $current = 0; while ($current < $this->length) { @@ -59,7 +61,8 @@ public function tokenize(): Generator { * @return Token The scanned token * @throws Exception If an unexpected character is encountered */ - protected static function scanToken(string $source, int $current): Token { + protected static function scanToken(string $source, int $current): Token + { $l = ''; $c = $source[$current]; @@ -89,7 +92,7 @@ protected static function scanToken(string $source, int $current): Token { self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, intval($l)), + self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, (int)$l), self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), // unknown token @@ -108,7 +111,8 @@ protected static function scanToken(string $source, int $current): Token { * @param bool $caseIgnore Whether to ignore case while matching * @return bool Whether the regex pattern was matched */ - protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { + protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool + { $regex = '/\G' . $regex . '/u'; if($caseIgnore) { $regex .= 'i'; diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index 2f24a75a52..41db64ddc3 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -6,10 +6,12 @@ use Exception; use ReflectionClass; -abstract class Visitor { +abstract class Visitor +{ protected Closure|null $interceptor = null; - function visitNode(AST\Node $node): mixed { + public function visitNode(AST\Node $node): mixed + { $shortName = (new ReflectionClass($node))->getShortName(); // remove the "Node" suffix @@ -20,23 +22,24 @@ function visitNode(AST\Node $node): mixed { return $this->$method($node); } - throw new Exception("No visitor method for " . $node::class); + throw new Exception('No visitor method for ' . $node::class); } - abstract function visitArgumentList(AST\ArgumentListNode $node): mixed; - abstract function visitArrayList(AST\ArrayListNode $node): mixed; - abstract function visitCoalesce(AST\CoalesceNode $node): mixed; - abstract function visitLiteral(AST\LiteralNode $node): mixed; - abstract function visitMemberAccess(AST\MemberAccessNode $node): mixed; - abstract function visitTernary(AST\TernaryNode $node): mixed; - abstract function visitVariable(AST\VariableNode $node): mixed; - abstract function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; - abstract function visitClosure(AST\ClosureNode $node): mixed; + abstract public function visitArgumentList(AST\ArgumentListNode $node): mixed; + abstract public function visitArrayList(AST\ArrayListNode $node): mixed; + abstract public function visitCoalesce(AST\CoalesceNode $node): mixed; + abstract public function visitLiteral(AST\LiteralNode $node): mixed; + abstract public function visitMemberAccess(AST\MemberAccessNode $node): mixed; + abstract public function visitTernary(AST\TernaryNode $node): mixed; + abstract public function visitVariable(AST\VariableNode $node): mixed; + abstract public function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; + abstract public function visitClosure(AST\ClosureNode $node): mixed; /** * Sets and activates an interceptor closure that is called for each resolved value. */ - public function setInterceptor(Closure $interceptor): void { + public function setInterceptor(Closure $interceptor): void + { $this->interceptor = $interceptor; } } diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index 5ba87109aa..486d4ddb2f 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -127,12 +127,7 @@ public function testResolveWithClosureWithArgument() */ public function testResolveWithInterceptor() { - $query = new class extends Query { - public function __construct() - { - parent::__construct('foo.getObj.name'); - } - + $query = new class ('foo.getObj.name') extends Query { public function intercept($result): mixed { if(is_object($result) === true) { From 52d94fd3952e6e01d981e7d9a1692f71bb27c340 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 13:53:16 +0100 Subject: [PATCH 12/31] Remove unused imports in Query.php --- src/Query/Query.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 38383f7a77..6e77d1746a 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -12,10 +12,8 @@ use Kirby\Cms\User; use Kirby\Image\QrCode; use Kirby\Toolkit\I18n; -use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Runners\Interpreted; use Kirby\Toolkit\Query\Runners\Transpiled; -use Kirby\Toolkit\Query\Visitor; /** * The Query class can be used to query arrays and objects, From a512a8ea4c566705790b196dc409bb5f9e8e5a02 Mon Sep 17 00:00:00 2001 From: Nico Hoffmann Date: Wed, 13 Nov 2024 18:03:50 +0100 Subject: [PATCH 13/31] First set of CS fixes --- src/Query/Query.php | 2 +- src/Toolkit/Query/BaseParser.php | 19 ++-- src/Toolkit/Query/Parser.php | 63 +++++++------- src/Toolkit/Query/Runners/Interpreted.php | 15 ++-- src/Toolkit/Query/Runners/Transpiled.php | 44 +++++++--- .../Query/Runners/Visitors/CodeGen.php | 68 ++++++++++----- .../Query/Runners/Visitors/Interpreter.php | 52 +++++++---- src/Toolkit/Query/Runtime.php | 36 +++++--- src/Toolkit/Query/Tokenizer.php | 86 ++++++++++++------- src/Toolkit/Query/Visitor.php | 34 +++++--- 10 files changed, 262 insertions(+), 157 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 6e77d1746a..043c0a9875 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -109,7 +109,7 @@ public function resolve(array|object $data = []): mixed $mode = App::instance()->option('query.runner', 'transpiled'); - if($mode === 'legacy') { + if ($mode === 'legacy') { return $this->resolve_legacy($data); } diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index b597cf391a..7e3fc459c4 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -19,13 +19,12 @@ abstract class BaseParser public function __construct( Tokenizer|Iterator $source, ) { - if($source instanceof Tokenizer) { - $this->tokens = $source->tokenize(); - } else { - $this->tokens = $source; + if ($source instanceof Tokenizer) { + $source = $source->tokenize(); } - $first = $this->tokens->current(); + $this->tokens = $source; + $first = $this->tokens->current(); if ($first === null) { throw new Exception('No tokens found.'); @@ -36,7 +35,7 @@ public function __construct( protected function consume(TokenType $type, string $message): Token { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } @@ -45,7 +44,7 @@ protected function consume(TokenType $type, string $message): Token protected function check(TokenType $type): bool { - if ($this->isAtEnd()) { + if ($this->isAtEnd() === true) { return false; } @@ -54,7 +53,7 @@ protected function check(TokenType $type): bool protected function advance(): Token|null { - if (!$this->isAtEnd()) { + if ($this->isAtEnd() === false) { $this->previous = $this->current; $this->tokens->next(); $this->current = $this->tokens->current(); @@ -71,7 +70,7 @@ protected function isAtEnd(): bool protected function match(TokenType $type): Token|false { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } @@ -81,7 +80,7 @@ protected function match(TokenType $type): Token|false protected function matchAny(array $types): Token|false { foreach ($types as $type) { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 8be677f1bf..f5b4801120 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -3,7 +3,6 @@ namespace Kirby\Toolkit\Query; use Exception; -use Iterator; use Kirby\Toolkit\Query\AST\ArgumentListNode; use Kirby\Toolkit\Query\AST\ArrayListNode; use Kirby\Toolkit\Query\AST\ClosureNode; @@ -17,18 +16,12 @@ class Parser extends BaseParser { - public function __construct( - Tokenizer|Iterator $source, - ) { - parent::__construct($source); - } - public function parse(): Node { $expression = $this->expression(); // ensure that we consumed all tokens - if(!$this->isAtEnd()) { + if ($this->isAtEnd() === false) { $this->consume(TokenType::T_EOF, 'Expect end of expression.'); } @@ -46,7 +39,7 @@ private function coalesce(): Node while ($this->match(TokenType::T_COALESCE)) { $right = $this->ternary(); - $left = new CoalesceNode($left, $right); + $left = new CoalesceNode($left, $right); } return $left; @@ -57,18 +50,23 @@ private function ternary(): Node $left = $this->memberAccess(); if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { - if($tok->type === TokenType::T_TERNARY_DEFAULT) { + if ($tok->type === TokenType::T_TERNARY_DEFAULT) { $trueIsDefault = true; - $trueBranch = null; - $falseBranch = $this->expression(); + $trueBranch = null; } else { $trueIsDefault = false; - $trueBranch = $this->expression(); + $trueBranch = $this->expression(); $this->consume(TokenType::T_COLON, 'Expect ":" after true branch.'); - $falseBranch = $this->expression(); } - return new TernaryNode($left, $trueBranch, $falseBranch, $trueIsDefault); + $falseBranch = $this->expression(); + + return new TernaryNode( + $left, + $trueBranch, + $falseBranch, + $trueIsDefault + ); } return $left; @@ -81,20 +79,24 @@ private function memberAccess(): Node while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { $nullSafe = $tok->type === TokenType::T_NULLSAFE; - if($right = $this->match(TokenType::T_IDENTIFIER)) { + if ($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } elseif($right = $this->match(TokenType::T_INTEGER)) { + } elseif ($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); } - if($this->match(TokenType::T_OPEN_PAREN)) { + if ($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); - } else { - $left = new MemberAccessNode($left, $right, null, $nullSafe); } + + $left = new MemberAccessNode( + $left, + $right, + $arguments ?? null, + $nullSafe + ); } return $left; @@ -104,10 +106,10 @@ private function listUntil(TokenType $until): array { $elements = []; - while (!$this->isAtEnd() && !$this->check($until)) { + while ($this->isAtEnd() === false && $this->check($until) === false) { $elements[] = $this->expression(); - if (!$this->match(TokenType::T_COMMA)) { + if ($this->match(TokenType::T_COMMA) == false) { break; } } @@ -129,10 +131,11 @@ private function atomic(): Node { // float numbers if ($integer = $this->match(TokenType::T_INTEGER)) { - if($this->match(TokenType::T_DOT)) { + if ($this->match(TokenType::T_DOT)) { $fractional = $this->match(TokenType::T_INTEGER); return new LiteralNode((float)($integer->literal . '.' . $fractional->literal)); } + return new LiteralNode($integer->literal); } @@ -155,7 +158,7 @@ private function atomic(): Node // global functions and variables if ($token = $this->match(TokenType::T_IDENTIFIER)) { - if($this->match(TokenType::T_OPEN_PAREN)) { + if ($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); return new GlobalFunctionNode($token->lexeme, $arguments); } @@ -167,7 +170,7 @@ private function atomic(): Node if ($token = $this->match(TokenType::T_OPEN_PAREN)) { $list = $this->listUntil(TokenType::T_CLOSE_PAREN); - if($this->match(TokenType::T_ARROW)) { + if ($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); /** @@ -175,7 +178,7 @@ private function atomic(): Node * @var VariableNode[] $list */ foreach($list as $element) { - if(!$element instanceof VariableNode) { + if ($element instanceof VariableNode === false) { throw new Exception('Expecting only variables in closure argument list.'); } } @@ -183,13 +186,13 @@ private function atomic(): Node $arguments = array_map(fn ($element) => $element->name, $list); return new ClosureNode($arguments, $expression); } - if(count($list) > 1) { + + if (count($list) > 1) { throw new Exception('Expecting \"=>\" after closure argument list.'); } + // this is just a grouping return $list[0]; - - } throw new Exception('Expect expression.'); diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 9ecf15aa03..139ed9ba77 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -5,6 +5,7 @@ use Closure; use Kirby\Toolkit\Query\Parser; use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Visitors\Interpreter; use Kirby\Toolkit\Query\Tokenizer; class Interpreted extends Runner @@ -20,22 +21,24 @@ public function __construct( protected function getResolver(string $query): Closure { // load closure from process cache - if(isset(self::$cache[$query])) { + if (isset(self::$cache[$query])) { return self::$cache[$query]; } // on cache miss, parse query and generate closure - $t = new Tokenizer($query); - $parser = new Parser($t); - $node = $parser->parse(); + $tokenizer = new Tokenizer($query); + $parser = new Parser($tokenizer); + $node = $parser->parse(); $self = $this; return self::$cache[$query] = function (array $binding) use ($node, $self) { - $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); - if($self->interceptor !== null) { + $interpreter = new Interpreter($self->allowedFunctions, $binding); + + if ($self->interceptor !== null) { $interpreter->setInterceptor($self->interceptor); } + return $node->accept($interpreter); }; } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 1d88e2042f..09a3d5fbfd 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -6,6 +6,7 @@ use Exception; use Kirby\Toolkit\Query\Parser; use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Visitors\CodeGen; use Kirby\Toolkit\Query\Tokenizer; class Transpiled extends Runner @@ -24,7 +25,6 @@ public function __construct( ) { } - /** * Retrieves the executor closure for a given query. * If the closure is not already cached, it will be generated and stored in `Runner::$cacheFolder`. @@ -35,33 +35,43 @@ public function __construct( protected function getResolver(string $query): Closure { // load closure from process memory - if(isset(self::$cache[$query])) { + if (isset(self::$cache[$query])) { return self::$cache[$query]; } // load closure from file-cache / opcache - $hash = crc32($query); + $hash = crc32($query); $filename = self::$cacheFolder . '/' . $hash . '.php'; - if(file_exists($filename)) { + + if (file_exists($filename)) { return self::$cache[$query] = include $filename; } // on cache miss, parse query and generate closure - $t = new Tokenizer($query); - $parser = new Parser($t); - $node = $parser->parse(); - $codeGen = new Visitors\CodeGen($this->allowedFunctions); + $tokenizer = new Tokenizer($query); + $parser = new Parser($tokenizer); + $node = $parser->parse(); + $codeGen = new CodeGen($this->allowedFunctions); $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn ($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; - $comment = join("\n", array_map(fn ($l) => "// $l", explode("\n", $query))); + $mappings = array_map( + fn ($k, $v) => "$k = $v;", + array_keys($codeGen->mappings), + $codeGen->mappings + ); + $mappings = join("\n", $mappings) . "\n"; + + $comment = array_map(fn ($l) => "// $l", explode("\n", $query)); + $comment = join("\n", $comment); + + $uses = array_map(fn ($k) => "use $k;", array_keys($codeGen->uses)); + $uses = join("\n", $uses) . "\n"; - $uses = join("\n", array_map(fn ($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; $function = "getResolver($query); - if(!is_callable($function)) { + + if (is_callable($function) === false) { throw new Exception('Query is not valid'); } - return $function($context, $this->allowedFunctions, $this->interceptor ?? fn ($v) => $v); + + return $function( + $context, + $this->allowedFunctions, + $this->interceptor ?? fn ($v) => $v + ); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 8b8c44defe..d28a766779 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -13,6 +13,7 @@ use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; +use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; /** @@ -51,8 +52,10 @@ private static function phpName(string $name): string * * @param array $validGlobalFunctions An array of valid global function closures. */ - public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor = []) - { + public function __construct( + public array $validGlobalFunctions = [], + public array $directAccessFor = [] + ) { } private function intercept(string $value): string @@ -62,19 +65,27 @@ private function intercept(string $value): string public function visitArgumentList(ArgumentListNode $node): string { - $arguments = array_map(fn ($argument) => $argument->accept($this), $node->arguments); + $arguments = array_map( + fn ($argument) => $argument->accept($this), + $node->arguments + ); + return join(', ', $arguments); } public function visitArrayList(ArrayListNode $node): string { - $elements = array_map(fn ($element) => $element->accept($this), $node->elements); + $elements = array_map( + fn ($element) => $element->accept($this), + $node->elements + ); + return '[' . join(', ', $elements) . ']'; } public function visitCoalesce(CoalesceNode $node): string { - $left = $node->left->accept($this); + $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } @@ -89,28 +100,33 @@ public function visitMemberAccess(MemberAccessNode $node): string $object = $node->object->accept($this); $member = $node->member; - $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $this->uses[Runtime::class] = true; $memberStr = var_export($member, true); - $nullSafe = $node->nullSafe ? 'true' : 'false'; + $nullSafe = $node->nullSafe ? 'true' : 'false'; - if($node->arguments) { + if ($node->arguments) { $arguments = $node->arguments->accept($this); - $member = var_export($member, true); + $member = var_export($member, true); - return $this->intercept("Runtime::access($object, $memberStr, $nullSafe, $arguments)"); + return $this->intercept( + "Runtime::access($object, $memberStr, $nullSafe, $arguments)" + ); } - return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); + return $this->intercept( + "Runtime::access($object, $memberStr, $nullSafe)" + ); } public function visitTernary(TernaryNode $node): string { - $left = $node->condition->accept($this); + $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); - if($node->trueBranchIsDefault) { + if ($node->trueBranchIsDefault === true) { return "($left ?: $falseBranch)"; } + $trueBranch = $node->trueBranch->accept($this); return "($left ? $trueBranch : $falseBranch)"; @@ -118,15 +134,15 @@ public function visitTernary(TernaryNode $node): string public function visitVariable(VariableNode $node): string { - $name = $node->name(); + $name = $node->name(); $namestr = var_export($name, true); + $key = static::phpName($name); - $key = self::phpName($name); - if(isset($this->directAccessFor[$name])) { + if (isset($this->directAccessFor[$name])) { return $this->intercept($key); } - if(!isset($this->mappings[$key])) { + if (isset($this->mappings[$key]) === false) { $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) && \$context[$namestr] instanceof Closure => \$context[$namestr](), isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } @@ -139,25 +155,31 @@ public function visitVariable(VariableNode $node): string public function visitGlobalFunction(GlobalFunctionNode $node): string { $name = $node->name(); - if(!isset($this->validGlobalFunctions[$name])) { + + if (isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); } $arguments = $node->arguments->accept($this); - $name = var_export($name, true); + $name = var_export($name, true); return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } public function visitClosure(ClosureNode $node): mixed { - $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $this->uses[Runtime::class] = true; - $args = array_map(self::phpName(...), $node->arguments); + $args = array_map(static::phpName(...), $node->arguments); $args = join(', ', $args); - $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($node->arguments, true)); + $newDirectAccessFor = [ + ...$this->directAccessFor, + ...array_fill_keys($node->arguments, true) + ]; - return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); + return "fn($args) => " . $node->body->accept( + new static($this->validGlobalFunctions, $newDirectAccessFor) + ); } } diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 484c1fa02e..01c1e74507 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -33,12 +33,18 @@ public function __construct( public function visitArgumentList(ArgumentListNode $node): array { - return array_map(fn ($argument) => $argument->accept($this), $node->arguments); + return array_map( + fn ($argument) => $argument->accept($this), + $node->arguments + ); } public function visitArrayList(ArrayListNode $node): mixed { - return array_map(fn ($element) => $element->accept($this), $node->elements); + return array_map( + fn ($element) => $element->accept($this), + $node->elements + ); } public function visitCoalesce(CoalesceNode $node): mixed @@ -50,7 +56,7 @@ public function visitLiteral(LiteralNode $node): mixed { $val = $node->value; - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $val = ($this->interceptor)($val); } @@ -60,15 +66,20 @@ public function visitLiteral(LiteralNode $node): mixed public function visitMemberAccess(MemberAccessNode $node): mixed { $left = $node->object->accept($this); - $item = null; - if($node->arguments !== null) { - $item = Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + + if ($node->arguments !== null) { + $item = Runtime::access( + $left, + $node->member, + $node->nullSafe, + ...$node->arguments->accept($this) + ); } else { $item = Runtime::access($left, $node->member, $node->nullSafe); } - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $item = ($this->interceptor)($item); } @@ -77,11 +88,16 @@ public function visitMemberAccess(MemberAccessNode $node): mixed public function visitTernary(TernaryNode $node): mixed { - if($node->trueBranchIsDefault) { - return $node->condition->accept($this) ?: $node->trueBranch->accept($this); + if ($node->trueBranchIsDefault === true) { + return + $node->condition->accept($this) ?: + $node->trueBranch->accept($this); } - return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + return + $node->condition->accept($this) ? + $node->trueBranch->accept($this) : + $node->falseBranch->accept($this); } public function visitVariable(VariableNode $node): mixed @@ -97,7 +113,7 @@ public function visitVariable(VariableNode $node): mixed default => null, }; - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $item = ($this->interceptor)($item); } @@ -108,18 +124,19 @@ public function visitGlobalFunction(GlobalFunctionNode $node): mixed { $name = $node->name(); - if(!isset($this->validGlobalFunctions[$name])) { + if (isset($this->validGlobalFunctions[$name]) === false) { throw new Exception("Invalid global function $name"); } $function = $this->validGlobalFunctions[$name]; - if($this->interceptor !== null) { + + if ($this->interceptor !== null) { $function = ($this->interceptor)($function); } $result = $function(...$node->arguments->accept($this)); - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $result = ($this->interceptor)($result); } @@ -131,7 +148,7 @@ public function visitClosure(ClosureNode $node): mixed $self = $this; return function (...$params) use ($self, $node) { - $context = $self->context; + $context = $self->context; $functions = $self->validGlobalFunctions; // [key1, key2] + [value1, value2] => [key1 => value1, key2 => value2] @@ -140,8 +157,9 @@ public function visitClosure(ClosureNode $node): mixed $params ); - $visitor = new self($functions, [...$context, ...$arguments]); - if($self->interceptor !== null) { + $visitor = new static($functions, [...$context, ...$arguments]); + + if ($self->interceptor !== null) { $visitor->setInterceptor($self->interceptor); } diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index 9c102b741f..66be6a59cf 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -2,40 +2,50 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; class Runtime { - public static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed - { - if($nullSafe && $object === null) { + public static function access( + array|object|null $object, + string|int $key, + bool $nullSafe = false, + ...$arguments + ): mixed { + if ($nullSafe === true && $object === null) { return null; } - if(is_array($object)) { - $item = ($object[$key] ?? $object[(string)$key] ?? null); - - if($item) { - if($arguments) { + if (is_array($object)) { + if ($item = $object[$key] ?? $object[(string)$key] ?? null) { + if ($arguments) { return $item(...$arguments); } - if($item instanceof \Closure) { + + if ($item instanceof Closure) { return $item(); } } return $item; } - if(is_object($object)) { - if(is_int($key)) { + + if (is_object($object)) { + if (is_int($key)) { $key = (string)$key; } - if(method_exists($object, $key) || method_exists($object, '__call')) { + + if ( + method_exists($object, $key) || + method_exists($object, '__call') + ) { return $object->$key(...$arguments); } + return $object->$key ?? null; } - throw new Exception("Cannot access \"$key\" on " . gettype($object)); + throw new Exception("Cannot access \"$key\" on " . gettype($object)); } } diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 993be68de0..26f76772e5 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -42,12 +42,14 @@ public function tokenize(): Generator $current = 0; while ($current < $this->length) { - $t = self::scanToken($this->source, $current); + $token = static::scanToken($this->source, $current); + // don't yield whitespace tokens (ignore them) - if($t->type !== TokenType::T_WHITESPACE) { - yield $t; + if ($token->type !== TokenType::T_WHITESPACE) { + yield $token; } - $current += mb_strlen($t->lexeme); + + $current += mb_strlen($token->lexeme); } yield new Token(TokenType::T_EOF, '', null); @@ -63,37 +65,50 @@ public function tokenize(): Generator */ protected static function scanToken(string $source, int $current): Token { - $l = ''; - $c = $source[$current]; + $lex = ''; + $char = $source[$current]; return match(true) { // single character tokens - $c === '.' => new Token(TokenType::T_DOT, '.'), - $c === '(' => new Token(TokenType::T_OPEN_PAREN, '('), - $c === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), - $c === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), - $c === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), - $c === ',' => new Token(TokenType::T_COMMA, ','), - $c === ':' => new Token(TokenType::T_COLON, ':'), + $char === '.' => new Token(TokenType::T_DOT, '.'), + $char === '(' => new Token(TokenType::T_OPEN_PAREN, '('), + $char === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), + $char === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), + $char === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), + $char === ',' => new Token(TokenType::T_COMMA, ','), + $char === ':' => new Token(TokenType::T_COLON, ':'), // two character tokens - self::match($source, $current, '\?\?', $l) => new Token(TokenType::T_COALESCE, $l), - self::match($source, $current, '\?\s*\.', $l) => new Token(TokenType::T_NULLSAFE, $l), - self::match($source, $current, '\?\s*:', $l) => new Token(TokenType::T_TERNARY_DEFAULT, $l), - self::match($source, $current, '=>', $l) => new Token(TokenType::T_ARROW, $l), - - // make sure this check comes after the two above that check for '?' in the beginning - $c === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), + static::match($source, $current, '\?\?', $lex) + => new Token(TokenType::T_COALESCE, $lex), + static::match($source, $current, '\?\s*\.', $lex) + => new Token(TokenType::T_NULLSAFE, $lex), + static::match($source, $current, '\?\s*:', $lex) + => new Token(TokenType::T_TERNARY_DEFAULT, $lex), + static::match($source, $current, '=>', $lex) + => new Token(TokenType::T_ARROW, $lex), + + // make sure this check comes after the two above + // that check for '?' in the beginning + $char === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), // multi character tokens - self::match($source, $current, '\s+', $l) => new Token(TokenType::T_WHITESPACE, $l), - self::match($source, $current, 'true', $l, true) => new Token(TokenType::T_TRUE, $l, true), - self::match($source, $current, 'false', $l, true) => new Token(TokenType::T_FALSE, $l, false), - self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), - self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, (int)$l), - self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), + static::match($source, $current, '\s+', $lex) + => new Token(TokenType::T_WHITESPACE, $lex), + static::match($source, $current, 'true', $lex, true) + => new Token(TokenType::T_TRUE, $lex, true), + static::match($source, $current, 'false', $lex, true) + => new Token(TokenType::T_FALSE, $lex, false), + static::match($source, $current, 'null', $lex, true) + => new Token(TokenType::T_NULL, $lex, null), + static::match($source, $current, static::DOUBLEQUOTE_STRING_REGEX, $lex) + => new Token(TokenType::T_STRING, $lex, stripcslashes(substr($lex, 1, -1))), + static::match($source, $current, static::SINGLEQUOTE_STRING_REGEX, $lex) + => new Token(TokenType::T_STRING, $lex, stripcslashes(substr($lex, 1, -1))), + static::match($source, $current, '\d+\b', $lex) + => new Token(TokenType::T_INTEGER, $lex, (int)$lex), + static::match($source, $current, static::IDENTIFIER_REGEX, $lex) + => new Token(TokenType::T_IDENTIFIER, $lex), // unknown token default => throw new Exception("Unexpected character: {$source[$current]}"), @@ -111,17 +126,26 @@ protected static function scanToken(string $source, int $current): Token * @param bool $caseIgnore Whether to ignore case while matching * @return bool Whether the regex pattern was matched */ - protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool - { + protected static function match( + string $source, + int $current, + string $regex, + string &$lexeme, + bool $caseIgnore = false + ): bool { $regex = '/\G' . $regex . '/u'; - if($caseIgnore) { + + if ($caseIgnore) { $regex .= 'i'; } + $matches = []; preg_match($regex, $source, $matches, 0, $current); + if (empty($matches[0])) { return false; } + $lexeme = $matches[0]; return true; } diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index 41db64ddc3..fc58257799 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -4,36 +4,46 @@ use Closure; use Exception; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\Node; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; use ReflectionClass; abstract class Visitor { protected Closure|null $interceptor = null; - public function visitNode(AST\Node $node): mixed + public function visitNode(Node $node): mixed { $shortName = (new ReflectionClass($node))->getShortName(); // remove the "Node" suffix $shortName = substr($shortName, 0, -4); + $method = 'visit' . $shortName; - $method = 'visit' . $shortName; - if(method_exists($this, $method)) { + if (method_exists($this, $method)) { return $this->$method($node); } throw new Exception('No visitor method for ' . $node::class); } - abstract public function visitArgumentList(AST\ArgumentListNode $node): mixed; - abstract public function visitArrayList(AST\ArrayListNode $node): mixed; - abstract public function visitCoalesce(AST\CoalesceNode $node): mixed; - abstract public function visitLiteral(AST\LiteralNode $node): mixed; - abstract public function visitMemberAccess(AST\MemberAccessNode $node): mixed; - abstract public function visitTernary(AST\TernaryNode $node): mixed; - abstract public function visitVariable(AST\VariableNode $node): mixed; - abstract public function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; - abstract public function visitClosure(AST\ClosureNode $node): mixed; + abstract public function visitArgumentList(ArgumentListNode $node): mixed; + abstract public function visitArrayList(ArrayListNode $node): mixed; + abstract public function visitCoalesce(CoalesceNode $node): mixed; + abstract public function visitLiteral(LiteralNode $node): mixed; + abstract public function visitMemberAccess(MemberAccessNode $node): mixed; + abstract public function visitTernary(TernaryNode $node): mixed; + abstract public function visitVariable(VariableNode $node): mixed; + abstract public function visitGlobalFunction(GlobalFunctionNode $node): mixed; + abstract public function visitClosure(ClosureNode $node): mixed; /** * Sets and activates an interceptor closure that is called for each resolved value. From d82b385f7fb7c1083f4b2091569bf3a265589252 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 18:23:06 +0100 Subject: [PATCH 14/31] set "interpreted" as default query runner --- src/Query/Query.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 043c0a9875..d271c40d08 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -107,7 +107,7 @@ public function resolve(array|object $data = []): mixed return $data; } - $mode = App::instance()->option('query.runner', 'transpiled'); + $mode = App::instance()->option('query.runner', 'interpreted'); if ($mode === 'legacy') { return $this->resolve_legacy($data); From a25e37d85c16e69385a8de67c0a064220476bfa2 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 12:36:19 +0100 Subject: [PATCH 15/31] Add new query parsers / compilers / runners --- src/Query/Query.php | 38 ++-- src/Toolkit/Query/AST/ArgumentList.php | 9 + src/Toolkit/Query/AST/ArrayList.php | 9 + src/Toolkit/Query/AST/Closure.php | 12 ++ src/Toolkit/Query/AST/Coalesce.php | 10 ++ src/Toolkit/Query/AST/GlobalFunction.php | 10 ++ src/Toolkit/Query/AST/Literal.php | 9 + src/Toolkit/Query/AST/MemberAccess.php | 12 ++ src/Toolkit/Query/AST/Node.php | 11 ++ src/Toolkit/Query/AST/Ternary.php | 13 ++ src/Toolkit/Query/AST/Variable.php | 9 + src/Toolkit/Query/BaseParser.php | 83 +++++++++ src/Toolkit/Query/Parser.php | 168 ++++++++++++++++++ src/Toolkit/Query/Runner.php | 27 +++ src/Toolkit/Query/Runners/Interpreted.php | 31 ++++ src/Toolkit/Query/Runners/Transpiled.php | 86 +++++++++ .../Query/Runners/Visitors/CodeGen.php | 151 ++++++++++++++++ .../Query/Runners/Visitors/Interpreter.php | 103 +++++++++++ src/Toolkit/Query/Runtime.php | 28 +++ src/Toolkit/Query/Token.php | 11 ++ src/Toolkit/Query/TokenType.php | 26 +++ src/Toolkit/Query/Tokenizer.php | 93 ++++++++++ src/Toolkit/Query/Visitor.php | 29 +++ tests/Query/QueryTest.php | 16 ++ 24 files changed, 983 insertions(+), 11 deletions(-) create mode 100644 src/Toolkit/Query/AST/ArgumentList.php create mode 100644 src/Toolkit/Query/AST/ArrayList.php create mode 100644 src/Toolkit/Query/AST/Closure.php create mode 100644 src/Toolkit/Query/AST/Coalesce.php create mode 100644 src/Toolkit/Query/AST/GlobalFunction.php create mode 100644 src/Toolkit/Query/AST/Literal.php create mode 100644 src/Toolkit/Query/AST/MemberAccess.php create mode 100644 src/Toolkit/Query/AST/Node.php create mode 100644 src/Toolkit/Query/AST/Ternary.php create mode 100644 src/Toolkit/Query/AST/Variable.php create mode 100644 src/Toolkit/Query/BaseParser.php create mode 100644 src/Toolkit/Query/Parser.php create mode 100644 src/Toolkit/Query/Runner.php create mode 100644 src/Toolkit/Query/Runners/Interpreted.php create mode 100644 src/Toolkit/Query/Runners/Transpiled.php create mode 100644 src/Toolkit/Query/Runners/Visitors/CodeGen.php create mode 100644 src/Toolkit/Query/Runners/Visitors/Interpreter.php create mode 100644 src/Toolkit/Query/Runtime.php create mode 100644 src/Toolkit/Query/Token.php create mode 100644 src/Toolkit/Query/TokenType.php create mode 100644 src/Toolkit/Query/Tokenizer.php create mode 100644 src/Toolkit/Query/Visitor.php diff --git a/src/Query/Query.php b/src/Query/Query.php index dc6fca8f4d..f799a70e78 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -11,6 +11,10 @@ use Kirby\Cms\User; use Kirby\Image\QrCode; use Kirby\Toolkit\I18n; +use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Interpreted; +use Kirby\Toolkit\Query\Runners\Transpiled; +use Kirby\Toolkit\Query\Visitor; /** * The Query class can be used to query arrays and objects, @@ -60,24 +64,16 @@ public static function factory(string|null $query): static /** * Method to help classes that extend Query * to intercept a segment's result. + * + * @deprecated 5.0.0 Will be removed in 6.0.0 */ public function intercept(mixed $result): mixed { return $result; } - /** - * Returns the query result if anything - * can be found, otherwise returns null - * - * @throws \Kirby\Exception\BadMethodCallException If an invalid method is accessed by the query - */ - public function resolve(array|object $data = []): mixed + private function resolve_legacy(array|object $data = []): mixed { - if (empty($this->query) === true) { - return $data; - } - // merge data with default entries if (is_array($data) === true) { $data = [...static::$entries, ...$data]; @@ -99,6 +95,26 @@ public function resolve(array|object $data = []): mixed // loop through all segments to resolve query return Expression::factory($this->query, $this)->resolve($data); + + } + + /** + * Returns the query result if anything + * can be found, otherwise returns null + * + * @throws \Kirby\Exception\BadMethodCallException If an invalid method is accessed by the query + */ + public function resolve(array|object $data = []): mixed + { + if (empty($this->query) === true) { + return $data; + } + + return match(option('query.runner', 'interpreted')) { + 'transpiled' => (new Transpiled(static::$entries))->run($this->query, $data), + 'interpreted' => (new Interpreted(static::$entries))->run($this->query, $data), + default => $this->resolve_legacy($data) + }; } } diff --git a/src/Toolkit/Query/AST/ArgumentList.php b/src/Toolkit/Query/AST/ArgumentList.php new file mode 100644 index 0000000000..4976433bd9 --- /dev/null +++ b/src/Toolkit/Query/AST/ArgumentList.php @@ -0,0 +1,9 @@ +visitNode($this); + } +} diff --git a/src/Toolkit/Query/AST/Ternary.php b/src/Toolkit/Query/AST/Ternary.php new file mode 100644 index 0000000000..df18106606 --- /dev/null +++ b/src/Toolkit/Query/AST/Ternary.php @@ -0,0 +1,13 @@ + + */ + protected Iterator $tokens; + + + public function __construct( + Tokenizer|Iterator $source, + ) { + if($source instanceof Tokenizer) { + $this->tokens = $source->tokenize(); + } else { + $this->tokens = $source; + } + + $first = $this->tokens->current(); + + if ($first === null) { + throw new \Exception('No tokens found.'); + } + + $this->current = $first; + } + + protected function consume(TokenType $type, string $message): Token { + if ($this->check($type)) { + return $this->advance(); + } + + throw new \Exception($message); + } + + protected function check(TokenType $type): bool { + if ($this->isAtEnd()) { + return false; + } + + return $this->current->type === $type; + } + + protected function advance(): ?Token { + if (!$this->isAtEnd()) { + $this->previous = $this->current; + $this->tokens->next(); + $this->current = $this->tokens->current(); + } + + return $this->previous; + } + + protected function isAtEnd(): bool { + return $this->current->type === TokenType::EOF; + } + + + protected function match(TokenType $type): Token|false { + if ($this->check($type)) { + return $this->advance(); + } + + return false; + } + + protected function matchAny(array $types): Token|false { + foreach ($types as $type) { + if ($this->check($type)) { + return $this->advance(); + } + } + + return false; + } +} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php new file mode 100644 index 0000000000..7f2e757b02 --- /dev/null +++ b/src/Toolkit/Query/Parser.php @@ -0,0 +1,168 @@ +expression(); + + // ensure that we consumed all tokens + if(!$this->isAtEnd()) + $this->consume(TokenType::EOF, 'Expect end of expression.'); + + return $expression; + } + + private function expression(): Node { + return $this->coalesce(); + } + + private function coalesce(): Node { + $left = $this->ternary(); + + while ($this->match(TokenType::COALESCE)) { + $operator = $this->previous; + $right = $this->ternary(); + $left = new Coalesce($left, $right); + } + + return $left; + } + + private function ternary(): Node { + $left = $this->memberAccess(); + + if ($tok = $this->matchAny([TokenType::QUESTION_MARK, TokenType::TERNARY_DEFAULT])) { + if($tok->type === TokenType::TERNARY_DEFAULT) { + $trueIsDefault = true; + $trueBranch = null; + $falseBranch = $this->expression(); + } else { + $trueIsDefault = false; + $trueBranch = $this->expression(); + $this->consume(TokenType::COLON, 'Expect ":" after true branch.'); + $falseBranch = $this->expression(); + } + + return new Ternary($left, $trueBranch, $falseBranch, $trueIsDefault); + } + + return $left; + } + + private function memberAccess(): Node { + $left = $this->atomic(); + + while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { + $nullSafe = $tok->type === TokenType::NULLSAFE; + + $right = $this->consume(TokenType::IDENTIFIER, 'Expect property name after ".".'); + + if($this->match(TokenType::OPEN_PAREN)) { + $arguments = $this->argumentList(); + $left = new MemberAccess($left, $right->lexeme, $arguments, $nullSafe); + } else { + $left = new MemberAccess($left, $right->lexeme, null, $nullSafe); + } + } + + return $left; + } + + private function listUntil(TokenType $until): array { + $elements = []; + + while (!$this->isAtEnd() && !$this->check($until)) { + $elements[] = $this->expression(); + + if (!$this->match(TokenType::COMMA)) { + break; + } + } + + // consume the closing token + $this->consume($until, 'Expect closing bracket after list.'); + + return $elements; + } + + private function argumentList(): Node { + $list = $this->listUntil(TokenType::CLOSE_PAREN); + + return new ArgumentList($list); + } + + private function atomic(): Node { + // primitives + if ($token = $this->matchAny([ + TokenType::TRUE, + TokenType::FALSE, + TokenType::NULL, + TokenType::STRING, + TokenType::NUMBER, + ])) { + return new Literal($token->literal); + } + + // array literals + if ($token = $this->match(TokenType::OPEN_BRACKET)) { + $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); + + return new ArrayList($arrayItems); + } + + // global functions and variables + if ($token = $this->match(TokenType::IDENTIFIER)) { + if($this->match(TokenType::OPEN_PAREN)) { + $arguments = $this->argumentList(); + return new GlobalFunction($token->lexeme, $arguments); + } + + return new Variable($token->lexeme); + } + + // grouping and closure argument lists + if ($token = $this->match(TokenType::OPEN_PAREN)) { + $list = $this->listUntil(TokenType::CLOSE_PAREN); + + if($this->match(TokenType::ARROW)) { + $expression = $this->expression(); + // check if all elements are variables + foreach($list as $element) { + if(!$element instanceof Variable) { + throw new \Exception('Expecting only variables in closure argument list.'); + } + } + $arguments = new ArgumentList($list); + return new Closure($arguments, $expression); + } else { + if(count($list) > 1) { + throw new \Exception('Expecting \"=>\" after closure argument list.'); + } else { + // this is just a grouping + return $list[0]; + } + } + } + + throw new \Exception('Expect expression.'); + } +} diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php new file mode 100644 index 0000000000..b5294a0840 --- /dev/null +++ b/src/Toolkit/Query/Runner.php @@ -0,0 +1,27 @@ +parse(); + + return self::$cache[$query] = fn(array $binding) => $node->accept(new Visitors\Interpreter($this->allowedFunctions, $binding)); + } + + public function run(string $query, array $bindings = []): mixed { + $resolver = $this->getResolver($query); + return $resolver($bindings); + } +} diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php new file mode 100644 index 0000000000..628343d903 --- /dev/null +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -0,0 +1,86 @@ +parse(); + $codeGen = new Visitors\CodeGen($this->allowedFunctions); + + $functionBody = $node->accept($codeGen); + + $mappings = join("\n", array_map(fn($k, $v) => "$$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); + + $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; + $function = "getResolver($query); + if(!is_callable($function)) { + throw new \Exception("Query is not valid"); + } + return $function($context, $this->allowedFunctions); + } +} diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php new file mode 100644 index 0000000000..a6b42ed8a0 --- /dev/null +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -0,0 +1,151 @@ + $argument->accept($this), $node->arguments); + return join(', ', $arguments); + } + + /** + * Generates code like `[element1, element2, element3]` from an array list node. + */ + public function visitArrayList(ArrayList $node): string { + $elements = array_map(fn($element) => $element->accept($this), $node->elements); + return '[' . join(', ', $elements) . ']'; + } + + /** + * Generates code like `$left ?? $right` from a coalesce node. + */ + public function visitCoalesce(Coalesce $node): string { + $left = $node->left->accept($this); + $right = $node->right->accept($this); + return "($left ?? $right)"; + } + + /** + * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. + */ + public function visitLiteral(Literal $node): string { + return var_export($node->value, true); + } + + /** + * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. + */ + public function visitMemberAccess(MemberAccess $node): string { + $object = $node->object->accept($this); + $member = $node->member; + + $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $memberStr = var_export($member, true); + $nullSafe = $node->nullSafe ? 'true' : 'false'; + + if($node->arguments) { + $arguments = $node->arguments->accept($this); + $member = var_export($member, true); + + return "Runtime::access($object, $memberStr, $nullSafe, $arguments)"; + } + + return "Runtime::access($object, $memberStr, $nullSafe)"; + } + + /** + * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. + */ + public function visitTernary(Ternary $node): string { + $left = $node->condition->accept($this); + $falseBranch = $node->falseBranch->accept($this); + + if($node->trueBranchIsDefault) { + return "($left ?: $falseBranch)"; + } else { + $trueBranch = $node->trueBranch->accept($this); + return "($left ? $trueBranch : $falseBranch)"; + } + } + + public function visitVariable(Variable $node): string { + $name = $node->name; + $namestr = var_export($name, true); + + $key = "_" . crc32($name); + if(isset($this->directAccessFor[$name])) { + return "$$key"; + } + + if(!isset($this->mappings[$key])) { + $this->mappings[$key] = "(match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null })"; + } + + return "\$$key"; + } + + /** + * Generates code like `$functions['function']($arguments)` from a global function node. + */ + public function visitGlobalFunction(GlobalFunction $node): string { + $name = $node->name; + if(!isset($this->validGlobalFunctions[$name])) { + throw new \Exception("Invalid global function $name"); + } + + $arguments = $node->arguments->accept($this); + $name = var_export($name, true); + + return "\$functions[$name]($arguments)"; + } + + public function visitClosure(Closure $node): mixed { + $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + + $names = array_map(fn($n) => $n->name, $node->arguments->arguments); + $args = array_map(fn(string $n) => '$_' . crc32($n), $names); + $args = join(', ', $args); + + $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); + + return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); + } +} diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php new file mode 100644 index 0000000000..034ba33da0 --- /dev/null +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -0,0 +1,103 @@ + $argument->accept($this), $node->arguments); + } + + public function visitArrayList(ArrayList $node): mixed { + return array_map(fn($element) => $element->accept($this), $node->elements); + } + + public function visitCoalesce(Coalesce $node): mixed { + return $node->left->accept($this) ?? $node->right->accept($this); + } + + public function visitLiteral(Literal $node): mixed { + return $node->value; + } + + public function visitMemberAccess(MemberAccess $node): mixed { + $left = $node->object->accept($this); + if($node->arguments !== null) { + return Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + } + return Runtime::access($left, $node->member, $node->nullSafe); + } + + public function visitTernary(Ternary $node): mixed { + if($node->trueBranchIsDefault) { + return $node->condition->accept($this) ?: $node->trueBranch->accept($this); + } else { + return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + } + } + + public function visitVariable(Variable $node): mixed { + // what looks like a variable might actually be a global function + // but if there is a variable with the same name, the variable takes precedence + + if(isset($this->context[$node->name])) { + return $this->context[$node->name]; + } + + if(isset($this->validGlobalFunctions[$node->name])) { + return $this->validGlobalFunctions[$node->name](); + } + + return null; + } + + public function visitGlobalFunction(GlobalFunction $node): mixed { + if(!isset($this->validGlobalFunctions[$node->name])) { + throw new Exception("Invalid global function $node->name"); + } + return $this->validGlobalFunctions[$node->name](...$node->arguments->accept($this)); + } + + public function visitClosure(Closure $node): mixed { + $self = $this; + + return function(...$params) use ($self, $node) { + $context = $self->context; + $functions = $self->validGlobalFunctions; + + $arguments = array_combine( + array_map(fn($param) => $param->name, $node->arguments->arguments), + $params + ); + + $visitor = new self($functions, [...$context, ...$arguments]); + + return $node->body->accept($visitor); + }; + } +} diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php new file mode 100644 index 0000000000..f1bf41e691 --- /dev/null +++ b/src/Toolkit/Query/Runtime.php @@ -0,0 +1,28 @@ +$key(...$arguments); + } + return $object->$key ?? null; + } else { + throw new \Exception("Cannot access \"$key\" on " . gettype($object)); + } + } +} diff --git a/src/Toolkit/Query/Token.php b/src/Toolkit/Query/Token.php new file mode 100644 index 0000000000..0a03c914ad --- /dev/null +++ b/src/Toolkit/Query/Token.php @@ -0,0 +1,11 @@ +length = mb_strlen($source); + } + + /** + * Tokenizes the source string and returns a generator of tokens. + * @return Generator + */ + public function tokenize(): Generator { + $current = 0; + + while ($current < $this->length) { + $t = self::scanToken($this->source, $current); + // don't yield whitespace tokens (ignore them) + if($t->type !== TokenType::WHITESPACE) { + yield $t; + } + $current += mb_strlen($t->lexeme); + } + + yield new Token(TokenType::EOF, '', null); + } + + protected static function scanToken(string $source, int $current): Token { + $l = ''; + $c = $source[$current]; + + return match(true) { + // single character tokens + $c === '.' => new Token(TokenType::DOT, '.'), + $c === '(' => new Token(TokenType::OPEN_PAREN, '('), + $c === ')' => new Token(TokenType::CLOSE_PAREN, ')'), + $c === '[' => new Token(TokenType::OPEN_BRACKET, '['), + $c === ']' => new Token(TokenType::CLOSE_BRACKET, ']'), + $c === ',' => new Token(TokenType::COMMA, ','), + $c === ':' => new Token(TokenType::COLON, ':'), + + // two character tokens + self::match($source, $current, '\\?\\?', $l) => new Token(TokenType::COALESCE, $l), + self::match($source, $current, '\\?\\s*\\.', $l) => new Token(TokenType::NULLSAFE, $l), + self::match($source, $current, '\\?\\s*:', $l) => new Token(TokenType::TERNARY_DEFAULT, $l), + self::match($source, $current, '=>', $l) => new Token(TokenType::ARROW, $l), + + // make sure this check comes after the two above that check for '?' in the beginning + $c === '?' => new Token(TokenType::QUESTION_MARK, '?'), + + // multi character tokens + self::match($source, $current, '\\s+', $l) => new Token(TokenType::WHITESPACE, $l), + self::match($source, $current, 'true', $l, true) => new Token(TokenType::TRUE, $l, true), + self::match($source, $current, 'false', $l, true) => new Token(TokenType::FALSE, $l, false), + self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), + self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '[0-9]+\\.[0-9]+', $l) => new Token(TokenType::NUMBER, $l, floatval($l)), + self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::NUMBER, $l, intval($l)), + self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), + + + // unknown token + default => throw new \Exception("Unexpected character: {$source[$current]}"), + }; + } + + /** + * Checks if a given regex matches the current position in the source. Returns the matched string or false. Advances the current position when a match is found. + * @param string $regex + * @return string|false + */ + protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { + $regex = '/\G' . $regex . '/u'; + if($caseIgnore) { + $regex .= 'i'; + } + $matches = []; + preg_match($regex, $source, $matches, 0, $current); + if (empty($matches[0])) { + return false; + } + $lexeme = $matches[0]; + return true; + } +} diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php new file mode 100644 index 0000000000..f1cedbcc45 --- /dev/null +++ b/src/Toolkit/Query/Visitor.php @@ -0,0 +1,29 @@ +getShortName(); + + $method = 'visit' . $shortName; + if(method_exists($this, $method)) { + return $this->$method($node); + } + + throw new Exception("No visitor method for " . $node::class); + } + + abstract function visitArgumentList(AST\ArgumentList $node): mixed; + abstract function visitArrayList(AST\ArrayList $node): mixed; + abstract function visitCoalesce(AST\Coalesce $node): mixed; + abstract function visitLiteral(AST\Literal $node): mixed; + abstract function visitMemberAccess(AST\MemberAccess $node): mixed; + abstract function visitTernary(AST\Ternary $node): mixed; + abstract function visitVariable(AST\Variable $node): mixed; + abstract function visitGlobalFunction(AST\GlobalFunction $node): mixed; + abstract function visitClosure(AST\Closure $node): mixed; +} diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index c51d961f48..b23f1025bf 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -89,7 +89,23 @@ public function testResolveWithClosureArgument() $bar = $query->resolve($data); $this->assertInstanceOf(Closure::class, $bar); + $bar = $bar(); $this->assertSame('simpson', $bar); } + + /** + * @covers ::resolve + */ + public function testResolveWithClosureWithArgument() + { + $query = new Query('(foo) => foo.homer'); + $data = []; + + $bar = $query->resolve($data); + $this->assertInstanceOf(Closure::class, $bar); + + $bar = $bar(['homer' => 'simpson']); + $this->assertSame('simpson', $bar); + } } From 85dd0e0cd30f8d9550e6cbbe5d95e3d891fed62f Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 13:37:55 +0100 Subject: [PATCH 16/31] Fix missing imports of Exception --- src/Toolkit/Query/BaseParser.php | 5 +++-- src/Toolkit/Query/Parser.php | 7 ++++--- src/Toolkit/Query/Runner.php | 3 +-- src/Toolkit/Query/Runners/Transpiled.php | 4 ++-- src/Toolkit/Query/Runners/Visitors/CodeGen.php | 3 ++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index a2baca5baa..5c4f1a96d8 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; abstract class BaseParser { @@ -26,7 +27,7 @@ public function __construct( $first = $this->tokens->current(); if ($first === null) { - throw new \Exception('No tokens found.'); + throw new Exception('No tokens found.'); } $this->current = $first; @@ -37,7 +38,7 @@ protected function consume(TokenType $type, string $message): Token { return $this->advance(); } - throw new \Exception($message); + throw new Exception($message); } protected function check(TokenType $type): bool { diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 7f2e757b02..8f35b2ecda 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Iterator; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; @@ -148,14 +149,14 @@ private function atomic(): Node { // check if all elements are variables foreach($list as $element) { if(!$element instanceof Variable) { - throw new \Exception('Expecting only variables in closure argument list.'); + throw new Exception('Expecting only variables in closure argument list.'); } } $arguments = new ArgumentList($list); return new Closure($arguments, $expression); } else { if(count($list) > 1) { - throw new \Exception('Expecting \"=>\" after closure argument list.'); + throw new Exception('Expecting \"=>\" after closure argument list.'); } else { // this is just a grouping return $list[0]; @@ -163,6 +164,6 @@ private function atomic(): Node { } } - throw new \Exception('Expect expression.'); + throw new Exception('Expect expression.'); } } diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index b5294a0840..00c9bacf46 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -2,7 +2,6 @@ namespace Kirby\Toolkit\Query; -use Closure; use Exception; abstract class Runner { @@ -21,7 +20,7 @@ public function __construct( * @param string $query The query string to be executed. * @param array $context An optional array of context variables to be passed to the query executor. * @return mixed The result of the executed query. - * @throws \Exception If the query is not valid or the executor is not callable. + * @throws Exception If the query is not valid or the executor is not callable. */ abstract public function run(string $query, array $context = []): mixed; } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 628343d903..1800fac7ee 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -74,12 +74,12 @@ protected function getResolver(string $query): Closure { * @param string $query The query string to be executed. * @param array $context An optional array of context variables to be passed to the query executor. * @return mixed The result of the executed query. - * @throws \Exception If the query is not valid or the executor is not callable. + * @throws Exception If the query is not valid or the executor is not callable. */ public function run(string $query, array $context = []): mixed { $function = $this->getResolver($query); if(!is_callable($function)) { - throw new \Exception("Query is not valid"); + throw new Exception("Query is not valid"); } return $function($context, $this->allowedFunctions); } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index a6b42ed8a0..cebf569498 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Exception; use Kirby\Toolkit\Query\AST\ArgumentList; use Kirby\Toolkit\Query\AST\ArrayList; use Kirby\Toolkit\Query\AST\Closure; @@ -128,7 +129,7 @@ public function visitVariable(Variable $node): string { public function visitGlobalFunction(GlobalFunction $node): string { $name = $node->name; if(!isset($this->validGlobalFunctions[$name])) { - throw new \Exception("Invalid global function $name"); + throw new Exception("Invalid global function $name"); } $arguments = $node->arguments->accept($this); From 46b67559b01747bbd37c2c1a54495422609f32d1 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 14:30:37 +0100 Subject: [PATCH 17/31] query parser fixes - cast data objects to arrays before passing to query runner - allow integers as identifiers of array keys when used after a dot - don't emit warnings when an array key is missing --- src/Query/Query.php | 4 ++-- src/Toolkit/Query/AST/MemberAccess.php | 2 +- src/Toolkit/Query/Parser.php | 26 +++++++++++++++++++++----- src/Toolkit/Query/Runtime.php | 26 ++++++++++++++++++-------- src/Toolkit/Query/TokenType.php | 2 +- src/Toolkit/Query/Tokenizer.php | 7 +++---- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index f799a70e78..e23d369835 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -111,8 +111,8 @@ public function resolve(array|object $data = []): mixed } return match(option('query.runner', 'interpreted')) { - 'transpiled' => (new Transpiled(static::$entries))->run($this->query, $data), - 'interpreted' => (new Interpreted(static::$entries))->run($this->query, $data), + 'transpiled' => (new Transpiled(static::$entries))->run($this->query, (array)$data), + 'interpreted' => (new Interpreted(static::$entries))->run($this->query, (array)$data), default => $this->resolve_legacy($data) }; } diff --git a/src/Toolkit/Query/AST/MemberAccess.php b/src/Toolkit/Query/AST/MemberAccess.php index c1f21476f4..dce29e4666 100644 --- a/src/Toolkit/Query/AST/MemberAccess.php +++ b/src/Toolkit/Query/AST/MemberAccess.php @@ -5,7 +5,7 @@ class MemberAccess extends Node { public function __construct( public Node $object, - public string $member, + public string|int $member, public ?ArgumentList $arguments = null, public bool $nullSafe = false, ) {} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 8f35b2ecda..73beb7a3b8 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -40,7 +40,6 @@ private function coalesce(): Node { $left = $this->ternary(); while ($this->match(TokenType::COALESCE)) { - $operator = $this->previous; $right = $this->ternary(); $left = new Coalesce($left, $right); } @@ -75,13 +74,19 @@ private function memberAccess(): Node { while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { $nullSafe = $tok->type === TokenType::NULLSAFE; - $right = $this->consume(TokenType::IDENTIFIER, 'Expect property name after ".".'); + if($right = $this->match(TokenType::IDENTIFIER)) { + $right = $right->lexeme; + } else if($right = $this->match(TokenType::INTEGER)) { + $right = $right->literal; + } else { + throw new Exception('Expect property name after ".".'); + } if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccess($left, $right->lexeme, $arguments, $nullSafe); + $left = new MemberAccess($left, $right, $arguments, $nullSafe); } else { - $left = new MemberAccess($left, $right->lexeme, null, $nullSafe); + $left = new MemberAccess($left, $right, null, $nullSafe); } } @@ -111,14 +116,25 @@ private function argumentList(): Node { return new ArgumentList($list); } + + private function atomic(): Node { + + // float numbers + if ($integer = $this->match(TokenType::INTEGER)) { + if($this->match(TokenType::DOT)) { + $fractional = $this->match(TokenType::INTEGER); + return new Literal(floatval($integer->literal . '.' . $fractional->literal)); + } + return new Literal($integer->literal); + } + // primitives if ($token = $this->matchAny([ TokenType::TRUE, TokenType::FALSE, TokenType::NULL, TokenType::STRING, - TokenType::NUMBER, ])) { return new Literal($token->literal); } diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index f1bf41e691..07eadc61d5 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -2,27 +2,37 @@ namespace Kirby\Toolkit\Query; +use Exception; + class Runtime { - static function access($object, $key, bool $nullSafe = false, ...$arguments): mixed { + static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed { if($nullSafe && $object === null) { return null; } if(is_array($object)) { - if($arguments) { - return $object[$key](...$arguments); - } - if($object[$key] instanceof \Closure) { - return $object[$key](); + $item = ($object[$key] ?? $object[(string)$key] ?? null); + + if($item) { + if($arguments) { + return $item(...$arguments); + } + if($item instanceof \Closure) { + return $item(); + } } - return $object[$key] ?? null; + + return $item; } else if(is_object($object)) { + if(is_int($key)) { + $key = (string)$key; + } if(method_exists($object, $key) || method_exists($object, '__call')) { return $object->$key(...$arguments); } return $object->$key ?? null; } else { - throw new \Exception("Cannot access \"$key\" on " . gettype($object)); + throw new Exception("Cannot access \"$key\" on " . gettype($object)); } } } diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index 519b952a74..dcfe904596 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -4,7 +4,7 @@ enum TokenType { case STRING; - case NUMBER; + case INTEGER; case WHITESPACE; case IDENTIFIER; case DOT; diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 48b38d7192..430499931d 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Exception; use Generator; class Tokenizer { @@ -62,13 +63,11 @@ protected static function scanToken(string $source, int $current): Token { self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '[0-9]+\\.[0-9]+', $l) => new Token(TokenType::NUMBER, $l, floatval($l)), - self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::NUMBER, $l, intval($l)), + self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::INTEGER, $l, intval($l)), self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), - // unknown token - default => throw new \Exception("Unexpected character: {$source[$current]}"), + default => throw new Exception("Unexpected character: {$source[$current]}"), }; } From 9392a434558a1eaa502bc29128d7860a12bb4a5c Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 16:58:04 +0100 Subject: [PATCH 18/31] Implement `intercept`, rename AST classes with a "Node" suffix - Implements the intercept mechanism for both runners - Renames the AST Node classes with a "Node" suffix to avoid confusion with some PHP internal classes (like `Closure` -> `ClosureNode`) --- src/Query/Query.php | 18 ++-- ...{ArgumentList.php => ArgumentListNode.php} | 2 +- .../AST/{ArrayList.php => ArrayListNode.php} | 2 +- .../AST/{Closure.php => ClosureNode.php} | 4 +- .../AST/{Coalesce.php => CoalesceNode.php} | 2 +- ...balFunction.php => GlobalFunctionNode.php} | 4 +- .../AST/{Literal.php => LiteralNode.php} | 2 +- ...{MemberAccess.php => MemberAccessNode.php} | 4 +- .../AST/{Ternary.php => TernaryNode.php} | 2 +- .../AST/{Variable.php => VariableNode.php} | 2 +- src/Toolkit/Query/Parser.php | 49 +++++----- src/Toolkit/Query/Runner.php | 3 + src/Toolkit/Query/Runners/Interpreted.php | 15 +++- src/Toolkit/Query/Runners/Transpiled.php | 5 +- .../Query/Runners/Visitors/CodeGen.php | 61 +++++++------ .../Query/Runners/Visitors/Interpreter.php | 89 +++++++++++++------ src/Toolkit/Query/Visitor.php | 31 +++++-- tests/Query/QueryTest.php | 32 +++++++ 18 files changed, 216 insertions(+), 111 deletions(-) rename src/Toolkit/Query/AST/{ArgumentList.php => ArgumentListNode.php} (73%) rename src/Toolkit/Query/AST/{ArrayList.php => ArrayListNode.php} (75%) rename src/Toolkit/Query/AST/{Closure.php => ClosureNode.php} (60%) rename src/Toolkit/Query/AST/{Coalesce.php => CoalesceNode.php} (78%) rename src/Toolkit/Query/AST/{GlobalFunction.php => GlobalFunctionNode.php} (57%) rename src/Toolkit/Query/AST/{Literal.php => LiteralNode.php} (75%) rename src/Toolkit/Query/AST/{MemberAccess.php => MemberAccessNode.php} (66%) rename src/Toolkit/Query/AST/{Ternary.php => TernaryNode.php} (86%) rename src/Toolkit/Query/AST/{Variable.php => VariableNode.php} (75%) diff --git a/src/Query/Query.php b/src/Query/Query.php index e23d369835..9f1455c8c7 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -3,6 +3,7 @@ namespace Kirby\Query; use Closure; +use Exception; use Kirby\Cms\App; use Kirby\Cms\Collection; use Kirby\Cms\File; @@ -64,8 +65,6 @@ public static function factory(string|null $query): static /** * Method to help classes that extend Query * to intercept a segment's result. - * - * @deprecated 5.0.0 Will be removed in 6.0.0 */ public function intercept(mixed $result): mixed { @@ -110,11 +109,18 @@ public function resolve(array|object $data = []): mixed return $data; } - return match(option('query.runner', 'interpreted')) { - 'transpiled' => (new Transpiled(static::$entries))->run($this->query, (array)$data), - 'interpreted' => (new Interpreted(static::$entries))->run($this->query, (array)$data), - default => $this->resolve_legacy($data) + if(option('query.runner', 'interpreted') == 'legacy') { + return $this->resolve_legacy($data); + } + + $runnerClass = match(option('query.runner', 'interpreted')) { + 'transpiled' => Transpiled::class, + 'interpreted' => Interpreted::class, + default => throw new Exception('Invalid query runner') }; + + $runner = new $runnerClass(static::$entries, $this->intercept(...)); + return $runner->run($this->query, (array)$data); } } diff --git a/src/Toolkit/Query/AST/ArgumentList.php b/src/Toolkit/Query/AST/ArgumentListNode.php similarity index 73% rename from src/Toolkit/Query/AST/ArgumentList.php rename to src/Toolkit/Query/AST/ArgumentListNode.php index 4976433bd9..1980223203 100644 --- a/src/Toolkit/Query/AST/ArgumentList.php +++ b/src/Toolkit/Query/AST/ArgumentListNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class ArgumentList extends Node { +class ArgumentListNode extends Node { public function __construct( public array $arguments, ) {} diff --git a/src/Toolkit/Query/AST/ArrayList.php b/src/Toolkit/Query/AST/ArrayListNode.php similarity index 75% rename from src/Toolkit/Query/AST/ArrayList.php rename to src/Toolkit/Query/AST/ArrayListNode.php index b5ae063d8a..0b092d7cb2 100644 --- a/src/Toolkit/Query/AST/ArrayList.php +++ b/src/Toolkit/Query/AST/ArrayListNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class ArrayList extends Node { +class ArrayListNode extends Node { public function __construct( public array $elements, ) {} diff --git a/src/Toolkit/Query/AST/Closure.php b/src/Toolkit/Query/AST/ClosureNode.php similarity index 60% rename from src/Toolkit/Query/AST/Closure.php rename to src/Toolkit/Query/AST/ClosureNode.php index edda31b30f..83a26c68de 100644 --- a/src/Toolkit/Query/AST/Closure.php +++ b/src/Toolkit/Query/AST/ClosureNode.php @@ -2,10 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class Closure extends Node +class ClosureNode extends Node { public function __construct( - public ArgumentList $arguments, + public ArgumentListNode $arguments, public Node $body, ) { } diff --git a/src/Toolkit/Query/AST/Coalesce.php b/src/Toolkit/Query/AST/CoalesceNode.php similarity index 78% rename from src/Toolkit/Query/AST/Coalesce.php rename to src/Toolkit/Query/AST/CoalesceNode.php index 63b7d9d445..1873347123 100644 --- a/src/Toolkit/Query/AST/Coalesce.php +++ b/src/Toolkit/Query/AST/CoalesceNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Coalesce extends Node { +class CoalesceNode extends Node { public function __construct( public Node $left, public Node $right, diff --git a/src/Toolkit/Query/AST/GlobalFunction.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php similarity index 57% rename from src/Toolkit/Query/AST/GlobalFunction.php rename to src/Toolkit/Query/AST/GlobalFunctionNode.php index 57a8e037be..ab59bee65e 100644 --- a/src/Toolkit/Query/AST/GlobalFunction.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,9 +2,9 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunction extends Node { +class GlobalFunctionNode extends Node { public function __construct( public string $name, - public ArgumentList $arguments, + public ArgumentListNode $arguments, ) {} } diff --git a/src/Toolkit/Query/AST/Literal.php b/src/Toolkit/Query/AST/LiteralNode.php similarity index 75% rename from src/Toolkit/Query/AST/Literal.php rename to src/Toolkit/Query/AST/LiteralNode.php index bdfe68602d..fc52cfc9eb 100644 --- a/src/Toolkit/Query/AST/Literal.php +++ b/src/Toolkit/Query/AST/LiteralNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Literal extends Node { +class LiteralNode extends Node { public function __construct( public mixed $value, ) {} diff --git a/src/Toolkit/Query/AST/MemberAccess.php b/src/Toolkit/Query/AST/MemberAccessNode.php similarity index 66% rename from src/Toolkit/Query/AST/MemberAccess.php rename to src/Toolkit/Query/AST/MemberAccessNode.php index dce29e4666..7280940abb 100644 --- a/src/Toolkit/Query/AST/MemberAccess.php +++ b/src/Toolkit/Query/AST/MemberAccessNode.php @@ -2,11 +2,11 @@ namespace Kirby\Toolkit\Query\AST; -class MemberAccess extends Node { +class MemberAccessNode extends Node { public function __construct( public Node $object, public string|int $member, - public ?ArgumentList $arguments = null, + public ?ArgumentListNode $arguments = null, public bool $nullSafe = false, ) {} } diff --git a/src/Toolkit/Query/AST/Ternary.php b/src/Toolkit/Query/AST/TernaryNode.php similarity index 86% rename from src/Toolkit/Query/AST/Ternary.php rename to src/Toolkit/Query/AST/TernaryNode.php index df18106606..38e662ba03 100644 --- a/src/Toolkit/Query/AST/Ternary.php +++ b/src/Toolkit/Query/AST/TernaryNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Ternary extends Node { +class TernaryNode extends Node { public function __construct( public Node $condition, public ?Node $trueBranch, diff --git a/src/Toolkit/Query/AST/Variable.php b/src/Toolkit/Query/AST/VariableNode.php similarity index 75% rename from src/Toolkit/Query/AST/Variable.php rename to src/Toolkit/Query/AST/VariableNode.php index 567f15d913..2088743eec 100644 --- a/src/Toolkit/Query/AST/Variable.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,7 +2,7 @@ namespace Kirby\Toolkit\Query\AST; -class Variable extends Node { +class VariableNode extends Node { public function __construct( public string $name, ) {} diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 73beb7a3b8..bde8c4d078 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -4,16 +4,16 @@ use Exception; use Iterator; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\GlobalFunction; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\Node; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; class Parser extends BaseParser { public function __construct( @@ -41,7 +41,7 @@ private function coalesce(): Node { while ($this->match(TokenType::COALESCE)) { $right = $this->ternary(); - $left = new Coalesce($left, $right); + $left = new CoalesceNode($left, $right); } return $left; @@ -62,7 +62,7 @@ private function ternary(): Node { $falseBranch = $this->expression(); } - return new Ternary($left, $trueBranch, $falseBranch, $trueIsDefault); + return new TernaryNode($left, $trueBranch, $falseBranch, $trueIsDefault); } return $left; @@ -84,9 +84,9 @@ private function memberAccess(): Node { if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccess($left, $right, $arguments, $nullSafe); + $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); } else { - $left = new MemberAccess($left, $right, null, $nullSafe); + $left = new MemberAccessNode($left, $right, null, $nullSafe); } } @@ -113,20 +113,17 @@ private function listUntil(TokenType $until): array { private function argumentList(): Node { $list = $this->listUntil(TokenType::CLOSE_PAREN); - return new ArgumentList($list); + return new ArgumentListNode($list); } - - private function atomic(): Node { - // float numbers if ($integer = $this->match(TokenType::INTEGER)) { if($this->match(TokenType::DOT)) { $fractional = $this->match(TokenType::INTEGER); - return new Literal(floatval($integer->literal . '.' . $fractional->literal)); + return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); } - return new Literal($integer->literal); + return new LiteralNode($integer->literal); } // primitives @@ -136,24 +133,24 @@ private function atomic(): Node { TokenType::NULL, TokenType::STRING, ])) { - return new Literal($token->literal); + return new LiteralNode($token->literal); } // array literals if ($token = $this->match(TokenType::OPEN_BRACKET)) { $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); - return new ArrayList($arrayItems); + return new ArrayListNode($arrayItems); } // global functions and variables if ($token = $this->match(TokenType::IDENTIFIER)) { if($this->match(TokenType::OPEN_PAREN)) { $arguments = $this->argumentList(); - return new GlobalFunction($token->lexeme, $arguments); + return new GlobalFunctionNode($token->lexeme, $arguments); } - return new Variable($token->lexeme); + return new VariableNode($token->lexeme); } // grouping and closure argument lists @@ -164,12 +161,12 @@ private function atomic(): Node { $expression = $this->expression(); // check if all elements are variables foreach($list as $element) { - if(!$element instanceof Variable) { + if(!$element instanceof VariableNode) { throw new Exception('Expecting only variables in closure argument list.'); } } - $arguments = new ArgumentList($list); - return new Closure($arguments, $expression); + $arguments = new ArgumentListNode($list); + return new ClosureNode($arguments, $expression); } else { if(count($list) > 1) { throw new Exception('Expecting \"=>\" after closure argument list.'); diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index 00c9bacf46..b327194d44 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; abstract class Runner { @@ -12,6 +13,7 @@ abstract class Runner { */ public function __construct( public array $allowedFunctions = [], + protected Closure|null $interceptor = null, ) {} /** @@ -23,4 +25,5 @@ public function __construct( * @throws Exception If the query is not valid or the executor is not callable. */ abstract public function run(string $query, array $context = []): mixed; + } diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 768a9caeba..055b74b10d 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -10,6 +10,11 @@ class Interpreted extends Runner { private static array $cache = []; + public function __construct( + public array $allowedFunctions = [], + protected Closure|null $interceptor = null, + ) {} + protected function getResolver(string $query): Closure { // load closure from process cache if(isset(self::$cache[$query])) { @@ -21,7 +26,15 @@ protected function getResolver(string $query): Closure { $parser = new Parser($t); $node = $parser->parse(); - return self::$cache[$query] = fn(array $binding) => $node->accept(new Visitors\Interpreter($this->allowedFunctions, $binding)); + $self = $this; + + return self::$cache[$query] = function(array $binding) use ($node, $self) { + $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); + if($self->interceptor !== null) { + $interpreter->setInterceptor($self->interceptor); + } + return $node->accept($interpreter); + }; } public function run(string $query, array $bindings = []): mixed { diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 1800fac7ee..9ca16ffcae 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -19,6 +19,7 @@ class Transpiled extends Runner { */ public function __construct( public array $allowedFunctions = [], + public Closure|null $interceptor = null, ) {} @@ -54,7 +55,7 @@ protected function getResolver(string $query): Closure { $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; - $function = "allowedFunctions); + return $function($context, $this->allowedFunctions, $this->interceptor ?? fn($v) => $v); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index cebf569498..77972c553c 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -3,15 +3,15 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; use Exception; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\GlobalFunction; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; use Kirby\Toolkit\Query\Visitor; @@ -29,20 +29,27 @@ class CodeGen extends Visitor { */ public array $uses = []; + /** + * @var array{string:string} + */ public array $mappings = []; /** * CodeGen constructor. * - * @param array{string:Closure} $validGlobalFunctions An array of valid global function closures. + * @param array{string:PHPClosure} $validGlobalFunctions An array of valid global function closures. */ public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} + private function intercept(string $value): string { + return "(\$intercept($value))"; + } + /** * Generates code like `arg1, arg2, arg3` from an argument list node. */ - public function visitArgumentList(ArgumentList $node): string { + public function visitArgumentList(ArgumentListNode $node): string { $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } @@ -50,7 +57,7 @@ public function visitArgumentList(ArgumentList $node): string { /** * Generates code like `[element1, element2, element3]` from an array list node. */ - public function visitArrayList(ArrayList $node): string { + public function visitArrayList(ArrayListNode $node): string { $elements = array_map(fn($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } @@ -58,7 +65,7 @@ public function visitArrayList(ArrayList $node): string { /** * Generates code like `$left ?? $right` from a coalesce node. */ - public function visitCoalesce(Coalesce $node): string { + public function visitCoalesce(CoalesceNode $node): string { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; @@ -67,14 +74,14 @@ public function visitCoalesce(Coalesce $node): string { /** * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. */ - public function visitLiteral(Literal $node): string { - return var_export($node->value, true); + public function visitLiteral(LiteralNode $node): string { + return '$intercept(' . var_export($node->value, true) . ')'; } /** * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. */ - public function visitMemberAccess(MemberAccess $node): string { + public function visitMemberAccess(MemberAccessNode $node): string { $object = $node->object->accept($this); $member = $node->member; @@ -86,16 +93,16 @@ public function visitMemberAccess(MemberAccess $node): string { $arguments = $node->arguments->accept($this); $member = var_export($member, true); - return "Runtime::access($object, $memberStr, $nullSafe, $arguments)"; + return $this->intercept("Runtime::access($object, $memberStr, $nullSafe, $arguments)"); } - return "Runtime::access($object, $memberStr, $nullSafe)"; + return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } /** * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. */ - public function visitTernary(Ternary $node): string { + public function visitTernary(TernaryNode $node): string { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); @@ -107,17 +114,17 @@ public function visitTernary(Ternary $node): string { } } - public function visitVariable(Variable $node): string { + public function visitVariable(VariableNode $node): string { $name = $node->name; $namestr = var_export($name, true); - $key = "_" . crc32($name); + $key = "_$name"; if(isset($this->directAccessFor[$name])) { - return "$$key"; + return $this->intercept("$$key"); } if(!isset($this->mappings[$key])) { - $this->mappings[$key] = "(match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null })"; + $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } return "\$$key"; @@ -126,7 +133,7 @@ public function visitVariable(Variable $node): string { /** * Generates code like `$functions['function']($arguments)` from a global function node. */ - public function visitGlobalFunction(GlobalFunction $node): string { + public function visitGlobalFunction(GlobalFunctionNode $node): string { $name = $node->name; if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); @@ -135,14 +142,14 @@ public function visitGlobalFunction(GlobalFunction $node): string { $arguments = $node->arguments->accept($this); $name = var_export($name, true); - return "\$functions[$name]($arguments)"; + return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } - public function visitClosure(Closure $node): mixed { + public function visitClosure(ClosureNode $node): mixed { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; $names = array_map(fn($n) => $n->name, $node->arguments->arguments); - $args = array_map(fn(string $n) => '$_' . crc32($n), $names); + $args = array_map(fn(string $n) => "\$_$n", $names); $args = join(', ', $args); $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 034ba33da0..5c72961029 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -2,16 +2,17 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Closure; use Exception; -use Kirby\Toolkit\Query\AST\ArgumentList; -use Kirby\Toolkit\Query\AST\ArrayList; -use Kirby\Toolkit\Query\AST\Closure; -use Kirby\Toolkit\Query\AST\Coalesce; -use Kirby\Toolkit\Query\AST\Literal; -use Kirby\Toolkit\Query\AST\MemberAccess; -use Kirby\Toolkit\Query\AST\Ternary; -use Kirby\Toolkit\Query\AST\Variable; -use Kirby\Toolkit\Query\AST\GlobalFunction; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; @@ -29,31 +30,46 @@ public function __construct( public array $context = [] ) {} - public function visitArgumentList(ArgumentList $node): array { + public function visitArgumentList(ArgumentListNode $node): array { return array_map(fn($argument) => $argument->accept($this), $node->arguments); } - public function visitArrayList(ArrayList $node): mixed { + public function visitArrayList(ArrayListNode $node): mixed { return array_map(fn($element) => $element->accept($this), $node->elements); } - public function visitCoalesce(Coalesce $node): mixed { + public function visitCoalesce(CoalesceNode $node): mixed { return $node->left->accept($this) ?? $node->right->accept($this); } - public function visitLiteral(Literal $node): mixed { - return $node->value; + public function visitLiteral(LiteralNode $node): mixed { + $val = $node->value; + + if($this->interceptor !== null) { + $val = ($this->interceptor)($val); + } + + return $val; } - public function visitMemberAccess(MemberAccess $node): mixed { + public function visitMemberAccess(MemberAccessNode $node): mixed { $left = $node->object->accept($this); + + $item = null; if($node->arguments !== null) { - return Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + $item = Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + } else { + $item = Runtime::access($left, $node->member, $node->nullSafe); + } + + if($this->interceptor !== null) { + $item = ($this->interceptor)($item); } - return Runtime::access($left, $node->member, $node->nullSafe); + + return $item; } - public function visitTernary(Ternary $node): mixed { + public function visitTernary(TernaryNode $node): mixed { if($node->trueBranchIsDefault) { return $node->condition->accept($this) ?: $node->trueBranch->accept($this); } else { @@ -61,29 +77,43 @@ public function visitTernary(Ternary $node): mixed { } } - public function visitVariable(Variable $node): mixed { + public function visitVariable(VariableNode $node): mixed { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence - if(isset($this->context[$node->name])) { - return $this->context[$node->name]; - } + $item = match (true) { + isset($this->context[$node->name]) => $this->context[$node->name], + isset($this->validGlobalFunctions[$node->name]) => $this->validGlobalFunctions[$node->name](), + default => null, + }; - if(isset($this->validGlobalFunctions[$node->name])) { - return $this->validGlobalFunctions[$node->name](); + if($this->interceptor !== null) { + $item = ($this->interceptor)($item); } - return null; + return $item; } - public function visitGlobalFunction(GlobalFunction $node): mixed { + public function visitGlobalFunction(GlobalFunctionNode $node): mixed { if(!isset($this->validGlobalFunctions[$node->name])) { throw new Exception("Invalid global function $node->name"); } - return $this->validGlobalFunctions[$node->name](...$node->arguments->accept($this)); + + $function = $this->validGlobalFunctions[$node->name]; + if($this->interceptor !== null) { + $function = ($this->interceptor)($function); + } + + $result = $function(...$node->arguments->accept($this)); + + if($this->interceptor !== null) { + $result = ($this->interceptor)($result); + } + + return $result; } - public function visitClosure(Closure $node): mixed { + public function visitClosure(ClosureNode $node): mixed { $self = $this; return function(...$params) use ($self, $node) { @@ -96,6 +126,9 @@ public function visitClosure(Closure $node): mixed { ); $visitor = new self($functions, [...$context, ...$arguments]); + if($self->interceptor !== null) { + $visitor->setInterceptor($self->interceptor); + } return $node->body->accept($visitor); }; diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index f1cedbcc45..2f24a75a52 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -2,13 +2,19 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; use ReflectionClass; abstract class Visitor { + protected Closure|null $interceptor = null; + function visitNode(AST\Node $node): mixed { $shortName = (new ReflectionClass($node))->getShortName(); + // remove the "Node" suffix + $shortName = substr($shortName, 0, -4); + $method = 'visit' . $shortName; if(method_exists($this, $method)) { return $this->$method($node); @@ -17,13 +23,20 @@ function visitNode(AST\Node $node): mixed { throw new Exception("No visitor method for " . $node::class); } - abstract function visitArgumentList(AST\ArgumentList $node): mixed; - abstract function visitArrayList(AST\ArrayList $node): mixed; - abstract function visitCoalesce(AST\Coalesce $node): mixed; - abstract function visitLiteral(AST\Literal $node): mixed; - abstract function visitMemberAccess(AST\MemberAccess $node): mixed; - abstract function visitTernary(AST\Ternary $node): mixed; - abstract function visitVariable(AST\Variable $node): mixed; - abstract function visitGlobalFunction(AST\GlobalFunction $node): mixed; - abstract function visitClosure(AST\Closure $node): mixed; + abstract function visitArgumentList(AST\ArgumentListNode $node): mixed; + abstract function visitArrayList(AST\ArrayListNode $node): mixed; + abstract function visitCoalesce(AST\CoalesceNode $node): mixed; + abstract function visitLiteral(AST\LiteralNode $node): mixed; + abstract function visitMemberAccess(AST\MemberAccessNode $node): mixed; + abstract function visitTernary(AST\TernaryNode $node): mixed; + abstract function visitVariable(AST\VariableNode $node): mixed; + abstract function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; + abstract function visitClosure(AST\ClosureNode $node): mixed; + + /** + * Sets and activates an interceptor closure that is called for each resolved value. + */ + public function setInterceptor(Closure $interceptor): void { + $this->interceptor = $interceptor; + } } diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index b23f1025bf..e9813ae9dd 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -108,4 +108,36 @@ public function testResolveWithClosureWithArgument() $bar = $bar(['homer' => 'simpson']); $this->assertSame('simpson', $bar); } + + /** + * @covers ::intercept + */ + public function testResolveWithInterceptor() + { + $query = new class extends Query { + public function __construct() + { + parent::__construct('foo.getObj.name'); + } + + public function intercept($result): mixed + { + if(is_object($result) === true) { + $result = clone $result; + $result->name .= ' simpson'; + } + + return $result; + } + }; + + $data = [ + 'foo' => [ + 'getObj' => fn () => (object)['name' => 'homer'] + ] + ]; + + $bar = $query->resolve($data); + $this->assertSame('homer simpson', $bar); + } } From b599e66669ed2600a4519ceef987ee29a0843779 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 17:18:11 +0100 Subject: [PATCH 19/31] fix type annotations for arrays in CodeGen and Interpreter classes --- src/Toolkit/Query/Runners/Visitors/CodeGen.php | 7 ++++--- src/Toolkit/Query/Runners/Visitors/Interpreter.php | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 77972c553c..06ba3a606a 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners\Visitors; +use Closure; use Exception; use Kirby\Toolkit\Query\AST\ArgumentListNode; use Kirby\Toolkit\Query\AST\ArrayListNode; @@ -25,19 +26,19 @@ class CodeGen extends Visitor { /** * If we need something from a namespace, we'll add the namespace here into the array key - * @var array{string:true} + * @var array */ public array $uses = []; /** - * @var array{string:string} + * @var array */ public array $mappings = []; /** * CodeGen constructor. * - * @param array{string:PHPClosure} $validGlobalFunctions An array of valid global function closures. + * @param array $validGlobalFunctions An array of valid global function closures. */ public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 5c72961029..34f041d2f6 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -22,8 +22,8 @@ */ class Interpreter extends Visitor { /** - * @param array{string:Closure} $validGlobalFunctions An array of valid global function closures. - * @param array{string:mixed} $context The data bindings for the query. + * @param array $validGlobalFunctions An array of valid global function closures. + * @param array $context The data bindings for the query. */ public function __construct( public array $validGlobalFunctions = [], From 90b1715980ac9b69d4cfcab9af08915e28755a0d Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 12 Nov 2024 17:20:30 +0100 Subject: [PATCH 20/31] rename parameter in run method from bindings to context to be fully compatible with parent --- src/Toolkit/Query/Runners/Interpreted.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 055b74b10d..455fcb1bec 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -37,8 +37,8 @@ protected function getResolver(string $query): Closure { }; } - public function run(string $query, array $bindings = []): mixed { + public function run(string $query, array $context = []): mixed { $resolver = $this->getResolver($query); - return $resolver($bindings); + return $resolver($context); } } From 42fbd4fc5892d0e21019027a3261bef5570e02c6 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:44:33 +0100 Subject: [PATCH 21/31] don't use option helper --- src/Query/Query.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 9f1455c8c7..38383f7a77 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -109,11 +109,13 @@ public function resolve(array|object $data = []): mixed return $data; } - if(option('query.runner', 'interpreted') == 'legacy') { + $mode = App::instance()->option('query.runner', 'transpiled'); + + if($mode === 'legacy') { return $this->resolve_legacy($data); } - $runnerClass = match(option('query.runner', 'interpreted')) { + $runnerClass = match($mode) { 'transpiled' => Transpiled::class, 'interpreted' => Interpreted::class, default => throw new Exception('Invalid query runner') From 96b8a41e6c05d62539c8bfdf3cebe2f7de036fcc Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:49:23 +0100 Subject: [PATCH 22/31] Update token type constants to include "T_" prefix for safety --- src/Toolkit/Query/BaseParser.php | 2 +- src/Toolkit/Query/Parser.php | 52 ++++++++++---------- src/Toolkit/Query/TokenType.php | 40 ++++++++-------- src/Toolkit/Query/Tokenizer.php | 82 ++++++++++++++++++++++---------- 4 files changed, 104 insertions(+), 72 deletions(-) diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index 5c4f1a96d8..f7cce1e872 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -60,7 +60,7 @@ protected function advance(): ?Token { } protected function isAtEnd(): bool { - return $this->current->type === TokenType::EOF; + return $this->current->type === TokenType::T_EOF; } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index bde8c4d078..01a5e4ecac 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -27,7 +27,7 @@ public function parse(): Node { // ensure that we consumed all tokens if(!$this->isAtEnd()) - $this->consume(TokenType::EOF, 'Expect end of expression.'); + $this->consume(TokenType::T_EOF, 'Expect end of expression.'); return $expression; } @@ -39,7 +39,7 @@ private function expression(): Node { private function coalesce(): Node { $left = $this->ternary(); - while ($this->match(TokenType::COALESCE)) { + while ($this->match(TokenType::T_COALESCE)) { $right = $this->ternary(); $left = new CoalesceNode($left, $right); } @@ -50,15 +50,15 @@ private function coalesce(): Node { private function ternary(): Node { $left = $this->memberAccess(); - if ($tok = $this->matchAny([TokenType::QUESTION_MARK, TokenType::TERNARY_DEFAULT])) { - if($tok->type === TokenType::TERNARY_DEFAULT) { + if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { + if($tok->type === TokenType::T_TERNARY_DEFAULT) { $trueIsDefault = true; $trueBranch = null; $falseBranch = $this->expression(); } else { $trueIsDefault = false; $trueBranch = $this->expression(); - $this->consume(TokenType::COLON, 'Expect ":" after true branch.'); + $this->consume(TokenType::T_COLON, 'Expect ":" after true branch.'); $falseBranch = $this->expression(); } @@ -71,18 +71,18 @@ private function ternary(): Node { private function memberAccess(): Node { $left = $this->atomic(); - while ($tok = $this->matchAny([TokenType::DOT, TokenType::NULLSAFE])) { - $nullSafe = $tok->type === TokenType::NULLSAFE; + while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { + $nullSafe = $tok->type === TokenType::T_NULLSAFE; - if($right = $this->match(TokenType::IDENTIFIER)) { + if($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } else if($right = $this->match(TokenType::INTEGER)) { + } else if($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); } - if($this->match(TokenType::OPEN_PAREN)) { + if($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); } else { @@ -99,7 +99,7 @@ private function listUntil(TokenType $until): array { while (!$this->isAtEnd() && !$this->check($until)) { $elements[] = $this->expression(); - if (!$this->match(TokenType::COMMA)) { + if (!$this->match(TokenType::T_COMMA)) { break; } } @@ -111,16 +111,16 @@ private function listUntil(TokenType $until): array { } private function argumentList(): Node { - $list = $this->listUntil(TokenType::CLOSE_PAREN); + $list = $this->listUntil(TokenType::T_CLOSE_PAREN); return new ArgumentListNode($list); } private function atomic(): Node { // float numbers - if ($integer = $this->match(TokenType::INTEGER)) { - if($this->match(TokenType::DOT)) { - $fractional = $this->match(TokenType::INTEGER); + if ($integer = $this->match(TokenType::T_INTEGER)) { + if($this->match(TokenType::T_DOT)) { + $fractional = $this->match(TokenType::T_INTEGER); return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); } return new LiteralNode($integer->literal); @@ -128,24 +128,24 @@ private function atomic(): Node { // primitives if ($token = $this->matchAny([ - TokenType::TRUE, - TokenType::FALSE, - TokenType::NULL, - TokenType::STRING, + TokenType::T_TRUE, + TokenType::T_FALSE, + TokenType::T_NULL, + TokenType::T_STRING, ])) { return new LiteralNode($token->literal); } // array literals - if ($token = $this->match(TokenType::OPEN_BRACKET)) { - $arrayItems = $this->listUntil(TokenType::CLOSE_BRACKET); + if ($token = $this->match(TokenType::T_OPEN_BRACKET)) { + $arrayItems = $this->listUntil(TokenType::T_CLOSE_BRACKET); return new ArrayListNode($arrayItems); } // global functions and variables - if ($token = $this->match(TokenType::IDENTIFIER)) { - if($this->match(TokenType::OPEN_PAREN)) { + if ($token = $this->match(TokenType::T_IDENTIFIER)) { + if($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); return new GlobalFunctionNode($token->lexeme, $arguments); } @@ -154,10 +154,10 @@ private function atomic(): Node { } // grouping and closure argument lists - if ($token = $this->match(TokenType::OPEN_PAREN)) { - $list = $this->listUntil(TokenType::CLOSE_PAREN); + if ($token = $this->match(TokenType::T_OPEN_PAREN)) { + $list = $this->listUntil(TokenType::T_CLOSE_PAREN); - if($this->match(TokenType::ARROW)) { + if($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); // check if all elements are variables foreach($list as $element) { diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index dcfe904596..2cdc14c93a 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -3,24 +3,24 @@ namespace Kirby\Toolkit\Query; enum TokenType { - case STRING; - case INTEGER; - case WHITESPACE; - case IDENTIFIER; - case DOT; - case OPEN_PAREN; - case CLOSE_PAREN; - case OPEN_BRACKET; - case CLOSE_BRACKET; - case QUESTION_MARK; - case TERNARY_DEFAULT; // ?: - case NULLSAFE; // ?. - case COLON; - case COALESCE; // ?? - case COMMA; - case EOF; - case TRUE; - case FALSE; - case NULL; - case ARROW; + case T_STRING; + case T_INTEGER; + case T_WHITESPACE; + case T_IDENTIFIER; + case T_DOT; + case T_OPEN_PAREN; + case T_CLOSE_PAREN; + case T_OPEN_BRACKET; + case T_CLOSE_BRACKET; + case T_QUESTION_MARK; + case T_TERNARY_DEFAULT; // ?: + case T_NULLSAFE; // ?. + case T_COLON; + case T_COALESCE; // ?? + case T_COMMA; + case T_EOF; + case T_TRUE; + case T_FALSE; + case T_NULL; + case T_ARROW; } diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 430499931d..a79f49051b 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -8,6 +8,24 @@ class Tokenizer { private int $length = 0; + /** + * The more complex regexes are written here in nowdoc format so we don't need to double or triple escape backslashes (that becomes ridiculous rather fast). + */ + + // Identifiers can contain letters, numbers, underscores and escaped dots. They can't start with a number. + // to match an array key like "foo.bar" we write the query as `foo\.bar`, to match an array key like "foo\.bar" we write the query as `foo\\.bar` + private const IDENTIFIER_REGEX = <<<'REGEX' + (?:[\p{L}\p{N}_]|\\\.|\\\\)* + REGEX; + + private const SINGLEQUOTE_STRING_REGEX = <<<'REGEX' + '([^'\\]*(?:\\.[^'\\]*)*)' + REGEX; + + private const DOUBLEQUOTE_STRING_REGEX = <<<'REGEX' + "([^"\\]*(?:\\.[^"\\]*)*)" + REGEX; + public function __construct( private readonly string $source, ) { @@ -24,47 +42,55 @@ public function tokenize(): Generator { while ($current < $this->length) { $t = self::scanToken($this->source, $current); // don't yield whitespace tokens (ignore them) - if($t->type !== TokenType::WHITESPACE) { + if($t->type !== TokenType::T_WHITESPACE) { yield $t; } $current += mb_strlen($t->lexeme); } - yield new Token(TokenType::EOF, '', null); + yield new Token(TokenType::T_EOF, '', null); } + /** + * Scans the source string for a token starting at the given position. + * @param string $source The source string + * @param int $current The current position in the source string + * + * @return Token The scanned token + * @throws Exception If an unexpected character is encountered + */ protected static function scanToken(string $source, int $current): Token { $l = ''; $c = $source[$current]; return match(true) { // single character tokens - $c === '.' => new Token(TokenType::DOT, '.'), - $c === '(' => new Token(TokenType::OPEN_PAREN, '('), - $c === ')' => new Token(TokenType::CLOSE_PAREN, ')'), - $c === '[' => new Token(TokenType::OPEN_BRACKET, '['), - $c === ']' => new Token(TokenType::CLOSE_BRACKET, ']'), - $c === ',' => new Token(TokenType::COMMA, ','), - $c === ':' => new Token(TokenType::COLON, ':'), + $c === '.' => new Token(TokenType::T_DOT, '.'), + $c === '(' => new Token(TokenType::T_OPEN_PAREN, '('), + $c === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), + $c === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), + $c === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), + $c === ',' => new Token(TokenType::T_COMMA, ','), + $c === ':' => new Token(TokenType::T_COLON, ':'), // two character tokens - self::match($source, $current, '\\?\\?', $l) => new Token(TokenType::COALESCE, $l), - self::match($source, $current, '\\?\\s*\\.', $l) => new Token(TokenType::NULLSAFE, $l), - self::match($source, $current, '\\?\\s*:', $l) => new Token(TokenType::TERNARY_DEFAULT, $l), - self::match($source, $current, '=>', $l) => new Token(TokenType::ARROW, $l), + self::match($source, $current, '\?\?', $l) => new Token(TokenType::T_COALESCE, $l), + self::match($source, $current, '\?\s*\.', $l) => new Token(TokenType::T_NULLSAFE, $l), + self::match($source, $current, '\?\s*:', $l) => new Token(TokenType::T_TERNARY_DEFAULT, $l), + self::match($source, $current, '=>', $l) => new Token(TokenType::T_ARROW, $l), // make sure this check comes after the two above that check for '?' in the beginning - $c === '?' => new Token(TokenType::QUESTION_MARK, '?'), + $c === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), // multi character tokens - self::match($source, $current, '\\s+', $l) => new Token(TokenType::WHITESPACE, $l), - self::match($source, $current, 'true', $l, true) => new Token(TokenType::TRUE, $l, true), - self::match($source, $current, 'false', $l, true) => new Token(TokenType::FALSE, $l, false), - self::match($source, $current, 'null', $l, true) => new Token(TokenType::NULL, $l, null), - self::match($source, $current, '"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'', $l) => new Token(TokenType::STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '[0-9]+', $l) => new Token(TokenType::INTEGER, $l, intval($l)), - self::match($source, $current, '[a-zA-Z_][a-zA-Z0-9_]*', $l) => new Token(TokenType::IDENTIFIER, $l), + self::match($source, $current, '\s+', $l) => new Token(TokenType::T_WHITESPACE, $l), + self::match($source, $current, 'true', $l, true) => new Token(TokenType::T_TRUE, $l, true), + self::match($source, $current, 'false', $l, true) => new Token(TokenType::T_FALSE, $l, false), + self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), + self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), + self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, intval($l)), + self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), // unknown token default => throw new Exception("Unexpected character: {$source[$current]}"), @@ -72,9 +98,15 @@ protected static function scanToken(string $source, int $current): Token { } /** - * Checks if a given regex matches the current position in the source. Returns the matched string or false. Advances the current position when a match is found. - * @param string $regex - * @return string|false + * Matches a regex pattern at the current position in the source string. + * The matched lexeme will be stored in the $lexeme variable. + * + * @param string $source The source string + * @param int $current The current position in the source string (used as offset for the regex) + * @param string $regex The regex pattern to match (without delimiters / flags) + * @param string $lexeme The matched lexeme will be stored in this variable + * @param bool $caseIgnore Whether to ignore case while matching + * @return bool Whether the regex pattern was matched */ protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { $regex = '/\G' . $regex . '/u'; From 0c8b8cdcfc39922d1429d02708c738fdf20b735d Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 12:50:20 +0100 Subject: [PATCH 23/31] allow identifiers to start with a number and contain (escaped) dots --- src/Toolkit/Query/AST/ClosureNode.php | 5 +- src/Toolkit/Query/AST/GlobalFunctionNode.php | 9 +++- src/Toolkit/Query/AST/IdentifierNode.php | 12 +++++ src/Toolkit/Query/AST/MemberAccessNode.php | 13 ++++- src/Toolkit/Query/AST/VariableNode.php | 9 +++- src/Toolkit/Query/Parser.php | 9 +++- src/Toolkit/Query/Runners/Transpiled.php | 2 +- .../Query/Runners/Visitors/CodeGen.php | 49 ++++++++----------- .../Query/Runners/Visitors/Interpreter.php | 17 ++++--- tests/Query/QueryTest.php | 17 ++++++- 10 files changed, 99 insertions(+), 43 deletions(-) create mode 100644 src/Toolkit/Query/AST/IdentifierNode.php diff --git a/src/Toolkit/Query/AST/ClosureNode.php b/src/Toolkit/Query/AST/ClosureNode.php index 83a26c68de..8eb1022845 100644 --- a/src/Toolkit/Query/AST/ClosureNode.php +++ b/src/Toolkit/Query/AST/ClosureNode.php @@ -4,8 +4,11 @@ class ClosureNode extends Node { + /** + * @param string[] $arguments The arguments names + */ public function __construct( - public ArgumentListNode $arguments, + public array $arguments, public Node $body, ) { } diff --git a/src/Toolkit/Query/AST/GlobalFunctionNode.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php index ab59bee65e..22674c8d49 100644 --- a/src/Toolkit/Query/AST/GlobalFunctionNode.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,9 +2,16 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunctionNode extends Node { +class GlobalFunctionNode extends IdentifierNode { public function __construct( public string $name, public ArgumentListNode $arguments, ) {} + + /** + * Replace escaped dots with real dots + */ + public function name(): string { + return str_replace('\.', '.', $this->name); + } } diff --git a/src/Toolkit/Query/AST/IdentifierNode.php b/src/Toolkit/Query/AST/IdentifierNode.php new file mode 100644 index 0000000000..e394576637 --- /dev/null +++ b/src/Toolkit/Query/AST/IdentifierNode.php @@ -0,0 +1,12 @@ +member)) { + return self::unescape($this->member); + } else { + return $this->member; + } + } } diff --git a/src/Toolkit/Query/AST/VariableNode.php b/src/Toolkit/Query/AST/VariableNode.php index 2088743eec..8ab46c2826 100644 --- a/src/Toolkit/Query/AST/VariableNode.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,8 +2,15 @@ namespace Kirby\Toolkit\Query\AST; -class VariableNode extends Node { +class VariableNode extends IdentifierNode { public function __construct( public string $name, ) {} + + /** + * Replaces escaped dots with real dots + */ + public function name(): string { + return self::unescape($this->name); + } } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 01a5e4ecac..7785c3424b 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -159,13 +159,18 @@ private function atomic(): Node { if($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); - // check if all elements are variables + + /** + * Assert that all elements are VariableNodes + * @var VariableNode[] $list + */ foreach($list as $element) { if(!$element instanceof VariableNode) { throw new Exception('Expecting only variables in closure argument list.'); } } - $arguments = new ArgumentListNode($list); + + $arguments = array_map(fn($element) => $element->name, $list); return new ClosureNode($arguments, $expression); } else { if(count($list) > 1) { diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 9ca16ffcae..0b67ef88f0 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -51,7 +51,7 @@ protected function getResolver(string $query): Closure { $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn($k, $v) => "$$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $mappings = join("\n", array_map(fn($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 06ba3a606a..13cc9f41d6 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -35,6 +35,19 @@ class CodeGen extends Visitor { */ public array $mappings = []; + + /** + * Variable names in Query Language are different from PHP variable names, + * they can start with a number and may contain escaped dots. + * + * This method returns a sanitized PHP variable name. + * + * @return string + */ + private static function phpName(string $name): string { + return '$_' . crc32($name); + } + /** * CodeGen constructor. * @@ -46,42 +59,26 @@ private function intercept(string $value): string { return "(\$intercept($value))"; } - - /** - * Generates code like `arg1, arg2, arg3` from an argument list node. - */ public function visitArgumentList(ArgumentListNode $node): string { $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } - /** - * Generates code like `[element1, element2, element3]` from an array list node. - */ public function visitArrayList(ArrayListNode $node): string { $elements = array_map(fn($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } - /** - * Generates code like `$left ?? $right` from a coalesce node. - */ public function visitCoalesce(CoalesceNode $node): string { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } - /** - * Generates code like `true`, `false`, `123.45`, `"foo bar"`, etc from a literal node. - */ public function visitLiteral(LiteralNode $node): string { return '$intercept(' . var_export($node->value, true) . ')'; } - /** - * Generates code like `$object->member` or `$object->member($arguments)` from a member access node. - */ public function visitMemberAccess(MemberAccessNode $node): string { $object = $node->object->accept($this); $member = $node->member; @@ -100,9 +97,6 @@ public function visitMemberAccess(MemberAccessNode $node): string { return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } - /** - * Generates code like `($condition ? $trueBranch : $falseBranch)` or `($condition ?: $falseBranch)` from a ternary node. - */ public function visitTernary(TernaryNode $node): string { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); @@ -116,26 +110,26 @@ public function visitTernary(TernaryNode $node): string { } public function visitVariable(VariableNode $node): string { - $name = $node->name; + $name = $node->name(); $namestr = var_export($name, true); - $key = "_$name"; + $key = self::phpName($name); if(isset($this->directAccessFor[$name])) { - return $this->intercept("$$key"); + return $this->intercept($key); } if(!isset($this->mappings[$key])) { - $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); + $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) && \$context[$namestr] instanceof Closure => \$context[$namestr](), isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } - return "\$$key"; + return $key; } /** * Generates code like `$functions['function']($arguments)` from a global function node. */ public function visitGlobalFunction(GlobalFunctionNode $node): string { - $name = $node->name; + $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); } @@ -149,11 +143,10 @@ public function visitGlobalFunction(GlobalFunctionNode $node): string { public function visitClosure(ClosureNode $node): mixed { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; - $names = array_map(fn($n) => $n->name, $node->arguments->arguments); - $args = array_map(fn(string $n) => "\$_$n", $names); + $args = array_map(self::phpName(...), $node->arguments); $args = join(', ', $args); - $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($names, true)); + $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($node->arguments, true)); return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); } diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 34f041d2f6..c800b8ea3b 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -81,9 +81,11 @@ public function visitVariable(VariableNode $node): mixed { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence + $name = $node->name(); + $item = match (true) { - isset($this->context[$node->name]) => $this->context[$node->name], - isset($this->validGlobalFunctions[$node->name]) => $this->validGlobalFunctions[$node->name](), + isset($this->context[$name]) => $this->context[$name] instanceof Closure ? $this->context[$name]() : $this->context[$name], + isset($this->validGlobalFunctions[$name]) => $this->validGlobalFunctions[$name](), default => null, }; @@ -95,11 +97,13 @@ public function visitVariable(VariableNode $node): mixed { } public function visitGlobalFunction(GlobalFunctionNode $node): mixed { - if(!isset($this->validGlobalFunctions[$node->name])) { - throw new Exception("Invalid global function $node->name"); + $name = $node->name(); + + if(!isset($this->validGlobalFunctions[$name])) { + throw new Exception("Invalid global function $name"); } - $function = $this->validGlobalFunctions[$node->name]; + $function = $this->validGlobalFunctions[$name]; if($this->interceptor !== null) { $function = ($this->interceptor)($function); } @@ -120,8 +124,9 @@ public function visitClosure(ClosureNode $node): mixed { $context = $self->context; $functions = $self->validGlobalFunctions; + // [key1, key2] + [value1, value2] => [key1 => value1, key2 => value2] $arguments = array_combine( - array_map(fn($param) => $param->name, $node->arguments->arguments), + $node->arguments, $params ); diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index e9813ae9dd..5ba87109aa 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -67,11 +67,24 @@ public function testResolveWithExactArrayMatch() $query = new Query('user'); $this->assertSame('homer', $query->resolve(['user' => 'homer'])); - $query = new Query('user.username'); + $query = new Query('user\.username'); $this->assertSame('homer', $query->resolve(['user.username' => 'homer'])); - $query = new Query('user.callback'); + $query = new Query('user\.callback'); $this->assertSame('homer', $query->resolve(['user.callback' => fn () => 'homer'])); + + // in the query, the first slash escapes the second, the third escapes the dot + $query = <<<'TXT' + user\\\.username + TXT; + + // this is actually the array key + $key = <<<'TXT' + user\.username + TXT; + + $query = new Query($query); + $this->assertSame('homer', $query->resolve([$key => 'homer'])); } /** From 045867c0df95c44c5c455d38ad418a7428861f25 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 13:46:50 +0100 Subject: [PATCH 24/31] coding style --- src/Toolkit/Query/AST/ArgumentListNode.php | 6 ++- src/Toolkit/Query/AST/ArrayListNode.php | 6 ++- src/Toolkit/Query/AST/CoalesceNode.php | 6 ++- src/Toolkit/Query/AST/GlobalFunctionNode.php | 9 ++-- src/Toolkit/Query/AST/IdentifierNode.php | 6 ++- src/Toolkit/Query/AST/LiteralNode.php | 6 ++- src/Toolkit/Query/AST/MemberAccessNode.php | 15 +++--- src/Toolkit/Query/AST/Node.php | 6 ++- src/Toolkit/Query/AST/TernaryNode.php | 9 ++-- src/Toolkit/Query/AST/VariableNode.php | 9 ++-- src/Toolkit/Query/BaseParser.php | 23 +++++--- src/Toolkit/Query/Parser.php | 50 ++++++++++------- src/Toolkit/Query/Runner.php | 6 ++- src/Toolkit/Query/Runners/Interpreted.php | 14 +++-- src/Toolkit/Query/Runners/Transpiled.php | 22 ++++---- .../Query/Runners/Visitors/CodeGen.php | 54 +++++++++++-------- .../Query/Runners/Visitors/Interpreter.php | 46 +++++++++------- src/Toolkit/Query/Runtime.php | 13 +++-- src/Toolkit/Query/Token.php | 6 ++- src/Toolkit/Query/TokenType.php | 3 +- src/Toolkit/Query/Tokenizer.php | 14 +++-- src/Toolkit/Query/Visitor.php | 29 +++++----- tests/Query/QueryTest.php | 7 +-- 23 files changed, 221 insertions(+), 144 deletions(-) diff --git a/src/Toolkit/Query/AST/ArgumentListNode.php b/src/Toolkit/Query/AST/ArgumentListNode.php index 1980223203..b6d9d3bf83 100644 --- a/src/Toolkit/Query/AST/ArgumentListNode.php +++ b/src/Toolkit/Query/AST/ArgumentListNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class ArgumentListNode extends Node { +class ArgumentListNode extends Node +{ public function __construct( public array $arguments, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/ArrayListNode.php b/src/Toolkit/Query/AST/ArrayListNode.php index 0b092d7cb2..a60ce194fb 100644 --- a/src/Toolkit/Query/AST/ArrayListNode.php +++ b/src/Toolkit/Query/AST/ArrayListNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class ArrayListNode extends Node { +class ArrayListNode extends Node +{ public function __construct( public array $elements, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/CoalesceNode.php b/src/Toolkit/Query/AST/CoalesceNode.php index 1873347123..9a216ac648 100644 --- a/src/Toolkit/Query/AST/CoalesceNode.php +++ b/src/Toolkit/Query/AST/CoalesceNode.php @@ -2,9 +2,11 @@ namespace Kirby\Toolkit\Query\AST; -class CoalesceNode extends Node { +class CoalesceNode extends Node +{ public function __construct( public Node $left, public Node $right, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/GlobalFunctionNode.php b/src/Toolkit/Query/AST/GlobalFunctionNode.php index 22674c8d49..2f0da7cd0b 100644 --- a/src/Toolkit/Query/AST/GlobalFunctionNode.php +++ b/src/Toolkit/Query/AST/GlobalFunctionNode.php @@ -2,16 +2,19 @@ namespace Kirby\Toolkit\Query\AST; -class GlobalFunctionNode extends IdentifierNode { +class GlobalFunctionNode extends IdentifierNode +{ public function __construct( public string $name, public ArgumentListNode $arguments, - ) {} + ) { + } /** * Replace escaped dots with real dots */ - public function name(): string { + public function name(): string + { return str_replace('\.', '.', $this->name); } } diff --git a/src/Toolkit/Query/AST/IdentifierNode.php b/src/Toolkit/Query/AST/IdentifierNode.php index e394576637..5c07dcd325 100644 --- a/src/Toolkit/Query/AST/IdentifierNode.php +++ b/src/Toolkit/Query/AST/IdentifierNode.php @@ -2,11 +2,13 @@ namespace Kirby\Toolkit\Query\AST; -abstract class IdentifierNode extends Node { +abstract class IdentifierNode extends Node +{ /** * Replaces the escaped identifier with the actual identifier */ - static public function unescape(string $name): string { + public static function unescape(string $name): string + { return stripslashes($name); } } diff --git a/src/Toolkit/Query/AST/LiteralNode.php b/src/Toolkit/Query/AST/LiteralNode.php index fc52cfc9eb..7976f127b3 100644 --- a/src/Toolkit/Query/AST/LiteralNode.php +++ b/src/Toolkit/Query/AST/LiteralNode.php @@ -2,8 +2,10 @@ namespace Kirby\Toolkit\Query\AST; -class LiteralNode extends Node { +class LiteralNode extends Node +{ public function __construct( public mixed $value, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/MemberAccessNode.php b/src/Toolkit/Query/AST/MemberAccessNode.php index deaa069931..7bcc5ec51e 100644 --- a/src/Toolkit/Query/AST/MemberAccessNode.php +++ b/src/Toolkit/Query/AST/MemberAccessNode.php @@ -2,22 +2,25 @@ namespace Kirby\Toolkit\Query\AST; -class MemberAccessNode extends IdentifierNode { +class MemberAccessNode extends IdentifierNode +{ public function __construct( public Node $object, public string|int $member, - public ?ArgumentListNode $arguments = null, + public ArgumentListNode|null $arguments = null, public bool $nullSafe = false, - ) {} + ) { + } /** * Returns the member name and replaces escaped dots with real dots if it's a string */ - public function member(): string|int { + public function member(): string|int + { if (is_string($this->member)) { return self::unescape($this->member); - } else { - return $this->member; } + return $this->member; + } } diff --git a/src/Toolkit/Query/AST/Node.php b/src/Toolkit/Query/AST/Node.php index c7507d6d99..3da71795da 100644 --- a/src/Toolkit/Query/AST/Node.php +++ b/src/Toolkit/Query/AST/Node.php @@ -4,8 +4,10 @@ use Kirby\Toolkit\Query\Visitor; -class Node { - public function accept(Visitor $visitor) { +class Node +{ + public function accept(Visitor $visitor) + { return $visitor->visitNode($this); } } diff --git a/src/Toolkit/Query/AST/TernaryNode.php b/src/Toolkit/Query/AST/TernaryNode.php index 38e662ba03..33aa203175 100644 --- a/src/Toolkit/Query/AST/TernaryNode.php +++ b/src/Toolkit/Query/AST/TernaryNode.php @@ -2,12 +2,13 @@ namespace Kirby\Toolkit\Query\AST; -class TernaryNode extends Node { +class TernaryNode extends Node +{ public function __construct( public Node $condition, - public ?Node $trueBranch, + public Node|null $trueBranch, public Node $falseBranch, - public bool $trueBranchIsDefault = false, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/AST/VariableNode.php b/src/Toolkit/Query/AST/VariableNode.php index 8ab46c2826..495bd3979f 100644 --- a/src/Toolkit/Query/AST/VariableNode.php +++ b/src/Toolkit/Query/AST/VariableNode.php @@ -2,15 +2,18 @@ namespace Kirby\Toolkit\Query\AST; -class VariableNode extends IdentifierNode { +class VariableNode extends IdentifierNode +{ public function __construct( public string $name, - ) {} + ) { + } /** * Replaces escaped dots with real dots */ - public function name(): string { + public function name(): string + { return self::unescape($this->name); } } diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index f7cce1e872..b597cf391a 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -5,8 +5,9 @@ use Exception; use Iterator; -abstract class BaseParser { - protected ?Token $previous; +abstract class BaseParser +{ + protected Token|null $previous; protected Token $current; /** @@ -33,7 +34,8 @@ public function __construct( $this->current = $first; } - protected function consume(TokenType $type, string $message): Token { + protected function consume(TokenType $type, string $message): Token + { if ($this->check($type)) { return $this->advance(); } @@ -41,7 +43,8 @@ protected function consume(TokenType $type, string $message): Token { throw new Exception($message); } - protected function check(TokenType $type): bool { + protected function check(TokenType $type): bool + { if ($this->isAtEnd()) { return false; } @@ -49,7 +52,8 @@ protected function check(TokenType $type): bool { return $this->current->type === $type; } - protected function advance(): ?Token { + protected function advance(): Token|null + { if (!$this->isAtEnd()) { $this->previous = $this->current; $this->tokens->next(); @@ -59,12 +63,14 @@ protected function advance(): ?Token { return $this->previous; } - protected function isAtEnd(): bool { + protected function isAtEnd(): bool + { return $this->current->type === TokenType::T_EOF; } - protected function match(TokenType $type): Token|false { + protected function match(TokenType $type): Token|false + { if ($this->check($type)) { return $this->advance(); } @@ -72,7 +78,8 @@ protected function match(TokenType $type): Token|false { return false; } - protected function matchAny(array $types): Token|false { + protected function matchAny(array $types): Token|false + { foreach ($types as $type) { if ($this->check($type)) { return $this->advance(); diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 7785c3424b..8be677f1bf 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -15,28 +15,33 @@ use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; -class Parser extends BaseParser { +class Parser extends BaseParser +{ public function __construct( Tokenizer|Iterator $source, ) { parent::__construct($source); } - public function parse(): Node { + public function parse(): Node + { $expression = $this->expression(); // ensure that we consumed all tokens - if(!$this->isAtEnd()) + if(!$this->isAtEnd()) { $this->consume(TokenType::T_EOF, 'Expect end of expression.'); + } return $expression; } - private function expression(): Node { + private function expression(): Node + { return $this->coalesce(); } - private function coalesce(): Node { + private function coalesce(): Node + { $left = $this->ternary(); while ($this->match(TokenType::T_COALESCE)) { @@ -47,7 +52,8 @@ private function coalesce(): Node { return $left; } - private function ternary(): Node { + private function ternary(): Node + { $left = $this->memberAccess(); if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { @@ -68,7 +74,8 @@ private function ternary(): Node { return $left; } - private function memberAccess(): Node { + private function memberAccess(): Node + { $left = $this->atomic(); while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { @@ -76,7 +83,7 @@ private function memberAccess(): Node { if($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } else if($right = $this->match(TokenType::T_INTEGER)) { + } elseif($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); @@ -93,7 +100,8 @@ private function memberAccess(): Node { return $left; } - private function listUntil(TokenType $until): array { + private function listUntil(TokenType $until): array + { $elements = []; while (!$this->isAtEnd() && !$this->check($until)) { @@ -110,18 +118,20 @@ private function listUntil(TokenType $until): array { return $elements; } - private function argumentList(): Node { + private function argumentList(): Node + { $list = $this->listUntil(TokenType::T_CLOSE_PAREN); return new ArgumentListNode($list); } - private function atomic(): Node { + private function atomic(): Node + { // float numbers if ($integer = $this->match(TokenType::T_INTEGER)) { if($this->match(TokenType::T_DOT)) { $fractional = $this->match(TokenType::T_INTEGER); - return new LiteralNode(floatval($integer->literal . '.' . $fractional->literal)); + return new LiteralNode((float)($integer->literal . '.' . $fractional->literal)); } return new LiteralNode($integer->literal); } @@ -170,16 +180,16 @@ private function atomic(): Node { } } - $arguments = array_map(fn($element) => $element->name, $list); + $arguments = array_map(fn ($element) => $element->name, $list); return new ClosureNode($arguments, $expression); - } else { - if(count($list) > 1) { - throw new Exception('Expecting \"=>\" after closure argument list.'); - } else { - // this is just a grouping - return $list[0]; - } } + if(count($list) > 1) { + throw new Exception('Expecting \"=>\" after closure argument list.'); + } + // this is just a grouping + return $list[0]; + + } throw new Exception('Expect expression.'); diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index b327194d44..d01b295660 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -5,7 +5,8 @@ use Closure; use Exception; -abstract class Runner { +abstract class Runner +{ /** * Runner constructor. * @@ -14,7 +15,8 @@ abstract class Runner { public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, - ) {} + ) { + } /** * Executes a query within a given data context. diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 455fcb1bec..9ecf15aa03 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -7,15 +7,18 @@ use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Tokenizer; -class Interpreted extends Runner { +class Interpreted extends Runner +{ private static array $cache = []; public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, - ) {} + ) { + } - protected function getResolver(string $query): Closure { + protected function getResolver(string $query): Closure + { // load closure from process cache if(isset(self::$cache[$query])) { return self::$cache[$query]; @@ -28,7 +31,7 @@ protected function getResolver(string $query): Closure { $self = $this; - return self::$cache[$query] = function(array $binding) use ($node, $self) { + return self::$cache[$query] = function (array $binding) use ($node, $self) { $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); if($self->interceptor !== null) { $interpreter->setInterceptor($self->interceptor); @@ -37,7 +40,8 @@ protected function getResolver(string $query): Closure { }; } - public function run(string $query, array $context = []): mixed { + public function run(string $query, array $context = []): mixed + { $resolver = $this->getResolver($query); return $resolver($context); } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 0b67ef88f0..1d88e2042f 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -8,7 +8,8 @@ use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Tokenizer; -class Transpiled extends Runner { +class Transpiled extends Runner +{ private static array $cache = []; public static string $cacheFolder = '/tmp/query_cache'; @@ -20,7 +21,8 @@ class Transpiled extends Runner { public function __construct( public array $allowedFunctions = [], public Closure|null $interceptor = null, - ) {} + ) { + } /** @@ -30,7 +32,8 @@ public function __construct( * @param string $query The query string to be executed. * @return Closure The executor closure for the given query. */ - protected function getResolver(string $query): Closure { + protected function getResolver(string $query): Closure + { // load closure from process memory if(isset(self::$cache[$query])) { return self::$cache[$query]; @@ -51,10 +54,10 @@ protected function getResolver(string $query): Closure { $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; - $comment = join("\n", array_map(fn($l) => "// $l", explode("\n", $query))); + $mappings = join("\n", array_map(fn ($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; + $comment = join("\n", array_map(fn ($l) => "// $l", explode("\n", $query))); - $uses = join("\n", array_map(fn($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; + $uses = join("\n", array_map(fn ($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; $function = "getResolver($query); if(!is_callable($function)) { - throw new Exception("Query is not valid"); + throw new Exception('Query is not valid'); } - return $function($context, $this->allowedFunctions, $this->interceptor ?? fn($v) => $v); + return $function($context, $this->allowedFunctions, $this->interceptor ?? fn ($v) => $v); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 13cc9f41d6..8b8c44defe 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -15,15 +15,14 @@ use Kirby\Toolkit\Query\AST\VariableNode; use Kirby\Toolkit\Query\Visitor; - /** * Visitor that generates code representations from query structures. * * The `CodeGen` class traverses query nodes and generates corresponding PHP code. * It extends the base `Visitor` class, providing implementations specific to code generation. */ -class CodeGen extends Visitor { - +class CodeGen extends Visitor +{ /** * If we need something from a namespace, we'll add the namespace here into the array key * @var array @@ -41,10 +40,9 @@ class CodeGen extends Visitor { * they can start with a number and may contain escaped dots. * * This method returns a sanitized PHP variable name. - * - * @return string */ - private static function phpName(string $name): string { + private static function phpName(string $name): string + { return '$_' . crc32($name); } @@ -53,33 +51,41 @@ private static function phpName(string $name): string { * * @param array $validGlobalFunctions An array of valid global function closures. */ - public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor=[]){} + public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor = []) + { + } - private function intercept(string $value): string { + private function intercept(string $value): string + { return "(\$intercept($value))"; } - public function visitArgumentList(ArgumentListNode $node): string { - $arguments = array_map(fn($argument) => $argument->accept($this), $node->arguments); + public function visitArgumentList(ArgumentListNode $node): string + { + $arguments = array_map(fn ($argument) => $argument->accept($this), $node->arguments); return join(', ', $arguments); } - public function visitArrayList(ArrayListNode $node): string { - $elements = array_map(fn($element) => $element->accept($this), $node->elements); + public function visitArrayList(ArrayListNode $node): string + { + $elements = array_map(fn ($element) => $element->accept($this), $node->elements); return '[' . join(', ', $elements) . ']'; } - public function visitCoalesce(CoalesceNode $node): string { + public function visitCoalesce(CoalesceNode $node): string + { $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } - public function visitLiteral(LiteralNode $node): string { + public function visitLiteral(LiteralNode $node): string + { return '$intercept(' . var_export($node->value, true) . ')'; } - public function visitMemberAccess(MemberAccessNode $node): string { + public function visitMemberAccess(MemberAccessNode $node): string + { $object = $node->object->accept($this); $member = $node->member; @@ -97,19 +103,21 @@ public function visitMemberAccess(MemberAccessNode $node): string { return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); } - public function visitTernary(TernaryNode $node): string { + public function visitTernary(TernaryNode $node): string + { $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); if($node->trueBranchIsDefault) { return "($left ?: $falseBranch)"; - } else { - $trueBranch = $node->trueBranch->accept($this); - return "($left ? $trueBranch : $falseBranch)"; } + $trueBranch = $node->trueBranch->accept($this); + return "($left ? $trueBranch : $falseBranch)"; + } - public function visitVariable(VariableNode $node): string { + public function visitVariable(VariableNode $node): string + { $name = $node->name(); $namestr = var_export($name, true); @@ -128,7 +136,8 @@ public function visitVariable(VariableNode $node): string { /** * Generates code like `$functions['function']($arguments)` from a global function node. */ - public function visitGlobalFunction(GlobalFunctionNode $node): string { + public function visitGlobalFunction(GlobalFunctionNode $node): string + { $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); @@ -140,7 +149,8 @@ public function visitGlobalFunction(GlobalFunctionNode $node): string { return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } - public function visitClosure(ClosureNode $node): mixed { + public function visitClosure(ClosureNode $node): mixed + { $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; $args = array_map(self::phpName(...), $node->arguments); diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index c800b8ea3b..484c1fa02e 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -8,19 +8,19 @@ use Kirby\Toolkit\Query\AST\ArrayListNode; use Kirby\Toolkit\Query\AST\ClosureNode; use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\AST\LiteralNode; use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; -use Kirby\Toolkit\Query\AST\GlobalFunctionNode; use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; - /** * Visitor that interprets and directly executes a query AST. */ -class Interpreter extends Visitor { +class Interpreter extends Visitor +{ /** * @param array $validGlobalFunctions An array of valid global function closures. * @param array $context The data bindings for the query. @@ -28,21 +28,26 @@ class Interpreter extends Visitor { public function __construct( public array $validGlobalFunctions = [], public array $context = [] - ) {} + ) { + } - public function visitArgumentList(ArgumentListNode $node): array { - return array_map(fn($argument) => $argument->accept($this), $node->arguments); + public function visitArgumentList(ArgumentListNode $node): array + { + return array_map(fn ($argument) => $argument->accept($this), $node->arguments); } - public function visitArrayList(ArrayListNode $node): mixed { - return array_map(fn($element) => $element->accept($this), $node->elements); + public function visitArrayList(ArrayListNode $node): mixed + { + return array_map(fn ($element) => $element->accept($this), $node->elements); } - public function visitCoalesce(CoalesceNode $node): mixed { + public function visitCoalesce(CoalesceNode $node): mixed + { return $node->left->accept($this) ?? $node->right->accept($this); } - public function visitLiteral(LiteralNode $node): mixed { + public function visitLiteral(LiteralNode $node): mixed + { $val = $node->value; if($this->interceptor !== null) { @@ -52,7 +57,8 @@ public function visitLiteral(LiteralNode $node): mixed { return $val; } - public function visitMemberAccess(MemberAccessNode $node): mixed { + public function visitMemberAccess(MemberAccessNode $node): mixed + { $left = $node->object->accept($this); $item = null; @@ -69,15 +75,17 @@ public function visitMemberAccess(MemberAccessNode $node): mixed { return $item; } - public function visitTernary(TernaryNode $node): mixed { + public function visitTernary(TernaryNode $node): mixed + { if($node->trueBranchIsDefault) { return $node->condition->accept($this) ?: $node->trueBranch->accept($this); - } else { - return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); } + return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + } - public function visitVariable(VariableNode $node): mixed { + public function visitVariable(VariableNode $node): mixed + { // what looks like a variable might actually be a global function // but if there is a variable with the same name, the variable takes precedence @@ -96,7 +104,8 @@ public function visitVariable(VariableNode $node): mixed { return $item; } - public function visitGlobalFunction(GlobalFunctionNode $node): mixed { + public function visitGlobalFunction(GlobalFunctionNode $node): mixed + { $name = $node->name(); if(!isset($this->validGlobalFunctions[$name])) { @@ -117,10 +126,11 @@ public function visitGlobalFunction(GlobalFunctionNode $node): mixed { return $result; } - public function visitClosure(ClosureNode $node): mixed { + public function visitClosure(ClosureNode $node): mixed + { $self = $this; - return function(...$params) use ($self, $node) { + return function (...$params) use ($self, $node) { $context = $self->context; $functions = $self->validGlobalFunctions; diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index 07eadc61d5..9c102b741f 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -4,8 +4,10 @@ use Exception; -class Runtime { - static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed { +class Runtime +{ + public static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed + { if($nullSafe && $object === null) { return null; } @@ -23,7 +25,8 @@ static function access(array|object|null $object, string|int $key, bool $nullSaf } return $item; - } else if(is_object($object)) { + } + if(is_object($object)) { if(is_int($key)) { $key = (string)$key; } @@ -31,8 +34,8 @@ static function access(array|object|null $object, string|int $key, bool $nullSaf return $object->$key(...$arguments); } return $object->$key ?? null; - } else { - throw new Exception("Cannot access \"$key\" on " . gettype($object)); } + throw new Exception("Cannot access \"$key\" on " . gettype($object)); + } } diff --git a/src/Toolkit/Query/Token.php b/src/Toolkit/Query/Token.php index 0a03c914ad..99c0a405a7 100644 --- a/src/Toolkit/Query/Token.php +++ b/src/Toolkit/Query/Token.php @@ -2,10 +2,12 @@ namespace Kirby\Toolkit\Query; -class Token { +class Token +{ public function __construct( public TokenType $type, public string $lexeme, public mixed $literal = null, - ) {} + ) { + } } diff --git a/src/Toolkit/Query/TokenType.php b/src/Toolkit/Query/TokenType.php index 2cdc14c93a..eade2dcc28 100644 --- a/src/Toolkit/Query/TokenType.php +++ b/src/Toolkit/Query/TokenType.php @@ -2,7 +2,8 @@ namespace Kirby\Toolkit\Query; -enum TokenType { +enum TokenType +{ case T_STRING; case T_INTEGER; case T_WHITESPACE; diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index a79f49051b..993be68de0 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -5,7 +5,8 @@ use Exception; use Generator; -class Tokenizer { +class Tokenizer +{ private int $length = 0; /** @@ -36,7 +37,8 @@ public function __construct( * Tokenizes the source string and returns a generator of tokens. * @return Generator */ - public function tokenize(): Generator { + public function tokenize(): Generator + { $current = 0; while ($current < $this->length) { @@ -59,7 +61,8 @@ public function tokenize(): Generator { * @return Token The scanned token * @throws Exception If an unexpected character is encountered */ - protected static function scanToken(string $source, int $current): Token { + protected static function scanToken(string $source, int $current): Token + { $l = ''; $c = $source[$current]; @@ -89,7 +92,7 @@ protected static function scanToken(string $source, int $current): Token { self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, intval($l)), + self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, (int)$l), self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), // unknown token @@ -108,7 +111,8 @@ protected static function scanToken(string $source, int $current): Token { * @param bool $caseIgnore Whether to ignore case while matching * @return bool Whether the regex pattern was matched */ - protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool { + protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool + { $regex = '/\G' . $regex . '/u'; if($caseIgnore) { $regex .= 'i'; diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index 2f24a75a52..41db64ddc3 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -6,10 +6,12 @@ use Exception; use ReflectionClass; -abstract class Visitor { +abstract class Visitor +{ protected Closure|null $interceptor = null; - function visitNode(AST\Node $node): mixed { + public function visitNode(AST\Node $node): mixed + { $shortName = (new ReflectionClass($node))->getShortName(); // remove the "Node" suffix @@ -20,23 +22,24 @@ function visitNode(AST\Node $node): mixed { return $this->$method($node); } - throw new Exception("No visitor method for " . $node::class); + throw new Exception('No visitor method for ' . $node::class); } - abstract function visitArgumentList(AST\ArgumentListNode $node): mixed; - abstract function visitArrayList(AST\ArrayListNode $node): mixed; - abstract function visitCoalesce(AST\CoalesceNode $node): mixed; - abstract function visitLiteral(AST\LiteralNode $node): mixed; - abstract function visitMemberAccess(AST\MemberAccessNode $node): mixed; - abstract function visitTernary(AST\TernaryNode $node): mixed; - abstract function visitVariable(AST\VariableNode $node): mixed; - abstract function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; - abstract function visitClosure(AST\ClosureNode $node): mixed; + abstract public function visitArgumentList(AST\ArgumentListNode $node): mixed; + abstract public function visitArrayList(AST\ArrayListNode $node): mixed; + abstract public function visitCoalesce(AST\CoalesceNode $node): mixed; + abstract public function visitLiteral(AST\LiteralNode $node): mixed; + abstract public function visitMemberAccess(AST\MemberAccessNode $node): mixed; + abstract public function visitTernary(AST\TernaryNode $node): mixed; + abstract public function visitVariable(AST\VariableNode $node): mixed; + abstract public function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; + abstract public function visitClosure(AST\ClosureNode $node): mixed; /** * Sets and activates an interceptor closure that is called for each resolved value. */ - public function setInterceptor(Closure $interceptor): void { + public function setInterceptor(Closure $interceptor): void + { $this->interceptor = $interceptor; } } diff --git a/tests/Query/QueryTest.php b/tests/Query/QueryTest.php index 5ba87109aa..486d4ddb2f 100644 --- a/tests/Query/QueryTest.php +++ b/tests/Query/QueryTest.php @@ -127,12 +127,7 @@ public function testResolveWithClosureWithArgument() */ public function testResolveWithInterceptor() { - $query = new class extends Query { - public function __construct() - { - parent::__construct('foo.getObj.name'); - } - + $query = new class ('foo.getObj.name') extends Query { public function intercept($result): mixed { if(is_object($result) === true) { From f62695d59e0128ede1f430ebe6e18690fc56f7f9 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 13:53:16 +0100 Subject: [PATCH 25/31] Remove unused imports in Query.php --- src/Query/Query.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 38383f7a77..6e77d1746a 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -12,10 +12,8 @@ use Kirby\Cms\User; use Kirby\Image\QrCode; use Kirby\Toolkit\I18n; -use Kirby\Toolkit\Query\Runner; use Kirby\Toolkit\Query\Runners\Interpreted; use Kirby\Toolkit\Query\Runners\Transpiled; -use Kirby\Toolkit\Query\Visitor; /** * The Query class can be used to query arrays and objects, From 2bf1ffc3528519303ee26e946fef3959729744b0 Mon Sep 17 00:00:00 2001 From: Nico Hoffmann Date: Wed, 13 Nov 2024 18:03:50 +0100 Subject: [PATCH 26/31] First set of CS fixes --- src/Query/Query.php | 2 +- src/Toolkit/Query/BaseParser.php | 19 ++-- src/Toolkit/Query/Parser.php | 63 +++++++------- src/Toolkit/Query/Runners/Interpreted.php | 15 ++-- src/Toolkit/Query/Runners/Transpiled.php | 44 +++++++--- .../Query/Runners/Visitors/CodeGen.php | 68 ++++++++++----- .../Query/Runners/Visitors/Interpreter.php | 52 +++++++---- src/Toolkit/Query/Runtime.php | 36 +++++--- src/Toolkit/Query/Tokenizer.php | 86 ++++++++++++------- src/Toolkit/Query/Visitor.php | 34 +++++--- 10 files changed, 262 insertions(+), 157 deletions(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 6e77d1746a..043c0a9875 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -109,7 +109,7 @@ public function resolve(array|object $data = []): mixed $mode = App::instance()->option('query.runner', 'transpiled'); - if($mode === 'legacy') { + if ($mode === 'legacy') { return $this->resolve_legacy($data); } diff --git a/src/Toolkit/Query/BaseParser.php b/src/Toolkit/Query/BaseParser.php index b597cf391a..7e3fc459c4 100644 --- a/src/Toolkit/Query/BaseParser.php +++ b/src/Toolkit/Query/BaseParser.php @@ -19,13 +19,12 @@ abstract class BaseParser public function __construct( Tokenizer|Iterator $source, ) { - if($source instanceof Tokenizer) { - $this->tokens = $source->tokenize(); - } else { - $this->tokens = $source; + if ($source instanceof Tokenizer) { + $source = $source->tokenize(); } - $first = $this->tokens->current(); + $this->tokens = $source; + $first = $this->tokens->current(); if ($first === null) { throw new Exception('No tokens found.'); @@ -36,7 +35,7 @@ public function __construct( protected function consume(TokenType $type, string $message): Token { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } @@ -45,7 +44,7 @@ protected function consume(TokenType $type, string $message): Token protected function check(TokenType $type): bool { - if ($this->isAtEnd()) { + if ($this->isAtEnd() === true) { return false; } @@ -54,7 +53,7 @@ protected function check(TokenType $type): bool protected function advance(): Token|null { - if (!$this->isAtEnd()) { + if ($this->isAtEnd() === false) { $this->previous = $this->current; $this->tokens->next(); $this->current = $this->tokens->current(); @@ -71,7 +70,7 @@ protected function isAtEnd(): bool protected function match(TokenType $type): Token|false { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } @@ -81,7 +80,7 @@ protected function match(TokenType $type): Token|false protected function matchAny(array $types): Token|false { foreach ($types as $type) { - if ($this->check($type)) { + if ($this->check($type) === true) { return $this->advance(); } } diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index 8be677f1bf..f5b4801120 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -3,7 +3,6 @@ namespace Kirby\Toolkit\Query; use Exception; -use Iterator; use Kirby\Toolkit\Query\AST\ArgumentListNode; use Kirby\Toolkit\Query\AST\ArrayListNode; use Kirby\Toolkit\Query\AST\ClosureNode; @@ -17,18 +16,12 @@ class Parser extends BaseParser { - public function __construct( - Tokenizer|Iterator $source, - ) { - parent::__construct($source); - } - public function parse(): Node { $expression = $this->expression(); // ensure that we consumed all tokens - if(!$this->isAtEnd()) { + if ($this->isAtEnd() === false) { $this->consume(TokenType::T_EOF, 'Expect end of expression.'); } @@ -46,7 +39,7 @@ private function coalesce(): Node while ($this->match(TokenType::T_COALESCE)) { $right = $this->ternary(); - $left = new CoalesceNode($left, $right); + $left = new CoalesceNode($left, $right); } return $left; @@ -57,18 +50,23 @@ private function ternary(): Node $left = $this->memberAccess(); if ($tok = $this->matchAny([TokenType::T_QUESTION_MARK, TokenType::T_TERNARY_DEFAULT])) { - if($tok->type === TokenType::T_TERNARY_DEFAULT) { + if ($tok->type === TokenType::T_TERNARY_DEFAULT) { $trueIsDefault = true; - $trueBranch = null; - $falseBranch = $this->expression(); + $trueBranch = null; } else { $trueIsDefault = false; - $trueBranch = $this->expression(); + $trueBranch = $this->expression(); $this->consume(TokenType::T_COLON, 'Expect ":" after true branch.'); - $falseBranch = $this->expression(); } - return new TernaryNode($left, $trueBranch, $falseBranch, $trueIsDefault); + $falseBranch = $this->expression(); + + return new TernaryNode( + $left, + $trueBranch, + $falseBranch, + $trueIsDefault + ); } return $left; @@ -81,20 +79,24 @@ private function memberAccess(): Node while ($tok = $this->matchAny([TokenType::T_DOT, TokenType::T_NULLSAFE])) { $nullSafe = $tok->type === TokenType::T_NULLSAFE; - if($right = $this->match(TokenType::T_IDENTIFIER)) { + if ($right = $this->match(TokenType::T_IDENTIFIER)) { $right = $right->lexeme; - } elseif($right = $this->match(TokenType::T_INTEGER)) { + } elseif ($right = $this->match(TokenType::T_INTEGER)) { $right = $right->literal; } else { throw new Exception('Expect property name after ".".'); } - if($this->match(TokenType::T_OPEN_PAREN)) { + if ($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); - $left = new MemberAccessNode($left, $right, $arguments, $nullSafe); - } else { - $left = new MemberAccessNode($left, $right, null, $nullSafe); } + + $left = new MemberAccessNode( + $left, + $right, + $arguments ?? null, + $nullSafe + ); } return $left; @@ -104,10 +106,10 @@ private function listUntil(TokenType $until): array { $elements = []; - while (!$this->isAtEnd() && !$this->check($until)) { + while ($this->isAtEnd() === false && $this->check($until) === false) { $elements[] = $this->expression(); - if (!$this->match(TokenType::T_COMMA)) { + if ($this->match(TokenType::T_COMMA) == false) { break; } } @@ -129,10 +131,11 @@ private function atomic(): Node { // float numbers if ($integer = $this->match(TokenType::T_INTEGER)) { - if($this->match(TokenType::T_DOT)) { + if ($this->match(TokenType::T_DOT)) { $fractional = $this->match(TokenType::T_INTEGER); return new LiteralNode((float)($integer->literal . '.' . $fractional->literal)); } + return new LiteralNode($integer->literal); } @@ -155,7 +158,7 @@ private function atomic(): Node // global functions and variables if ($token = $this->match(TokenType::T_IDENTIFIER)) { - if($this->match(TokenType::T_OPEN_PAREN)) { + if ($this->match(TokenType::T_OPEN_PAREN)) { $arguments = $this->argumentList(); return new GlobalFunctionNode($token->lexeme, $arguments); } @@ -167,7 +170,7 @@ private function atomic(): Node if ($token = $this->match(TokenType::T_OPEN_PAREN)) { $list = $this->listUntil(TokenType::T_CLOSE_PAREN); - if($this->match(TokenType::T_ARROW)) { + if ($this->match(TokenType::T_ARROW)) { $expression = $this->expression(); /** @@ -175,7 +178,7 @@ private function atomic(): Node * @var VariableNode[] $list */ foreach($list as $element) { - if(!$element instanceof VariableNode) { + if ($element instanceof VariableNode === false) { throw new Exception('Expecting only variables in closure argument list.'); } } @@ -183,13 +186,13 @@ private function atomic(): Node $arguments = array_map(fn ($element) => $element->name, $list); return new ClosureNode($arguments, $expression); } - if(count($list) > 1) { + + if (count($list) > 1) { throw new Exception('Expecting \"=>\" after closure argument list.'); } + // this is just a grouping return $list[0]; - - } throw new Exception('Expect expression.'); diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 9ecf15aa03..139ed9ba77 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -5,6 +5,7 @@ use Closure; use Kirby\Toolkit\Query\Parser; use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Visitors\Interpreter; use Kirby\Toolkit\Query\Tokenizer; class Interpreted extends Runner @@ -20,22 +21,24 @@ public function __construct( protected function getResolver(string $query): Closure { // load closure from process cache - if(isset(self::$cache[$query])) { + if (isset(self::$cache[$query])) { return self::$cache[$query]; } // on cache miss, parse query and generate closure - $t = new Tokenizer($query); - $parser = new Parser($t); - $node = $parser->parse(); + $tokenizer = new Tokenizer($query); + $parser = new Parser($tokenizer); + $node = $parser->parse(); $self = $this; return self::$cache[$query] = function (array $binding) use ($node, $self) { - $interpreter = new Visitors\Interpreter($self->allowedFunctions, $binding); - if($self->interceptor !== null) { + $interpreter = new Interpreter($self->allowedFunctions, $binding); + + if ($self->interceptor !== null) { $interpreter->setInterceptor($self->interceptor); } + return $node->accept($interpreter); }; } diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 1d88e2042f..09a3d5fbfd 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -6,6 +6,7 @@ use Exception; use Kirby\Toolkit\Query\Parser; use Kirby\Toolkit\Query\Runner; +use Kirby\Toolkit\Query\Runners\Visitors\CodeGen; use Kirby\Toolkit\Query\Tokenizer; class Transpiled extends Runner @@ -24,7 +25,6 @@ public function __construct( ) { } - /** * Retrieves the executor closure for a given query. * If the closure is not already cached, it will be generated and stored in `Runner::$cacheFolder`. @@ -35,33 +35,43 @@ public function __construct( protected function getResolver(string $query): Closure { // load closure from process memory - if(isset(self::$cache[$query])) { + if (isset(self::$cache[$query])) { return self::$cache[$query]; } // load closure from file-cache / opcache - $hash = crc32($query); + $hash = crc32($query); $filename = self::$cacheFolder . '/' . $hash . '.php'; - if(file_exists($filename)) { + + if (file_exists($filename)) { return self::$cache[$query] = include $filename; } // on cache miss, parse query and generate closure - $t = new Tokenizer($query); - $parser = new Parser($t); - $node = $parser->parse(); - $codeGen = new Visitors\CodeGen($this->allowedFunctions); + $tokenizer = new Tokenizer($query); + $parser = new Parser($tokenizer); + $node = $parser->parse(); + $codeGen = new CodeGen($this->allowedFunctions); $functionBody = $node->accept($codeGen); - $mappings = join("\n", array_map(fn ($k, $v) => "$k = $v;", array_keys($codeGen->mappings), $codeGen->mappings)) . "\n"; - $comment = join("\n", array_map(fn ($l) => "// $l", explode("\n", $query))); + $mappings = array_map( + fn ($k, $v) => "$k = $v;", + array_keys($codeGen->mappings), + $codeGen->mappings + ); + $mappings = join("\n", $mappings) . "\n"; + + $comment = array_map(fn ($l) => "// $l", explode("\n", $query)); + $comment = join("\n", $comment); + + $uses = array_map(fn ($k) => "use $k;", array_keys($codeGen->uses)); + $uses = join("\n", $uses) . "\n"; - $uses = join("\n", array_map(fn ($k) => "use $k;", array_keys($codeGen->uses))) . "\n"; $function = "getResolver($query); - if(!is_callable($function)) { + + if (is_callable($function) === false) { throw new Exception('Query is not valid'); } - return $function($context, $this->allowedFunctions, $this->interceptor ?? fn ($v) => $v); + + return $function( + $context, + $this->allowedFunctions, + $this->interceptor ?? fn ($v) => $v + ); } } diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index 8b8c44defe..d28a766779 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -13,6 +13,7 @@ use Kirby\Toolkit\Query\AST\MemberAccessNode; use Kirby\Toolkit\Query\AST\TernaryNode; use Kirby\Toolkit\Query\AST\VariableNode; +use Kirby\Toolkit\Query\Runtime; use Kirby\Toolkit\Query\Visitor; /** @@ -51,8 +52,10 @@ private static function phpName(string $name): string * * @param array $validGlobalFunctions An array of valid global function closures. */ - public function __construct(public array $validGlobalFunctions = [], public array $directAccessFor = []) - { + public function __construct( + public array $validGlobalFunctions = [], + public array $directAccessFor = [] + ) { } private function intercept(string $value): string @@ -62,19 +65,27 @@ private function intercept(string $value): string public function visitArgumentList(ArgumentListNode $node): string { - $arguments = array_map(fn ($argument) => $argument->accept($this), $node->arguments); + $arguments = array_map( + fn ($argument) => $argument->accept($this), + $node->arguments + ); + return join(', ', $arguments); } public function visitArrayList(ArrayListNode $node): string { - $elements = array_map(fn ($element) => $element->accept($this), $node->elements); + $elements = array_map( + fn ($element) => $element->accept($this), + $node->elements + ); + return '[' . join(', ', $elements) . ']'; } public function visitCoalesce(CoalesceNode $node): string { - $left = $node->left->accept($this); + $left = $node->left->accept($this); $right = $node->right->accept($this); return "($left ?? $right)"; } @@ -89,28 +100,33 @@ public function visitMemberAccess(MemberAccessNode $node): string $object = $node->object->accept($this); $member = $node->member; - $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $this->uses[Runtime::class] = true; $memberStr = var_export($member, true); - $nullSafe = $node->nullSafe ? 'true' : 'false'; + $nullSafe = $node->nullSafe ? 'true' : 'false'; - if($node->arguments) { + if ($node->arguments) { $arguments = $node->arguments->accept($this); - $member = var_export($member, true); + $member = var_export($member, true); - return $this->intercept("Runtime::access($object, $memberStr, $nullSafe, $arguments)"); + return $this->intercept( + "Runtime::access($object, $memberStr, $nullSafe, $arguments)" + ); } - return $this->intercept("Runtime::access($object, $memberStr, $nullSafe)"); + return $this->intercept( + "Runtime::access($object, $memberStr, $nullSafe)" + ); } public function visitTernary(TernaryNode $node): string { - $left = $node->condition->accept($this); + $left = $node->condition->accept($this); $falseBranch = $node->falseBranch->accept($this); - if($node->trueBranchIsDefault) { + if ($node->trueBranchIsDefault === true) { return "($left ?: $falseBranch)"; } + $trueBranch = $node->trueBranch->accept($this); return "($left ? $trueBranch : $falseBranch)"; @@ -118,15 +134,15 @@ public function visitTernary(TernaryNode $node): string public function visitVariable(VariableNode $node): string { - $name = $node->name(); + $name = $node->name(); $namestr = var_export($name, true); + $key = static::phpName($name); - $key = self::phpName($name); - if(isset($this->directAccessFor[$name])) { + if (isset($this->directAccessFor[$name])) { return $this->intercept($key); } - if(!isset($this->mappings[$key])) { + if (isset($this->mappings[$key]) === false) { $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) && \$context[$namestr] instanceof Closure => \$context[$namestr](), isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); } @@ -139,25 +155,31 @@ public function visitVariable(VariableNode $node): string public function visitGlobalFunction(GlobalFunctionNode $node): string { $name = $node->name(); - if(!isset($this->validGlobalFunctions[$name])) { + + if (isset($this->validGlobalFunctions[$name])) { throw new Exception("Invalid global function $name"); } $arguments = $node->arguments->accept($this); - $name = var_export($name, true); + $name = var_export($name, true); return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); } public function visitClosure(ClosureNode $node): mixed { - $this->uses['Kirby\\Toolkit\\Query\\Runtime'] = true; + $this->uses[Runtime::class] = true; - $args = array_map(self::phpName(...), $node->arguments); + $args = array_map(static::phpName(...), $node->arguments); $args = join(', ', $args); - $newDirectAccessFor = array_merge($this->directAccessFor, array_fill_keys($node->arguments, true)); + $newDirectAccessFor = [ + ...$this->directAccessFor, + ...array_fill_keys($node->arguments, true) + ]; - return "fn($args) => " . $node->body->accept(new self($this->validGlobalFunctions, $newDirectAccessFor)); + return "fn($args) => " . $node->body->accept( + new static($this->validGlobalFunctions, $newDirectAccessFor) + ); } } diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 484c1fa02e..01c1e74507 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -33,12 +33,18 @@ public function __construct( public function visitArgumentList(ArgumentListNode $node): array { - return array_map(fn ($argument) => $argument->accept($this), $node->arguments); + return array_map( + fn ($argument) => $argument->accept($this), + $node->arguments + ); } public function visitArrayList(ArrayListNode $node): mixed { - return array_map(fn ($element) => $element->accept($this), $node->elements); + return array_map( + fn ($element) => $element->accept($this), + $node->elements + ); } public function visitCoalesce(CoalesceNode $node): mixed @@ -50,7 +56,7 @@ public function visitLiteral(LiteralNode $node): mixed { $val = $node->value; - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $val = ($this->interceptor)($val); } @@ -60,15 +66,20 @@ public function visitLiteral(LiteralNode $node): mixed public function visitMemberAccess(MemberAccessNode $node): mixed { $left = $node->object->accept($this); - $item = null; - if($node->arguments !== null) { - $item = Runtime::access($left, $node->member, $node->nullSafe, ...$node->arguments->accept($this)); + + if ($node->arguments !== null) { + $item = Runtime::access( + $left, + $node->member, + $node->nullSafe, + ...$node->arguments->accept($this) + ); } else { $item = Runtime::access($left, $node->member, $node->nullSafe); } - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $item = ($this->interceptor)($item); } @@ -77,11 +88,16 @@ public function visitMemberAccess(MemberAccessNode $node): mixed public function visitTernary(TernaryNode $node): mixed { - if($node->trueBranchIsDefault) { - return $node->condition->accept($this) ?: $node->trueBranch->accept($this); + if ($node->trueBranchIsDefault === true) { + return + $node->condition->accept($this) ?: + $node->trueBranch->accept($this); } - return $node->condition->accept($this) ? $node->trueBranch->accept($this) : $node->falseBranch->accept($this); + return + $node->condition->accept($this) ? + $node->trueBranch->accept($this) : + $node->falseBranch->accept($this); } public function visitVariable(VariableNode $node): mixed @@ -97,7 +113,7 @@ public function visitVariable(VariableNode $node): mixed default => null, }; - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $item = ($this->interceptor)($item); } @@ -108,18 +124,19 @@ public function visitGlobalFunction(GlobalFunctionNode $node): mixed { $name = $node->name(); - if(!isset($this->validGlobalFunctions[$name])) { + if (isset($this->validGlobalFunctions[$name]) === false) { throw new Exception("Invalid global function $name"); } $function = $this->validGlobalFunctions[$name]; - if($this->interceptor !== null) { + + if ($this->interceptor !== null) { $function = ($this->interceptor)($function); } $result = $function(...$node->arguments->accept($this)); - if($this->interceptor !== null) { + if ($this->interceptor !== null) { $result = ($this->interceptor)($result); } @@ -131,7 +148,7 @@ public function visitClosure(ClosureNode $node): mixed $self = $this; return function (...$params) use ($self, $node) { - $context = $self->context; + $context = $self->context; $functions = $self->validGlobalFunctions; // [key1, key2] + [value1, value2] => [key1 => value1, key2 => value2] @@ -140,8 +157,9 @@ public function visitClosure(ClosureNode $node): mixed $params ); - $visitor = new self($functions, [...$context, ...$arguments]); - if($self->interceptor !== null) { + $visitor = new static($functions, [...$context, ...$arguments]); + + if ($self->interceptor !== null) { $visitor->setInterceptor($self->interceptor); } diff --git a/src/Toolkit/Query/Runtime.php b/src/Toolkit/Query/Runtime.php index 9c102b741f..66be6a59cf 100644 --- a/src/Toolkit/Query/Runtime.php +++ b/src/Toolkit/Query/Runtime.php @@ -2,40 +2,50 @@ namespace Kirby\Toolkit\Query; +use Closure; use Exception; class Runtime { - public static function access(array|object|null $object, string|int $key, bool $nullSafe = false, ...$arguments): mixed - { - if($nullSafe && $object === null) { + public static function access( + array|object|null $object, + string|int $key, + bool $nullSafe = false, + ...$arguments + ): mixed { + if ($nullSafe === true && $object === null) { return null; } - if(is_array($object)) { - $item = ($object[$key] ?? $object[(string)$key] ?? null); - - if($item) { - if($arguments) { + if (is_array($object)) { + if ($item = $object[$key] ?? $object[(string)$key] ?? null) { + if ($arguments) { return $item(...$arguments); } - if($item instanceof \Closure) { + + if ($item instanceof Closure) { return $item(); } } return $item; } - if(is_object($object)) { - if(is_int($key)) { + + if (is_object($object)) { + if (is_int($key)) { $key = (string)$key; } - if(method_exists($object, $key) || method_exists($object, '__call')) { + + if ( + method_exists($object, $key) || + method_exists($object, '__call') + ) { return $object->$key(...$arguments); } + return $object->$key ?? null; } - throw new Exception("Cannot access \"$key\" on " . gettype($object)); + throw new Exception("Cannot access \"$key\" on " . gettype($object)); } } diff --git a/src/Toolkit/Query/Tokenizer.php b/src/Toolkit/Query/Tokenizer.php index 993be68de0..26f76772e5 100644 --- a/src/Toolkit/Query/Tokenizer.php +++ b/src/Toolkit/Query/Tokenizer.php @@ -42,12 +42,14 @@ public function tokenize(): Generator $current = 0; while ($current < $this->length) { - $t = self::scanToken($this->source, $current); + $token = static::scanToken($this->source, $current); + // don't yield whitespace tokens (ignore them) - if($t->type !== TokenType::T_WHITESPACE) { - yield $t; + if ($token->type !== TokenType::T_WHITESPACE) { + yield $token; } - $current += mb_strlen($t->lexeme); + + $current += mb_strlen($token->lexeme); } yield new Token(TokenType::T_EOF, '', null); @@ -63,37 +65,50 @@ public function tokenize(): Generator */ protected static function scanToken(string $source, int $current): Token { - $l = ''; - $c = $source[$current]; + $lex = ''; + $char = $source[$current]; return match(true) { // single character tokens - $c === '.' => new Token(TokenType::T_DOT, '.'), - $c === '(' => new Token(TokenType::T_OPEN_PAREN, '('), - $c === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), - $c === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), - $c === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), - $c === ',' => new Token(TokenType::T_COMMA, ','), - $c === ':' => new Token(TokenType::T_COLON, ':'), + $char === '.' => new Token(TokenType::T_DOT, '.'), + $char === '(' => new Token(TokenType::T_OPEN_PAREN, '('), + $char === ')' => new Token(TokenType::T_CLOSE_PAREN, ')'), + $char === '[' => new Token(TokenType::T_OPEN_BRACKET, '['), + $char === ']' => new Token(TokenType::T_CLOSE_BRACKET, ']'), + $char === ',' => new Token(TokenType::T_COMMA, ','), + $char === ':' => new Token(TokenType::T_COLON, ':'), // two character tokens - self::match($source, $current, '\?\?', $l) => new Token(TokenType::T_COALESCE, $l), - self::match($source, $current, '\?\s*\.', $l) => new Token(TokenType::T_NULLSAFE, $l), - self::match($source, $current, '\?\s*:', $l) => new Token(TokenType::T_TERNARY_DEFAULT, $l), - self::match($source, $current, '=>', $l) => new Token(TokenType::T_ARROW, $l), - - // make sure this check comes after the two above that check for '?' in the beginning - $c === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), + static::match($source, $current, '\?\?', $lex) + => new Token(TokenType::T_COALESCE, $lex), + static::match($source, $current, '\?\s*\.', $lex) + => new Token(TokenType::T_NULLSAFE, $lex), + static::match($source, $current, '\?\s*:', $lex) + => new Token(TokenType::T_TERNARY_DEFAULT, $lex), + static::match($source, $current, '=>', $lex) + => new Token(TokenType::T_ARROW, $lex), + + // make sure this check comes after the two above + // that check for '?' in the beginning + $char === '?' => new Token(TokenType::T_QUESTION_MARK, '?'), // multi character tokens - self::match($source, $current, '\s+', $l) => new Token(TokenType::T_WHITESPACE, $l), - self::match($source, $current, 'true', $l, true) => new Token(TokenType::T_TRUE, $l, true), - self::match($source, $current, 'false', $l, true) => new Token(TokenType::T_FALSE, $l, false), - self::match($source, $current, 'null', $l, true) => new Token(TokenType::T_NULL, $l, null), - self::match($source, $current, self::DOUBLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, self::SINGLEQUOTE_STRING_REGEX, $l) => new Token(TokenType::T_STRING, $l, stripcslashes(substr($l, 1, -1))), - self::match($source, $current, '\d+\b', $l) => new Token(TokenType::T_INTEGER, $l, (int)$l), - self::match($source, $current, self::IDENTIFIER_REGEX, $l) => new Token(TokenType::T_IDENTIFIER, $l), + static::match($source, $current, '\s+', $lex) + => new Token(TokenType::T_WHITESPACE, $lex), + static::match($source, $current, 'true', $lex, true) + => new Token(TokenType::T_TRUE, $lex, true), + static::match($source, $current, 'false', $lex, true) + => new Token(TokenType::T_FALSE, $lex, false), + static::match($source, $current, 'null', $lex, true) + => new Token(TokenType::T_NULL, $lex, null), + static::match($source, $current, static::DOUBLEQUOTE_STRING_REGEX, $lex) + => new Token(TokenType::T_STRING, $lex, stripcslashes(substr($lex, 1, -1))), + static::match($source, $current, static::SINGLEQUOTE_STRING_REGEX, $lex) + => new Token(TokenType::T_STRING, $lex, stripcslashes(substr($lex, 1, -1))), + static::match($source, $current, '\d+\b', $lex) + => new Token(TokenType::T_INTEGER, $lex, (int)$lex), + static::match($source, $current, static::IDENTIFIER_REGEX, $lex) + => new Token(TokenType::T_IDENTIFIER, $lex), // unknown token default => throw new Exception("Unexpected character: {$source[$current]}"), @@ -111,17 +126,26 @@ protected static function scanToken(string $source, int $current): Token * @param bool $caseIgnore Whether to ignore case while matching * @return bool Whether the regex pattern was matched */ - protected static function match(string $source, int $current, string $regex, string &$lexeme, bool $caseIgnore = false): bool - { + protected static function match( + string $source, + int $current, + string $regex, + string &$lexeme, + bool $caseIgnore = false + ): bool { $regex = '/\G' . $regex . '/u'; - if($caseIgnore) { + + if ($caseIgnore) { $regex .= 'i'; } + $matches = []; preg_match($regex, $source, $matches, 0, $current); + if (empty($matches[0])) { return false; } + $lexeme = $matches[0]; return true; } diff --git a/src/Toolkit/Query/Visitor.php b/src/Toolkit/Query/Visitor.php index 41db64ddc3..fc58257799 100644 --- a/src/Toolkit/Query/Visitor.php +++ b/src/Toolkit/Query/Visitor.php @@ -4,36 +4,46 @@ use Closure; use Exception; +use Kirby\Toolkit\Query\AST\ArgumentListNode; +use Kirby\Toolkit\Query\AST\ArrayListNode; +use Kirby\Toolkit\Query\AST\ClosureNode; +use Kirby\Toolkit\Query\AST\CoalesceNode; +use Kirby\Toolkit\Query\AST\GlobalFunctionNode; +use Kirby\Toolkit\Query\AST\LiteralNode; +use Kirby\Toolkit\Query\AST\MemberAccessNode; +use Kirby\Toolkit\Query\AST\Node; +use Kirby\Toolkit\Query\AST\TernaryNode; +use Kirby\Toolkit\Query\AST\VariableNode; use ReflectionClass; abstract class Visitor { protected Closure|null $interceptor = null; - public function visitNode(AST\Node $node): mixed + public function visitNode(Node $node): mixed { $shortName = (new ReflectionClass($node))->getShortName(); // remove the "Node" suffix $shortName = substr($shortName, 0, -4); + $method = 'visit' . $shortName; - $method = 'visit' . $shortName; - if(method_exists($this, $method)) { + if (method_exists($this, $method)) { return $this->$method($node); } throw new Exception('No visitor method for ' . $node::class); } - abstract public function visitArgumentList(AST\ArgumentListNode $node): mixed; - abstract public function visitArrayList(AST\ArrayListNode $node): mixed; - abstract public function visitCoalesce(AST\CoalesceNode $node): mixed; - abstract public function visitLiteral(AST\LiteralNode $node): mixed; - abstract public function visitMemberAccess(AST\MemberAccessNode $node): mixed; - abstract public function visitTernary(AST\TernaryNode $node): mixed; - abstract public function visitVariable(AST\VariableNode $node): mixed; - abstract public function visitGlobalFunction(AST\GlobalFunctionNode $node): mixed; - abstract public function visitClosure(AST\ClosureNode $node): mixed; + abstract public function visitArgumentList(ArgumentListNode $node): mixed; + abstract public function visitArrayList(ArrayListNode $node): mixed; + abstract public function visitCoalesce(CoalesceNode $node): mixed; + abstract public function visitLiteral(LiteralNode $node): mixed; + abstract public function visitMemberAccess(MemberAccessNode $node): mixed; + abstract public function visitTernary(TernaryNode $node): mixed; + abstract public function visitVariable(VariableNode $node): mixed; + abstract public function visitGlobalFunction(GlobalFunctionNode $node): mixed; + abstract public function visitClosure(ClosureNode $node): mixed; /** * Sets and activates an interceptor closure that is called for each resolved value. From 738674e642f648643f1327313e8b490cba9c522a Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 13 Nov 2024 18:23:06 +0100 Subject: [PATCH 27/31] set "interpreted" as default query runner --- src/Query/Query.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Query/Query.php b/src/Query/Query.php index 043c0a9875..d271c40d08 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -107,7 +107,7 @@ public function resolve(array|object $data = []): mixed return $data; } - $mode = App::instance()->option('query.runner', 'transpiled'); + $mode = App::instance()->option('query.runner', 'interpreted'); if ($mode === 'legacy') { return $this->resolve_legacy($data); From a22d4ca544fa0305e5127af20ec8f382199ab735 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 3 Dec 2024 15:51:19 +0100 Subject: [PATCH 28/31] fix query: sequential member access was not resetting method arguments --- src/Toolkit/Query/Parser.php | 9 ++++---- tests/Toolkit/Query/ParserTest.php | 37 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 tests/Toolkit/Query/ParserTest.php diff --git a/src/Toolkit/Query/Parser.php b/src/Toolkit/Query/Parser.php index f5b4801120..582314bca8 100644 --- a/src/Toolkit/Query/Parser.php +++ b/src/Toolkit/Query/Parser.php @@ -87,14 +87,15 @@ private function memberAccess(): Node throw new Exception('Expect property name after ".".'); } - if ($this->match(TokenType::T_OPEN_PAREN)) { - $arguments = $this->argumentList(); - } + $arguments = match ($this->match(TokenType::T_OPEN_PAREN)) { + false => null, + default => $this->argumentList(), + }; $left = new MemberAccessNode( $left, $right, - $arguments ?? null, + $arguments, $nullSafe ); } diff --git a/tests/Toolkit/Query/ParserTest.php b/tests/Toolkit/Query/ParserTest.php new file mode 100644 index 0000000000..5e2a18e156 --- /dev/null +++ b/tests/Toolkit/Query/ParserTest.php @@ -0,0 +1,37 @@ +parse(); + $this->assertEquals($ast, + new MemberAccessNode( + new VariableNode('user'), 'name' + ) + ); + } + + function testSquentialMemberAccess() { + $tokenizer = new Tokenizer('user.name("arg").age'); + $ast = (new Parser($tokenizer))->parse(); + $this->assertEquals($ast, + new MemberAccessNode( + new MemberAccessNode( + new VariableNode('user'), + 'name', + new ArgumentListNode([new LiteralNode("arg")]) + ), + 'age' + ) + ); + } +} From ae94934e8271adb28f949c3bbd9ac811554fc3ea Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Tue, 3 Dec 2024 16:02:05 +0100 Subject: [PATCH 29/31] code style --- tests/Toolkit/Query/ParserTest.php | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/Toolkit/Query/ParserTest.php b/tests/Toolkit/Query/ParserTest.php index 5e2a18e156..f5172625b4 100644 --- a/tests/Toolkit/Query/ParserTest.php +++ b/tests/Toolkit/Query/ParserTest.php @@ -1,34 +1,39 @@ parse(); - $this->assertEquals($ast, + $this->assertEquals( + $ast, new MemberAccessNode( - new VariableNode('user'), 'name' + new VariableNode('user'), + 'name' ) ); } - function testSquentialMemberAccess() { + public function testSquentialMemberAccess() + { $tokenizer = new Tokenizer('user.name("arg").age'); $ast = (new Parser($tokenizer))->parse(); - $this->assertEquals($ast, + $this->assertEquals( + $ast, new MemberAccessNode( new MemberAccessNode( new VariableNode('user'), 'name', - new ArgumentListNode([new LiteralNode("arg")]) + new ArgumentListNode([new LiteralNode('arg')]) ), 'age' ) From d9d6e8db538b8763b4895de2007b0822566c17db Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 4 Dec 2024 16:21:24 +0100 Subject: [PATCH 30/31] only intercept accessed objects instead of intercepting everything, intercept only objects or array on which the query actually want's to access methods or fields --- .../Query/Runners/Visitors/CodeGen.php | 40 ++++++++++++------- .../Query/Runners/Visitors/Interpreter.php | 25 +++--------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/Toolkit/Query/Runners/Visitors/CodeGen.php b/src/Toolkit/Query/Runners/Visitors/CodeGen.php index d28a766779..0c59935672 100644 --- a/src/Toolkit/Query/Runners/Visitors/CodeGen.php +++ b/src/Toolkit/Query/Runners/Visitors/CodeGen.php @@ -92,7 +92,7 @@ public function visitCoalesce(CoalesceNode $node): string public function visitLiteral(LiteralNode $node): string { - return '$intercept(' . var_export($node->value, true) . ')'; + return var_export($node->value, true); } public function visitMemberAccess(MemberAccessNode $node): string @@ -104,18 +104,16 @@ public function visitMemberAccess(MemberAccessNode $node): string $memberStr = var_export($member, true); $nullSafe = $node->nullSafe ? 'true' : 'false'; + $object = $this->intercept($object); + if ($node->arguments) { $arguments = $node->arguments->accept($this); $member = var_export($member, true); - return $this->intercept( - "Runtime::access($object, $memberStr, $nullSafe, $arguments)" - ); + return "Runtime::access($object, $memberStr, $nullSafe, $arguments)"; } - return $this->intercept( - "Runtime::access($object, $memberStr, $nullSafe)" - ); + return "Runtime::access($object, $memberStr, $nullSafe)"; } public function visitTernary(TernaryNode $node): string @@ -139,11 +137,18 @@ public function visitVariable(VariableNode $node): string $key = static::phpName($name); if (isset($this->directAccessFor[$name])) { - return $this->intercept($key); + return $key; } if (isset($this->mappings[$key]) === false) { - $this->mappings[$key] = $this->intercept("match(true) { isset(\$context[$namestr]) && \$context[$namestr] instanceof Closure => \$context[$namestr](), isset(\$context[$namestr]) => \$context[$namestr], isset(\$functions[$namestr]) => \$functions[$namestr](), default => null }"); + $this->mappings[$key] = << \$context[$namestr](), + isset(\$context[$namestr]) => \$context[$namestr], + isset(\$functions[$namestr]) => \$functions[$namestr](), + default => null + } + PHP; } return $key; @@ -156,14 +161,14 @@ public function visitGlobalFunction(GlobalFunctionNode $node): string { $name = $node->name(); - if (isset($this->validGlobalFunctions[$name])) { - throw new Exception("Invalid global function $name"); + if (isset($this->validGlobalFunctions[$name]) === false) { + throw new Exception("Invalid global function $name, valid functions are: " . join(', ', array_keys($this->validGlobalFunctions))); } $arguments = $node->arguments->accept($this); $name = var_export($name, true); - return $this->intercept($this->intercept("\$functions[$name]") . "($arguments)"); + return "\$functions[$name]($arguments)"; } public function visitClosure(ClosureNode $node): mixed @@ -178,8 +183,13 @@ public function visitClosure(ClosureNode $node): mixed ...array_fill_keys($node->arguments, true) ]; - return "fn($args) => " . $node->body->accept( - new static($this->validGlobalFunctions, $newDirectAccessFor) - ); + $nestedVisitor = new static($this->validGlobalFunctions, $newDirectAccessFor); + $code = $node->body->accept($nestedVisitor); + + // promote the nested visitor's uses and mappings to the current visitor + $this->uses += $nestedVisitor->uses; + $this->mappings += $nestedVisitor->mappings; + + return "fn($args) => $code"; } } diff --git a/src/Toolkit/Query/Runners/Visitors/Interpreter.php b/src/Toolkit/Query/Runners/Visitors/Interpreter.php index 01c1e74507..ac82bb9dd3 100644 --- a/src/Toolkit/Query/Runners/Visitors/Interpreter.php +++ b/src/Toolkit/Query/Runners/Visitors/Interpreter.php @@ -56,16 +56,17 @@ public function visitLiteral(LiteralNode $node): mixed { $val = $node->value; - if ($this->interceptor !== null) { - $val = ($this->interceptor)($val); - } - return $val; } public function visitMemberAccess(MemberAccessNode $node): mixed { $left = $node->object->accept($this); + + if ($this->interceptor !== null) { + $left = ($this->interceptor)($left); + } + $item = null; if ($node->arguments !== null) { @@ -79,10 +80,6 @@ public function visitMemberAccess(MemberAccessNode $node): mixed $item = Runtime::access($left, $node->member, $node->nullSafe); } - if ($this->interceptor !== null) { - $item = ($this->interceptor)($item); - } - return $item; } @@ -113,10 +110,6 @@ public function visitVariable(VariableNode $node): mixed default => null, }; - if ($this->interceptor !== null) { - $item = ($this->interceptor)($item); - } - return $item; } @@ -130,16 +123,8 @@ public function visitGlobalFunction(GlobalFunctionNode $node): mixed $function = $this->validGlobalFunctions[$name]; - if ($this->interceptor !== null) { - $function = ($this->interceptor)($function); - } - $result = $function(...$node->arguments->accept($this)); - if ($this->interceptor !== null) { - $result = ($this->interceptor)($result); - } - return $result; } From 002664224ed2a6d6fcbbe3b9da051a925b1c4e56 Mon Sep 17 00:00:00 2001 From: Roman Steiner Date: Wed, 4 Dec 2024 16:24:54 +0100 Subject: [PATCH 31/31] query fixes, more tests, inject cache as dependency Dependency Injecting the query cache into runners allows the Query class can manage it, since it's also the Query class which controls the global functions available to queries --- src/Query/Query.php | 4 +- src/Toolkit/Query/Runner.php | 1 + src/Toolkit/Query/Runners/Interpreted.php | 15 +- src/Toolkit/Query/Runners/Transpiled.php | 20 +- tests/Toolkit/Query/RunnerTest.php | 234 ++++++++++++++++++++++ 5 files changed, 258 insertions(+), 16 deletions(-) create mode 100644 tests/Toolkit/Query/RunnerTest.php diff --git a/src/Query/Query.php b/src/Query/Query.php index d271c40d08..34199d57a8 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -41,6 +41,8 @@ class Query */ public static array $entries = []; + private static array $resolverCache = []; + /** * Creates a new Query object */ @@ -119,7 +121,7 @@ public function resolve(array|object $data = []): mixed default => throw new Exception('Invalid query runner') }; - $runner = new $runnerClass(static::$entries, $this->intercept(...)); + $runner = new $runnerClass(static::$entries, $this->intercept(...), static::$resolverCache); return $runner->run($this->query, (array)$data); } } diff --git a/src/Toolkit/Query/Runner.php b/src/Toolkit/Query/Runner.php index d01b295660..7f10e4856f 100644 --- a/src/Toolkit/Query/Runner.php +++ b/src/Toolkit/Query/Runner.php @@ -15,6 +15,7 @@ abstract class Runner public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, + private array &$cache = [], ) { } diff --git a/src/Toolkit/Query/Runners/Interpreted.php b/src/Toolkit/Query/Runners/Interpreted.php index 139ed9ba77..a8f4bc6489 100644 --- a/src/Toolkit/Query/Runners/Interpreted.php +++ b/src/Toolkit/Query/Runners/Interpreted.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners; +use ArrayAccess; use Closure; use Kirby\Toolkit\Query\Parser; use Kirby\Toolkit\Query\Runner; @@ -10,29 +11,27 @@ class Interpreted extends Runner { - private static array $cache = []; - public function __construct( public array $allowedFunctions = [], protected Closure|null $interceptor = null, + private ArrayAccess|array &$resolverCache = [], ) { } protected function getResolver(string $query): Closure { - // load closure from process cache - if (isset(self::$cache[$query])) { - return self::$cache[$query]; + // load closure from cache + if (isset($this->resolverCache[$query])) { + return $this->resolverCache[$query]; } // on cache miss, parse query and generate closure $tokenizer = new Tokenizer($query); $parser = new Parser($tokenizer); $node = $parser->parse(); + $self = $this; - $self = $this; - - return self::$cache[$query] = function (array $binding) use ($node, $self) { + return $this->resolverCache[$query] = function (array $binding) use ($node, $self) { $interpreter = new Interpreter($self->allowedFunctions, $binding); if ($self->interceptor !== null) { diff --git a/src/Toolkit/Query/Runners/Transpiled.php b/src/Toolkit/Query/Runners/Transpiled.php index 09a3d5fbfd..a1a182fff8 100644 --- a/src/Toolkit/Query/Runners/Transpiled.php +++ b/src/Toolkit/Query/Runners/Transpiled.php @@ -2,6 +2,7 @@ namespace Kirby\Toolkit\Query\Runners; +use ArrayAccess; use Closure; use Exception; use Kirby\Toolkit\Query\Parser; @@ -11,7 +12,6 @@ class Transpiled extends Runner { - private static array $cache = []; public static string $cacheFolder = '/tmp/query_cache'; /** @@ -22,9 +22,16 @@ class Transpiled extends Runner public function __construct( public array $allowedFunctions = [], public Closure|null $interceptor = null, + private ArrayAccess|array &$cache = [], ) { } + public static function getCacheFile(string $query): string + { + $hash = crc32($query); + return self::$cacheFolder . '/' . $hash . '.php'; + } + /** * Retrieves the executor closure for a given query. * If the closure is not already cached, it will be generated and stored in `Runner::$cacheFolder`. @@ -35,16 +42,15 @@ public function __construct( protected function getResolver(string $query): Closure { // load closure from process memory - if (isset(self::$cache[$query])) { - return self::$cache[$query]; + if (isset($this->cache[$query])) { + return $this->cache[$query]; } // load closure from file-cache / opcache - $hash = crc32($query); - $filename = self::$cacheFolder . '/' . $hash . '.php'; + $filename = self::getCacheFile($query); if (file_exists($filename)) { - return self::$cache[$query] = include $filename; + return $this->cache[$query] = include $filename; } // on cache miss, parse query and generate closure @@ -78,7 +84,7 @@ protected function getResolver(string $query): Closure file_put_contents($filename, $function); // load from file-cache to create opcache entry - return self::$cache[$query] = include $filename; + return $this->cache[$query] = include $filename; } diff --git a/tests/Toolkit/Query/RunnerTest.php b/tests/Toolkit/Query/RunnerTest.php new file mode 100644 index 0000000000..781f879cec --- /dev/null +++ b/tests/Toolkit/Query/RunnerTest.php @@ -0,0 +1,234 @@ +run($query, $context); + + $this->assertEquals($intercept, $actuallyIntercepted); + } + + /** + * @dataProvider interceptProvider + */ + public function testTranspiledIntercept(string $query, array $context, array $intercept, array $globalFunctions = []) + { + $actuallyItercepted = []; + + $interceptorSpy = function ($value) use (&$actuallyItercepted) { + $actuallyItercepted[] = $value; + return $value; + }; + + Transpiled::$cacheFolder = static::TMP; + + $runner = new Transpiled( + allowedFunctions: $globalFunctions, + interceptor: $interceptorSpy + ); + $runner->run($query, $context); + + $this->assertEquals($intercept, $actuallyItercepted, 'Generated PHP Code:' . PHP_EOL . file_get_contents(Transpiled::getCacheFile($query))); + } + + /** + * @dataProvider resultProvider + */ + public function testInterpretedResult(string $query, array $context, mixed $result, array $globalFunctions = []) + { + $runner = new Interpreted( + allowedFunctions: $globalFunctions, + ); + + $actualResult = $runner->run($query, $context); + + $this->assertEquals($result, $actualResult); + } + + /** + * @dataProvider resultProvider + */ + public function testTranspiledResult(string $query, array $context, mixed $result, array $globalFunctions = []) + { + Transpiled::$cacheFolder = static::TMP; + + $runner = new Transpiled( + allowedFunctions: $globalFunctions, + ); + + $actualResult = $runner->run($query, $context); + $code = file_get_contents(Transpiled::getCacheFile($query)); + + $this->assertEquals($result, $actualResult, 'Generated PHP Code:' . PHP_EOL . $code); + } + + /** + * Runners should keep a cache of parsed queries to avoid parsing the same query multiple times + */ + public function testInterpretParsesOnlyOnce() + { + $cache = []; + + $cacheSpy = $this->createStub(ArrayAccess::class); + + $cacheSpy + ->expects($this->exactly(2)) + ->method('offsetExists') + ->willReturnCallback(function ($key) use (&$cache) { + return isset($cache[$key]); + }); + + $cacheSpy + ->expects($this->exactly(1)) + ->method('offsetGet') + ->willReturnCallback(function ($key) use (&$cache) { + return $cache[$key] ?? null; + }); + + $cacheSpy + ->expects($this->exactly(1)) + ->method('offsetSet') + ->willReturnCallback(function ($key, $val) use (&$cache) { + $cache[$key] = $val; + }); + + $runner1 = new Interpreted(resolverCache: $cacheSpy); + $runner2 = new Interpreted(resolverCache: $cacheSpy); + + // it should still give different results for different contexts + $this->assertEquals(42, $runner1->run('foo.bar', ['foo' => ['bar' => 42]])); + $this->assertEquals(84, $runner2->run('foo.bar', ['foo' => ['bar' => 84]])); + } + + public static function resultProvider() + { + return [ + 'field' => [ + 'user.name', // query + ['user' => ['name' => 'Homer']], // context + 'Homer', // result + ], + + 'nested field' => [ + 'user.name.first', // query + ['user' => ['name' => ['first' => 'Homer']]], // context + 'Homer' // result + ], + + 'method result' => [ + 'user.get("arg").thing', // query + ['user' => ['get' => fn ($a) => ['thing' => $a]]], // context + 'arg' // result + ], + + 'closure access to parent context' => [ + 'thing.call(() => result).field', // query + ['result' => ['field' => 42], 'thing' => ['call' => fn ($callback) => $callback()]], // context + 42 // result + ], + + 'function result for explicit global function' => [ + 'foo(42).bar', // query + [], // context + 84, // result + ['foo' => fn ($a) => ['bar' => $a * 2]] // globalFunctions + ], + + 'global function result when function looks like variable - i' => [ + 'foo.bar', // query + [], // context + 42, // result + ['foo' => fn () => ['bar' => 42]] // globalFunctions + ], + ]; + } + + + public static function interceptProvider() + { + return [ + 'field' => [ + 'user.name', // query + ['user' => $user = ['name' => 'Homer']], // context + [$user], // intercept + ], + + 'nested field' => [ + 'user.name.first', // query + ['user' => $user = ['name' => $name = ['first' => 'Homer']]], // context + [$user, $name] // intercept + ], + + 'method result' => (function () { + $closureResult = ['age' => 42]; + $user = ['get' => fn () => $closureResult]; + + return [ + 'user.get("arg").age', // query + ['user' => $user], // context + [$user, $closureResult] // intercept + ]; + })(), + + 'closure result' => (function () { + $result = ['field' => 'value']; + $thing = ['call' => fn ($callback) => $callback()]; + + return [ + 'thing.call(() => result).field', // query + ['thing' => $thing, 'result' => $result], // context + [$thing, $result] // intercept + ]; + })(), + + 'function result for explicit global function' => (function () { + $functionResult = ['bar' => 'baz']; + $globalFunctions = ['foo' => fn () => $functionResult]; + + return [ + 'foo("arg").bar', // query + [], // context + [$functionResult], // intercept + $globalFunctions // globalFunctions + ]; + })(), + + 'global function result when function looks like variable - a' => (function () { + $functionResult = ['bar' => 'baz']; + $globalFunctions = ['foo' => fn () => $functionResult]; + + return [ + 'foo.bar', // query + [], // context + [$functionResult], // intercept + $globalFunctions // globalFunctions + ]; + })() + ]; + } +}