From a39bce3058407767d1c2cf13790d13dda7def3a5 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Sat, 27 Jan 2024 04:47:26 +0100 Subject: [PATCH] PHP 8.3 | Tokenizer/PHP: add support for readonly anonymous classes PHP 8.3 introduced readonly anonymous classes, fixing an oversight in the PHP 8.2 introduction of readonly classes. As things were, for PHP 8.1+, the tokenizer would change the token code for the `readonly` keyword from `T_READONLY` to `T_STRING` in the "context sensitive keyword" layer, thinking it to be a class name. And for PHP < 8.1, the readonly polyfill would ignore the token as it being preceded by the `new` keyword would be seen as conflicting with the "context sensitive keyword" layer, which meant it would not be re-tokenized from `T_STRING` to `T_READONLY`. This commit fixes both. Includes adding tests in a number of pre-existing test classes to cover this change. --- src/Tokenizers/PHP.php | 18 +++++++++++++++++- .../AnonClassParenthesisOwnerTest.inc | 10 ++++++++++ .../AnonClassParenthesisOwnerTest.php | 6 ++++++ tests/Core/Tokenizer/BackfillReadonlyTest.inc | 13 +++++++++++++ tests/Core/Tokenizer/BackfillReadonlyTest.php | 11 +++++++++++ .../Tokenizer/ContextSensitiveKeywordsTest.inc | 6 +++++- .../Tokenizer/ContextSensitiveKeywordsTest.php | 6 +++++- 7 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index e8ad0db793..fbb38fb31c 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -621,6 +621,21 @@ protected function tokenize($string) $preserveKeyword = true; } + // `new readonly class` should be preserved. + if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW) { + for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { + if (is_array($tokens[$i]) === false + || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false + ) { + break; + } + } + + if (is_array($tokens[$i]) === true && $tokens[$i][0] === T_CLASS) { + $preserveKeyword = true; + } + } + // `new class extends` `new class implements` should be preserved if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS) && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS @@ -1315,7 +1330,8 @@ protected function tokenize($string) if ($tokenIsArray === true && strtolower($token[1]) === 'readonly' - && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false + && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false + || $finalTokens[$lastNotEmptyToken]['code'] === T_NEW) ) { // Get the next non-whitespace token. for ($i = ($stackPtr + 1); $i < $numTokens; $i++) { diff --git a/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc b/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc index 5867691c64..3ee1afd0d0 100644 --- a/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc +++ b/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc @@ -5,6 +5,11 @@ $anonClass = new class { function __construct() {} }; +/* testReadonlyNoParentheses */ +$anonClass = new readonly class { + function __construct() {} +}; + /* testNoParenthesesAndEmptyTokens */ $anonClass = new class // phpcs:ignore Standard.Cat { @@ -14,6 +19,11 @@ $anonClass = new class // phpcs:ignore Standard.Cat /* testWithParentheses */ $anonClass = new class() {}; +/* testReadonlyWithParentheses */ +$anonClass = new readonly class() { + function __construct() {} +}; + /* testWithParenthesesAndEmptyTokens */ $anonClass = new class /*comment */ () {}; diff --git a/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php b/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php index a22481c0cf..79eb742e19 100644 --- a/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php +++ b/tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php @@ -76,6 +76,9 @@ public static function dataAnonClassNoParentheses() 'plain' => [ 'testMarker' => '/* testNoParentheses */', ], + 'readonly' => [ + 'testMarker' => '/* testReadonlyNoParentheses */', + ], 'declaration contains comments and extra whitespace' => [ 'testMarker' => '/* testNoParenthesesAndEmptyTokens */', ], @@ -139,6 +142,9 @@ public static function dataAnonClassWithParentheses() 'plain' => [ 'testMarker' => '/* testWithParentheses */', ], + 'readonly' => [ + 'testMarker' => '/* testReadonlyWithParentheses */', + ], 'declaration contains comments and extra whitespace' => [ 'testMarker' => '/* testWithParenthesesAndEmptyTokens */', ], diff --git a/tests/Core/Tokenizer/BackfillReadonlyTest.inc b/tests/Core/Tokenizer/BackfillReadonlyTest.inc index cbc91f13a3..29c202a6ed 100644 --- a/tests/Core/Tokenizer/BackfillReadonlyTest.inc +++ b/tests/Core/Tokenizer/BackfillReadonlyTest.inc @@ -138,6 +138,19 @@ class ReadonlyWithDisjunctiveNormalForm public function readonly (A&B $param): void {} } +/* testReadonlyAnonClassWithParens */ +$anon = new readonly class() {}; + +/* testReadonlyAnonClassWithoutParens */ +$anon = new Readonly class {}; + +/* testReadonlyAnonClassWithCommentsAndWhitespace */ +$anon = new +// comment +ReadOnly +// phpcs:ignore Stnd.Cat.Sniff +class {}; + /* testParseErrorLiveCoding */ // This must be the last test in the file. readonly diff --git a/tests/Core/Tokenizer/BackfillReadonlyTest.php b/tests/Core/Tokenizer/BackfillReadonlyTest.php index 8fbc3ffa6c..993e1a6f30 100644 --- a/tests/Core/Tokenizer/BackfillReadonlyTest.php +++ b/tests/Core/Tokenizer/BackfillReadonlyTest.php @@ -151,6 +151,17 @@ public static function dataReadonly() 'property declaration, constructor property promotion, DNF type and reference' => [ 'testMarker' => '/* testReadonlyConstructorPropertyPromotionWithDNFAndReference */', ], + 'anon class declaration, with parentheses' => [ + 'testMarker' => '/* testReadonlyAnonClassWithParens */', + ], + 'anon class declaration, without parentheses' => [ + 'testMarker' => '/* testReadonlyAnonClassWithoutParens */', + 'testContent' => 'Readonly', + ], + 'anon class declaration, with comments and whitespace' => [ + 'testMarker' => '/* testReadonlyAnonClassWithCommentsAndWhitespace */', + 'testContent' => 'ReadOnly', + ], 'live coding / parse error' => [ 'testMarker' => '/* testParseErrorLiveCoding */', ], diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc index aedc49cc67..abbfa62d4d 100644 --- a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc +++ b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc @@ -99,12 +99,16 @@ namespace /* testNamespaceNameIsString1 */ my\ /* testNamespaceNameIsString2 */ /* testVarIsKeyword */ var $var; /* testStaticIsKeyword */ static $static; - /* testReadonlyIsKeyword */ readonly $readonly; + /* testReadonlyIsKeywordForProperty */ readonly $readonly; /* testFinalIsKeyword */ final /* testFunctionIsKeyword */ function someFunction( /* testCallableIsKeyword */ callable $callable, ) { + $anon = new /* testReadonlyIsKeywordForAnonClass */ readonly class() { + public function foo() {} + }; + /* testReturnIsKeyword */ return $this; } diff --git a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php index 9116782479..0cc403e94a 100644 --- a/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php +++ b/tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php @@ -232,7 +232,7 @@ public static function dataKeywords() 'expectedTokenType' => 'T_STATIC', ], 'readonly: property declaration' => [ - 'testMarker' => '/* testReadonlyIsKeyword */', + 'testMarker' => '/* testReadonlyIsKeywordForProperty */', 'expectedTokenType' => 'T_READONLY', ], 'final: function declaration' => [ @@ -247,6 +247,10 @@ public static function dataKeywords() 'testMarker' => '/* testCallableIsKeyword */', 'expectedTokenType' => 'T_CALLABLE', ], + 'readonly: anon class declaration' => [ + 'testMarker' => '/* testReadonlyIsKeywordForAnonClass */', + 'expectedTokenType' => 'T_READONLY', + ], 'return: statement' => [ 'testMarker' => '/* testReturnIsKeyword */', 'expectedTokenType' => 'T_RETURN',