Merge branch 'php-8.1/tokenizer-php-bugfix-octal-explicit-notation' of …

…https://github.com/jrfnl/PHP_CodeSniffer
BafS · Jun 13, 2022 · 7f5c55d · 7f5c55d
2 parents e9f6c43 + 915b12a
commit 7f5c55d
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 34 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -752,16 +752,32 @@ protected function tokenize($string)
                 && $tokens[($stackPtr + 1)][0] === T_STRING
                 && strtolower($tokens[($stackPtr + 1)][1][0]) === 'o'
                 && $tokens[($stackPtr + 1)][1][1] !== '_')
+                && preg_match('`^(o[0-7]+(?:_[0-7]+)?)([0-9_]*)$`i', $tokens[($stackPtr + 1)][1], $matches) === 1
             ) {
                 $finalTokens[$newStackPtr] = [
                     'code'    => T_LNUMBER,
                     'type'    => 'T_LNUMBER',
-                    'content' => $token[1] .= $tokens[($stackPtr + 1)][1],
+                    'content' => $token[1] .= $matches[1],
                 ];
-                $stackPtr++;
                 $newStackPtr++;
+
+                if (isset($matches[2]) === true && $matches[2] !== '') {
+                    $type = 'T_LNUMBER';
+                    if ($matches[2][0] === '_') {
+                        $type = 'T_STRING';
+                    }
+
+                    $finalTokens[$newStackPtr] = [
+                        'code'    => constant($type),
+                        'type'    => $type,
+                        'content' => $matches[2],
+                    ];
+                    $newStackPtr++;
+                }
+
+                $stackPtr++;
                 continue;
-            }
+            }//end if
 
             /*
                 PHP 8.1 introduced two dedicated tokens for the & character.

diff --git a/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc b/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc
@@ -14,3 +14,15 @@ $foo = 0o_137;
 
 /* testInvalid2 */
 $foo = 0O_41;
+
+/* testInvalid3 */
+$foo = 0o91;
+
+/* testInvalid4 */
+$foo = 0O282;
+
+/* testInvalid5 */
+$foo = 0o28_2;
+
+/* testInvalid6 */
+$foo = 0o2_82;
diff --git a/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.php b/tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.php
@@ -18,22 +18,26 @@ class BackfillExplicitOctalNotationTest extends AbstractMethodUnitTest
     /**
      * Test that explicitly-defined octal values are tokenized as a single number and not as a number and a string.
      *
-     * @param array $testData The data required for the specific test case.
+     * @param string     $marker      The comment which prefaces the target token in the test file.
+     * @param string     $value       The expected content of the token
+     * @param int|string $nextToken   The expected next token.
+     * @param string     $nextContent The expected content of the next token.
      *
      * @dataProvider dataExplicitOctalNotation
      * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
      *
      * @return void
      */
-    public function testExplicitOctalNotation($testData)
+    public function testExplicitOctalNotation($marker, $value, $nextToken, $nextContent)
     {
         $tokens = self::$phpcsFile->getTokens();
 
-        $number = $this->getTargetToken($testData['marker'], [T_LNUMBER]);
+        $number = $this->getTargetToken($marker, [T_LNUMBER]);
 
-        $this->assertSame(constant($testData['type']), $tokens[$number]['code']);
-        $this->assertSame($testData['type'], $tokens[$number]['type']);
-        $this->assertSame($testData['value'], $tokens[$number]['content']);
+        $this->assertSame($value, $tokens[$number]['content'], 'Content of integer token does not match expectation');
+
+        $this->assertSame($nextToken, $tokens[($number + 1)]['code'], 'Next token is not the expected type, but '.$tokens[($number + 1)]['type']);
+        $this->assertSame($nextContent, $tokens[($number + 1)]['content'], 'Next token did not have the expected contents');
 
     }//end testExplicitOctalNotation()
 
@@ -49,39 +53,58 @@ public function dataExplicitOctalNotation()
     {
         return [
             [
-                [
-                    'marker' => '/* testExplicitOctal */',
-                    'type'   => 'T_LNUMBER',
-                    'value'  => '0o137041',
-                ],
+                'marker'      => '/* testExplicitOctal */',
+                'value'       => '0o137041',
+                'nextToken'   => T_SEMICOLON,
+                'nextContent' => ';',
+            ],
+            [
+                'marker'      => '/* testExplicitOctalCapitalised */',
+                'value'       => '0O137041',
+                'nextToken'   => T_SEMICOLON,
+                'nextContent' => ';',
+            ],
+            [
+                'marker'      => '/* testExplicitOctalWithNumericSeparator */',
+                'value'       => '0o137_041',
+                'nextToken'   => T_SEMICOLON,
+                'nextContent' => ';',
+            ],
+            [
+                'marker'      => '/* testInvalid1 */',
+                'value'       => '0',
+                'nextToken'   => T_STRING,
+                'nextContent' => 'o_137',
+            ],
+            [
+                'marker'      => '/* testInvalid2 */',
+                'value'       => '0',
+                'nextToken'   => T_STRING,
+                'nextContent' => 'O_41',
             ],
             [
-                [
-                    'marker' => '/* testExplicitOctalCapitalised */',
-                    'type'   => 'T_LNUMBER',
-                    'value'  => '0O137041',
-                ],
+                'marker'      => '/* testInvalid3 */',
+                'value'       => '0',
+                'nextToken'   => T_STRING,
+                'nextContent' => 'o91',
             ],
             [
-                [
-                    'marker' => '/* testExplicitOctalWithNumericSeparator */',
-                    'type'   => 'T_LNUMBER',
-                    'value'  => '0o137_041',
-                ],
+                'marker'      => '/* testInvalid4 */',
+                'value'       => '0O2',
+                'nextToken'   => T_LNUMBER,
+                'nextContent' => '82',
             ],
             [
-                [
-                    'marker' => '/* testInvalid1 */',
-                    'type'   => 'T_LNUMBER',
-                    'value'  => '0',
-                ],
+                'marker'      => '/* testInvalid5 */',
+                'value'       => '0o2',
+                'nextToken'   => T_LNUMBER,
+                'nextContent' => '8_2',
             ],
             [
-                [
-                    'marker' => '/* testInvalid2 */',
-                    'type'   => 'T_LNUMBER',
-                    'value'  => '0',
-                ],
+                'marker'      => '/* testInvalid6 */',
+                'value'       => '0o2',
+                'nextToken'   => T_STRING,
+                'nextContent' => '_82',
             ],
         ];