Upstream doctrine/lexer state of the touched files before 7b15e53ef2

src/AbstractLexer.php

@@ -0,0 +1,331 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Doctrine\Common\Lexer;
+
+use ReflectionClass;
+use UnitEnum;
+
+use function implode;
+use function preg_split;
+use function sprintf;
+use function substr;
+
+use const PREG_SPLIT_DELIM_CAPTURE;
+use const PREG_SPLIT_NO_EMPTY;
+use const PREG_SPLIT_OFFSET_CAPTURE;
+
+/**
+ * Base class for writing simple lexers, i.e. for creating small DSLs.
+ *
+ * @template T of UnitEnum|string|int
+ * @template V of string|int|float|bool
+ */
+abstract class AbstractLexer
+{
+    /**
+     * Lexer original input string.
+     */
+    private string $input;
+
+    /**
+     * Array of scanned tokens.
+     *
+     * @var list<Token<T, V>>
+     */
+    private array $tokens = [];
+
+    /**
+     * Current lexer position in input string.
+     */
+    private int $position = 0;
+
+    /**
+     * Current peek of current lexer position.
+     */
+    private int $peek = 0;
+
+    /**
+     * The next token in the input.
+     *
+     * @var Token<T, V>|null
+     */
+    public Token|null $lookahead;
+
+    /**
+     * The last matched/seen token.
+     *
+     * @var Token<T, V>|null
+     */
+    public Token|null $token;
+
+    /**
+     * Composed regex for input parsing.
+     *
+     * @var non-empty-string|null
+     */
+    private string|null $regex = null;
+
+    /**
+     * Sets the input data to be tokenized.
+     *
+     * The Lexer is immediately reset and the new input tokenized.
+     * Any unprocessed tokens from any previous input are lost.
+     *
+     * @param string $input The input to be tokenized.
+     *
+     * @return void
+     */
+    public function setInput(string $input)
+    {
+        $this->input  = $input;
+        $this->tokens = [];
+
+        $this->reset();
+        $this->scan($input);
+    }
+
+    /**
+     * Resets the lexer.
+     *
+     * @return void
+     */
+    public function reset()
+    {
+        $this->lookahead = null;
+        $this->token     = null;
+        $this->peek      = 0;
+        $this->position  = 0;
+    }
+
+    /**
+     * Resets the peek pointer to 0.
+     *
+     * @return void
+     */
+    public function resetPeek()
+    {
+        $this->peek = 0;
+    }
+
+    /**
+     * Resets the lexer position on the input to the given position.
+     *
+     * @param int $position Position to place the lexical scanner.
+     *
+     * @return void
+     */
+    public function resetPosition(int $position = 0)
+    {
+        $this->position = $position;
+    }
+
+    /**
+     * Retrieve the original lexer's input until a given position.
+     *
+     * @return string
+     */
+    public function getInputUntilPosition(int $position)
+    {
+        return substr($this->input, 0, $position);
+    }
+
+    /**
+     * Checks whether a given token matches the current lookahead.
+     *
+     * @param T $type
+     *
+     * @return bool
+     *
+     * @phpstan-assert-if-true !=null $this->lookahead
+     */
+    public function isNextToken(int|string|UnitEnum $type)
+    {
+        return $this->lookahead !== null && $this->lookahead->isA($type);
+    }
+
+    /**
+     * Checks whether any of the given tokens matches the current lookahead.
+     *
+     * @param list<T> $types
+     *
+     * @return bool
+     *
+     * @phpstan-assert-if-true !=null $this->lookahead
+     */
+    public function isNextTokenAny(array $types)
+    {
+        return $this->lookahead !== null && $this->lookahead->isA(...$types);
+    }
+
+    /**
+     * Moves to the next token in the input string.
+     *
+     * @return bool
+     *
+     * @phpstan-impure
+     * @phpstan-assert-if-true !null $this->lookahead
+     */
+    public function moveNext()
+    {
+        $this->peek      = 0;
+        $this->token     = $this->lookahead;
+        $this->lookahead = isset($this->tokens[$this->position])
+            ? $this->tokens[$this->position++] : null;
+
+        return $this->lookahead !== null;
+    }
+
+    /**
+     * Tells the lexer to skip input tokens until it sees a token with the given value.
+     *
+     * @param T $type The token type to skip until.
+     *
+     * @return void
+     */
+    public function skipUntil(int|string|UnitEnum $type)
+    {
+        while ($this->lookahead !== null && ! $this->lookahead->isA($type)) {
+            $this->moveNext();
+        }
+    }
+
+    /**
+     * Checks if given value is identical to the given token.
+     *
+     * @return bool
+     */
+    public function isA(string $value, int|string|UnitEnum $token)
+    {
+        return $this->getType($value) === $token;
+    }
+
+    /**
+     * Moves the lookahead token forward.
+     *
+     * @return Token<T, V>|null The next token or NULL if there are no more tokens ahead.
+     */
+    public function peek()
+    {
+        if (isset($this->tokens[$this->position + $this->peek])) {
+            return $this->tokens[$this->position + $this->peek++];
+        }
+
+        return null;
+    }
+
+    /**
+     * Peeks at the next token, returns it and immediately resets the peek.
+     *
+     * @return Token<T, V>|null The next token or NULL if there are no more tokens ahead.
+     *
+     * @phpstan-impure
+     */
+    public function glimpse()
+    {
+        $peek       = $this->peek();
+        $this->peek = 0;
+
+        return $peek;
+    }
+
+    /**
+     * Scans the input string for tokens.
+     *
+     * @param string $input A query string.
+     *
+     * @return void
+     */
+    protected function scan(string $input)
+    {
+        if (! isset($this->regex)) {
+            $this->regex = sprintf(
+                '/(%s)|%s/%s',
+                implode(')|(', $this->getCatchablePatterns()),
+                implode('|', $this->getNonCatchablePatterns()),
+                $this->getModifiers(),
+            );
+        }
+
+        $flags   = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
+        $matches = preg_split($this->regex, $input, -1, $flags);
+
+        if ($matches === false) {
+            // Work around https://bugs.php.net/78122
+            $matches = [[$input, 0]];
+        }
+
+        foreach ($matches as $match) {
+            // Must remain before 'value' assignment since it can change content
+            $firstMatch = $match[0];
+            $type       = $this->getType($firstMatch);
+
+            $this->tokens[] = new Token(
+                $firstMatch,
+                $type,
+                $match[1],
+            );
+        }
+    }
+
+    /**
+     * Gets the literal for a given token.
+     *
+     * @param T $token
+     *
+     * @return int|string
+     */
+    public function getLiteral(int|string|UnitEnum $token)
+    {
+        if ($token instanceof UnitEnum) {
+            return $token::class . '::' . $token->name;
+        }
+
+        $className = static::class;
+
+        $reflClass = new ReflectionClass($className);
+        $constants = $reflClass->getConstants();
+
+        foreach ($constants as $name => $value) {
+            if ($value === $token) {
+                return $className . '::' . $name;
+            }
+        }
+
+        return $token;
+    }
+
+    /**
+     * Regex modifiers
+     *
+     * @return string
+     */
+    protected function getModifiers()
+    {
+        return 'iu';
+    }
+
+    /**
+     * Lexical catchable patterns.
+     *
+     * @return string[]
+     */
+    abstract protected function getCatchablePatterns();
+
+    /**
+     * Lexical non-catchable patterns.
+     *
+     * @return string[]
+     */
+    abstract protected function getNonCatchablePatterns();
+
+    /**
+     * Retrieve token type. Also processes the token value if necessary.
+     *
+     * @return T|null
+     *
+     * @param-out V $value
+     */
+    abstract protected function getType(string &$value);
+}

tests/AbstractLexerTest.php

@@ -0,0 +1,341 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Doctrine\Tests\Common\Lexer;
+
+use Doctrine\Common\Lexer\AbstractLexer;
+use Doctrine\Common\Lexer\Token;
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\TestCase;
+
+use function array_map;
+use function assert;
+use function count;
+use function is_int;
+use function is_numeric;
+use function setlocale;
+
+use const LC_ALL;
+
+class AbstractLexerTest extends TestCase
+{
+    private ConcreteLexer $concreteLexer;
+
+    public function setUp(): void
+    {
+        $this->concreteLexer = new ConcreteLexer();
+    }
+
+    public function tearDown(): void
+    {
+        setlocale(LC_ALL, null);
+    }
+
+    /** @phpstan-return list<array{string, list<Token<string, string|int>>}> */
+    public static function dataProvider(): array
+    {
+        return [
+            [
+                'price=10',
+                [
+                    new Token('price', 'string', 0),
+                    new Token('=', 'operator', 5),
+                    new Token(10, 'int', 6),
+                ],
+            ],
+        ];
+    }
+
+    public function testResetPeek(): void
+    {
+        $expectedTokens = [
+            new Token('price', 'string', 0),
+            new Token('=', 'operator', 5),
+            new Token(10, 'int', 6),
+        ];
+
+        $this->concreteLexer->setInput('price=10');
+
+        $this->assertEquals($expectedTokens[0], $this->concreteLexer->peek());
+        $this->assertEquals($expectedTokens[1], $this->concreteLexer->peek());
+        $this->concreteLexer->resetPeek();
+        $this->assertEquals($expectedTokens[0], $this->concreteLexer->peek());
+    }
+
+    public function testResetPosition(): void
+    {
+        $expectedTokens = [
+            new Token('price', 'string', 0),
+            new Token('=', 'operator', 5),
+            new Token(10, 'int', 6),
+        ];
+
+        $this->concreteLexer->setInput('price=10');
+        $this->assertNull($this->concreteLexer->lookahead);
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead);
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->assertEquals($expectedTokens[1], $this->concreteLexer->lookahead);
+
+        $this->concreteLexer->resetPosition(0);
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead);
+    }
+
+    /** @phpstan-param list<Token<string, string|int>>  $expectedTokens */
+    #[DataProvider('dataProvider')]
+    public function testMoveNext(string $input, array $expectedTokens): void
+    {
+        $this->concreteLexer->setInput($input);
+        $this->assertNull($this->concreteLexer->lookahead);
+
+        for ($i = 0; $i < count($expectedTokens); $i++) {
+            $this->assertTrue($this->concreteLexer->moveNext());
+            $this->assertEquals($expectedTokens[$i], $this->concreteLexer->lookahead);
+        }
+
+        $this->assertFalse($this->concreteLexer->moveNext());
+        $this->assertNull($this->concreteLexer->lookahead);
+    }
+
+    public function testSkipUntil(): void
+    {
+        $this->concreteLexer->setInput('price=10');
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->concreteLexer->skipUntil('operator');
+
+        $this->assertEquals(
+            new Token('=', 'operator', 5),
+            $this->concreteLexer->lookahead,
+        );
+    }
+
+    public function testUtf8Mismatch(): void
+    {
+        $this->concreteLexer->setInput("\xE9=10");
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+
+        $this->assertEquals(
+            new Token("\xE9=10", 'string', 0),
+            $this->concreteLexer->lookahead,
+        );
+    }
+
+    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */
+    #[DataProvider('dataProvider')]
+    public function testPeek(string $input, array $expectedTokens): void
+    {
+        $this->concreteLexer->setInput($input);
+        foreach ($expectedTokens as $expectedToken) {
+            $actualToken = $this->concreteLexer->peek();
+            assert($actualToken !== null);
+            $this->assertEquals($expectedToken, $actualToken);
+            $this->assertSame($expectedToken->value, $actualToken->value);
+            $this->assertSame($expectedToken->type, $actualToken->type);
+            $this->assertSame($expectedToken->position, $actualToken->position);
+        }
+
+        $this->assertNull($this->concreteLexer->peek());
+    }
+
+    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */
+    #[DataProvider('dataProvider')]
+    public function testGlimpse(string $input, array $expectedTokens): void
+    {
+        $this->concreteLexer->setInput($input);
+
+        foreach ($expectedTokens as $expectedToken) {
+            $actualToken = $this->concreteLexer->glimpse();
+            assert($actualToken !== null);
+            $this->assertEquals($expectedToken, $actualToken);
+            $this->assertEquals($expectedToken, $this->concreteLexer->glimpse());
+            $this->assertSame($expectedToken->value, $actualToken->value);
+            $this->assertSame($expectedToken->type, $actualToken->type);
+            $this->assertSame($expectedToken->position, $actualToken->position);
+            $this->concreteLexer->moveNext();
+        }
+
+        $this->assertNull($this->concreteLexer->peek());
+    }
+
+    /** @phpstan-return list<array{string, int, string}> */
+    public static function inputUntilPositionDataProvider(): array
+    {
+        return [
+            ['price=10', 5, 'price'],
+        ];
+    }
+
+    #[DataProvider('inputUntilPositionDataProvider')]
+    public function testGetInputUntilPosition(
+        string $input,
+        int $position,
+        string $expectedInput,
+    ): void {
+        $this->concreteLexer->setInput($input);
+
+        $this->assertSame($expectedInput, $this->concreteLexer->getInputUntilPosition($position));
+    }
+
+    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */
+    #[DataProvider('dataProvider')]
+    public function testIsNextToken(string $input, array $expectedTokens): void
+    {
+        $this->concreteLexer->setInput($input);
+
+        $this->concreteLexer->moveNext();
+        for ($i = 0; $i < count($expectedTokens); $i++) {
+            assert($expectedTokens[$i]->type !== null);
+            $this->assertTrue($this->concreteLexer->isNextToken($expectedTokens[$i]->type));
+            $this->concreteLexer->moveNext();
+        }
+    }
+
+    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */
+    #[DataProvider('dataProvider')]
+    public function testIsNextTokenAny(string $input, array $expectedTokens): void
+    {
+        $allTokenTypes = array_map(static function ($token): string {
+            assert($token->type !== null);
+
+            return $token->type;
+        }, $expectedTokens);
+
+        $this->concreteLexer->setInput($input);
+
+        $this->concreteLexer->moveNext();
+        for ($i = 0; $i < count($expectedTokens); $i++) {
+            assert($expectedTokens[$i]->type !== null);
+            $this->assertTrue($this->concreteLexer->isNextTokenAny([$expectedTokens[$i]->type]));
+            $this->assertTrue($this->concreteLexer->isNextTokenAny($allTokenTypes));
+            $this->concreteLexer->moveNext();
+        }
+    }
+
+    public function testGetLiteral(): void
+    {
+        $this->assertSame('Doctrine\Tests\Common\Lexer\ConcreteLexer::INT', $this->concreteLexer->getLiteral('int'));
+        $this->assertSame('fake_token', $this->concreteLexer->getLiteral('fake_token'));
+    }
+
+    public function testGetLiteralWithEnumLexer(): void
+    {
+        $enumLexer = new EnumLexer();
+        $this->assertSame(
+            'Doctrine\Tests\Common\Lexer\TokenType::OPERATOR',
+            $enumLexer->getLiteral(TokenType::OPERATOR),
+        );
+    }
+
+    public function testIsA(): void
+    {
+        $this->assertTrue($this->concreteLexer->isA('11', 'int'));
+        $this->assertTrue($this->concreteLexer->isA('1.1', 'int'));
+        $this->assertTrue($this->concreteLexer->isA('=', 'operator'));
+        $this->assertTrue($this->concreteLexer->isA('>', 'operator'));
+        $this->assertTrue($this->concreteLexer->isA('<', 'operator'));
+        $this->assertTrue($this->concreteLexer->isA('fake_text', 'string'));
+    }
+
+    public function testAddCatchablePatternsToMutableLexer(): void
+    {
+        $mutableLexer = new MutableLexer();
+        $mutableLexer->addCatchablePattern('[a-z]');
+        $mutableLexer->setInput('one');
+        $token = $mutableLexer->glimpse();
+
+        $this->assertNotNull($token);
+        $this->assertEquals('o', $token->value);
+
+        $mutableLexer = new MutableLexer();
+        $mutableLexer->addCatchablePattern('[a-z]+');
+        $mutableLexer->setInput('one');
+        $token = $mutableLexer->glimpse();
+
+        $this->assertNotNull($token);
+        $this->assertEquals('one', $token->value);
+    }
+
+    public function testMarkerAnnotationLocaleTr(): void
+    {
+        setlocale(LC_ALL, 'tr_TR.utf8', 'tr_TR');
+        $mutableLexer = new MutableLexer();
+        $mutableLexer->addCatchablePattern('[a-z_\\\][a-z0-9_\:\\\]*[a-z_][a-z0-9_]*');
+        $mutableLexer->addCatchablePattern('(?:[+-]?[0-9]+(?:[\.][0-9]+)*)(?:[eE][+-]?[0-9]+)?');
+        $mutableLexer->addCatchablePattern('"(?:""|[^"])*+"');
+        $mutableLexer->setInput('@ODM\Id');
+
+        self::assertNull($mutableLexer->token);
+        self::assertNull($mutableLexer->lookahead);
+        self::assertTrue($mutableLexer->moveNext());
+        self::assertNull($mutableLexer->token);
+        self::assertNotNull($mutableLexer->lookahead);
+        self::assertEquals('@', $mutableLexer->lookahead->value);
+        self::assertTrue($mutableLexer->moveNext());
+        self::assertNotNull($mutableLexer->token);
+        self::assertEquals('@', $mutableLexer->token->value);
+        self::assertEquals('ODM\Id', $mutableLexer->lookahead->value);
+    }
+
+    public function testCanTokenizeFloatValue(): void
+    {
+        $lexer = new /** @template-extends AbstractLexer<int, string|int|float|bool> */ class () extends AbstractLexer {
+            final public const T_NONE    = 1;
+            final public const T_INTEGER = 2;
+            final public const T_FLOAT   = 4;
+            final public const T_BOOL    = 8;
+
+            protected function getType(string|int|float|bool &$value): int
+            {
+                if ($value === 'y') {
+                    $value = true;
+
+                    return self::T_BOOL;
+                }
+
+                if (is_numeric($value)) {
+                    $value += 0;
+
+                    if (is_int($value)) {
+                        return self::T_INTEGER;
+                    }
+
+                    return self::T_FLOAT;
+                }
+
+                return self::T_NONE;
+            }
+
+            /** {@inheritDoc} */
+            protected function getCatchablePatterns(): array
+            {
+                return [
+                    '(?:[0-9]+)(?:[\.][0-9]+)?(?:e[+-]?[0-9]+)?',
+                    'y',
+                ];
+            }
+
+            /** {@inheritDoc} */
+            protected function getNonCatchablePatterns(): array
+            {
+                return ['\s+'];
+            }
+        };
+
+        $lexer->setInput('123.456');
+        $token = $lexer->peek();
+        assert($token !== null);
+        self::assertSame(123.456, $token->value, 'expect a real float, not a numerical string');
+
+        $lexer->setInput('y');
+        $token = $lexer->peek();
+        assert($token !== null);
+        self::assertTrue($token->value, 'expect a real bool, not a numerical string');
+    }
+}