diff --git a/src/AbstractLexer.php b/src/AbstractLexer.php index 260ace4..20988d2 100644 --- a/src/AbstractLexer.php +++ b/src/AbstractLexer.php @@ -7,6 +7,7 @@ namespace Doctrine\Common\Lexer; use ReflectionClass; use UnitEnum; +use function count; use function implode; use function preg_split; use function sprintf; @@ -112,13 +113,31 @@ abstract class AbstractLexer /** * Resets the lexer position on the input to the given position. * - * @param int $position Position to place the lexical scanner. + * @param int $position Character offset into the input, as found in + * Token::$position; not an index in the token list. * * @return void */ public function resetPosition(int $position = 0) { - $this->position = $position; + $this->position = $this->findTokenIndexAtOrAfterOffset($position); + } + + /** + * Finds the index of the first token at or after the given offset. + * + * Token offsets are strictly increasing, so a linear scan suffices. + * Returns one past the end when there is no such token. + */ + private function findTokenIndexAtOrAfterOffset(int $offset): int + { + foreach ($this->tokens as $index => $token) { + if ($token->position >= $offset) { + return $index; + } + } + + return count($this->tokens); } /** diff --git a/tests/AbstractLexerTest.php b/tests/AbstractLexerTest.php index 0c00f66..8b9d12e 100644 --- a/tests/AbstractLexerTest.php +++ b/tests/AbstractLexerTest.php @@ -86,6 +86,39 @@ class AbstractLexerTest extends TestCase $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead); } + /** + * @see https://github.com/doctrine/lexer/issues/53 + * + * resetPosition() takes a character offset, not a token index. The input + * below has multi-character tokens, so the second token's offset (6) + * differs from its index (1). + */ + public function testResetPositionByTokenPosition(): void + { + $this->concreteLexer->setInput('price=10'); + + $this->assertTrue($this->concreteLexer->moveNext()); + $this->assertTrue($this->concreteLexer->moveNext()); + + $target = $this->concreteLexer->lookahead; + $this->assertSame('=', $target->value); + $this->assertSame(5, $target->position); + + // Move the cursor past the index that used to coincide with the target. + $remainingTokens = 0; + while ($this->concreteLexer->moveNext()) { + $remainingTokens++; + } + + $this->assertSame(1, $remainingTokens); + $this->assertNull($this->concreteLexer->lookahead); + + $this->concreteLexer->resetPosition($target->position); + + $this->assertTrue($this->concreteLexer->moveNext()); + $this->assertEquals($target, $this->concreteLexer->lookahead); + } + /** @phpstan-param list> $expectedTokens */ #[DataProvider('dataProvider')] public function testMoveNext(string $input, array $expectedTokens): void