fix: make resetPosition() accept a token character offset

resetPosition() used its argument directly as an index into the
internal tokens array. The only position value the public API
exposes is Token::$position, a character offset into the input
string, so passing a multi-character token's offset silently
resumed the lexer at the wrong token, or past the end of input.

resetPosition() now locates the token whose character offset is
greater than or equal to the given position and resumes from there,
matching the offset semantics already used by getInputUntilPosition().

Closes #53

src/AbstractLexer.php

@@ -7,6 +7,7 @@ namespace Doctrine\Common\Lexer;
 use ReflectionClass;
 use UnitEnum;
 
+use function count;
 use function implode;
 use function preg_split;
 use function sprintf;
@@ -112,13 +113,31 @@ abstract class AbstractLexer
     /**
      * Resets the lexer position on the input to the given position.
      *
-     * @param int $position Position to place the lexical scanner.
+     * @param int $position Character offset into the input, as found in
+     *                      Token::$position; not an index in the token list.
      *
      * @return void
      */
     public function resetPosition(int $position = 0)
     {
-        $this->position = $position;
+        $this->position = $this->findTokenIndexAtOrAfterOffset($position);
+    }
+
+    /**
+     * Finds the index of the first token at or after the given offset.
+     *
+     * Token offsets are strictly increasing, so a linear scan suffices.
+     * Returns one past the end when there is no such token.
+     */
+    private function findTokenIndexAtOrAfterOffset(int $offset): int
+    {
+        foreach ($this->tokens as $index => $token) {
+            if ($token->position >= $offset) {
+                return $index;
+            }
+        }
+
+        return count($this->tokens);
     }
 
     /**

tests/AbstractLexerTest.php

@@ -86,6 +86,39 @@ class AbstractLexerTest extends TestCase
         $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead);
     }
 
+    /**
+     * @see https://github.com/doctrine/lexer/issues/53
+     *
+     * resetPosition() takes a character offset, not a token index. The input
+     * below has multi-character tokens, so the second token's offset (6)
+     * differs from its index (1).
+     */
+    public function testResetPositionByTokenPosition(): void
+    {
+        $this->concreteLexer->setInput('price=10');
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->assertTrue($this->concreteLexer->moveNext());
+
+        $target = $this->concreteLexer->lookahead;
+        $this->assertSame('=', $target->value);
+        $this->assertSame(5, $target->position);
+
+        // Move the cursor past the index that used to coincide with the target.
+        $remainingTokens = 0;
+        while ($this->concreteLexer->moveNext()) {
+            $remainingTokens++;
+        }
+
+        $this->assertSame(1, $remainingTokens);
+        $this->assertNull($this->concreteLexer->lookahead);
+
+        $this->concreteLexer->resetPosition($target->position);
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+        $this->assertEquals($target, $this->concreteLexer->lookahead);
+    }
+
     /** @phpstan-param list<Token<string, string|int>>  $expectedTokens */
     #[DataProvider('dataProvider')]
     public function testMoveNext(string $input, array $expectedTokens): void