fix: guard against zero-width match loops in the lexer

alecthomas · claude · alecthomas · commit 34ba3a7de1bc · 2026-07-08T12:06:37.000+10:00
A rule matching the empty string without changing the state stack
made no progress and iterated forever (e.g. pattern a* at a
non-matching position). Such matches now emit an error token for
the current rune and advance, mirroring the no-match path. All
lexer golden tests are unchanged.

Co-Authored-By: Claude Fable 5 &lt;noreply@anthropic.com&gt;
diff --git a/regexp.go b/regexp.go
@@ -249,14 +249,24 @@ func (l *LexerState) Iterator() iter.Seq[Token] { // nolint: gocognit
 			l.Groups = groups
 			l.NamedGroups = namedGroups
 			l.Pos += utf8.RuneCountInString(groups[0])
+			progressed := groups[0] != ""
 			if rule.Mutator != nil {
+				depth, top := len(l.Stack), l.State
 				if err := rule.Mutator.Mutate(l); err != nil {
 					panic(err)
 				}
+				progressed = progressed || len(l.Stack) != depth || len(l.Stack) == 0 || l.Stack[len(l.Stack)-1] != top
 			}
 			if rule.Type != nil {
 				l.pushIterator(rule.Type.Emit(l.Groups, l))
 			}
+			if !progressed {
+				// A zero-width match that did not change state will never advance.
+				l.Pos++
+				if !yield(Token{Error, string(l.Text[l.Pos-1 : l.Pos])}) {
+					return
+				}
+			}
 		}
 		if !l.drainIteratorStack(yield) {
 			return
diff --git a/regexp_test.go b/regexp_test.go
@@ -54,6 +54,17 @@ func TestMatchingAtStart(t *testing.T) {
 		slices.Collect(it))
 }
 
+func TestZeroWidthMatchTerminates(t *testing.T) {
+	l := Coalesce(mustNewLexer(t, &Config{}, Rules{ // nolint: forbidigo
+		"root": {
+			{`a*`, Text, nil},
+		},
+	}))
+	it, err := l.Tokenise(nil, "b")
+	assert.NoError(t, err)
+	assert.Equal(t, []Token{{Error, "b"}}, slices.Collect(it))
+}
+
 func TestEnsureLFOption(t *testing.T) {
 	l := Coalesce(mustNewLexer(t, &Config{}, Rules{ // nolint: forbidigo
 		"root": {
