Only slice once, and tokenise leading * as whitespace

sandersn · sandersn · commit 794b0f39fbcb · 2023-03-07T09:26:34.000-08:00
From my other experiment. Maybe together they'll be fast!
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -2161,6 +2161,9 @@ namespace Parser {
         return nextTokenWithoutCheck();
     }
 
+    function nextTokenJSDocInitialIndent(): JSDocSyntaxKind {
+        return currentToken = scanner.scanJSDocInitialIndent();
+    }
     function nextTokenJSDoc(): JSDocSyntaxKind {
         return currentToken = scanner.scanJsDocToken();
     }
@@ -8641,7 +8644,7 @@ namespace Parser {
                                 comments.push(scanner.getTokenText());
                             }
                             else if (margin !== undefined && indent + whitespaceLength > margin) {
-                                comments.push(scanner.getTokenText().slice(margin - indent));
+                                comments.push(scanner.getText().slice(scanner.getTokenPos() + margin - indent, scanner.getTextPos()));
                             }
                             indent += whitespaceLength;
                             break;
@@ -8671,7 +8674,12 @@ namespace Parser {
                             pushComment(scanner.getTokenText());
                             break;
                     }
-                    nextTokenJSDoc();
+                    if (state === JSDocState.BeginningOfLine) {
+                        nextTokenJSDocInitialIndent();
+                    }
+                    else {
+                        nextTokenJSDoc();
+                    }
                 }
                 removeTrailingWhitespace(comments);
                 if (parts.length && comments.length) {
@@ -8891,7 +8899,7 @@ namespace Parser {
                                 const whitespaceLength = scanner.getTextPos() - scanner.getTokenPos();
                                 // if the whitespace crosses the margin, take only the whitespace that passes the margin
                                 if (margin !== undefined && indent + whitespaceLength > margin) {
-                                    comments.push(scanner.getTokenText().slice(margin - indent));
+                                    comments.push(scanner.getText().slice(scanner.getTokenPos() + margin - indent, scanner.getTextPos()));
                                 }
                                 indent += whitespaceLength;
                             }
@@ -8937,7 +8945,12 @@ namespace Parser {
                             break;
                     }
                     previousWhitespace = token() === SyntaxKind.WhitespaceTrivia;
-                    tok = nextTokenJSDoc();
+                    if (state === JSDocState.BeginningOfLine) {
+                        tok = nextTokenJSDocInitialIndent();
+                    }
+                    else {
+                        tok = nextTokenJSDoc();
+                    }
                 }
 
                 removeLeadingNewlines(comments);
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -74,6 +74,8 @@ export interface Scanner {
     reScanQuestionToken(): SyntaxKind;
     reScanInvalidIdentifier(): SyntaxKind;
     scanJsxToken(): JsxTokenSyntaxKind;
+    /** @internal */
+    scanJSDocInitialIndent(): JSDocSyntaxKind;
     scanJsDocToken(): JSDocSyntaxKind;
     scan(): SyntaxKind;
 
@@ -1019,6 +1021,7 @@ export function createScanner(languageVersion: ScriptTarget,
         reScanQuestionToken,
         reScanInvalidIdentifier,
         scanJsxToken,
+        scanJSDocInitialIndent,
         scanJsDocToken,
         scan,
         getText,
@@ -2455,6 +2458,22 @@ export function createScanner(languageVersion: ScriptTarget,
         return scanJsxAttributeValue();
     }
 
+    function scanJSDocInitialIndent(): JSDocSyntaxKind {
+        startPos = tokenPos = pos;
+        tokenFlags = TokenFlags.None;
+        if (pos >= end) {
+            return token = SyntaxKind.EndOfFileToken;
+        }
+        for (let ch = codePointAt(text, pos); // TODO: Only allow 1 (one) asterisk!
+             pos < end && (isWhiteSpaceSingleLine(ch) || ch === CharacterCodes.asterisk);
+             ch = codePointAt(text, pos += charSize(ch))) {
+        }
+        if (pos === tokenPos) {
+            return scanJsDocToken();
+        }
+        return token = SyntaxKind.WhitespaceTrivia;
+    }
+
     function scanJsDocToken(): JSDocSyntaxKind {
         startPos = tokenPos = pos;
         tokenFlags = TokenFlags.None;

Original file line number	Diff line number	Diff line change
`@@ -2161,6 +2161,9 @@ namespace Parser {`
`2161`	`2161`	`return nextTokenWithoutCheck();`
`2162`	`2162`	`}`
`2163`	`2163`
	`2164`	`+ function nextTokenJSDocInitialIndent(): JSDocSyntaxKind {`
	`2165`	`+ return currentToken = scanner.scanJSDocInitialIndent();`
	`2166`	`+ }`
`2164`	`2167`	`function nextTokenJSDoc(): JSDocSyntaxKind {`
`2165`	`2168`	`return currentToken = scanner.scanJsDocToken();`
`2166`	`2169`	`}`
`@@ -8641,7 +8644,7 @@ namespace Parser {`
`8641`	`8644`	`comments.push(scanner.getTokenText());`
`8642`	`8645`	`}`
`8643`	`8646`	`else if (margin !== undefined && indent + whitespaceLength > margin) {`
`8644`		`- comments.push(scanner.getTokenText().slice(margin - indent));`
	`8647`	`+ comments.push(scanner.getText().slice(scanner.getTokenPos() + margin - indent, scanner.getTextPos()));`
`8645`	`8648`	`}`
`8646`	`8649`	`indent += whitespaceLength;`
`8647`	`8650`	`break;`
`@@ -8671,7 +8674,12 @@ namespace Parser {`
`8671`	`8674`	`pushComment(scanner.getTokenText());`
`8672`	`8675`	`break;`
`8673`	`8676`	`}`
`8674`		`- nextTokenJSDoc();`
	`8677`	`+ if (state === JSDocState.BeginningOfLine) {`
	`8678`	`+ nextTokenJSDocInitialIndent();`
	`8679`	`+ }`
	`8680`	`+ else {`
	`8681`	`+ nextTokenJSDoc();`
	`8682`	`+ }`
`8675`	`8683`	`}`
`8676`	`8684`	`removeTrailingWhitespace(comments);`
`8677`	`8685`	`if (parts.length && comments.length) {`
`@@ -8891,7 +8899,7 @@ namespace Parser {`
`8891`	`8899`	`const whitespaceLength = scanner.getTextPos() - scanner.getTokenPos();`
`8892`	`8900`	`// if the whitespace crosses the margin, take only the whitespace that passes the margin`
`8893`	`8901`	`if (margin !== undefined && indent + whitespaceLength > margin) {`
`8894`		`- comments.push(scanner.getTokenText().slice(margin - indent));`
	`8902`	`+ comments.push(scanner.getText().slice(scanner.getTokenPos() + margin - indent, scanner.getTextPos()));`
`8895`	`8903`	`}`
`8896`	`8904`	`indent += whitespaceLength;`
`8897`	`8905`	`}`
`@@ -8937,7 +8945,12 @@ namespace Parser {`
`8937`	`8945`	`break;`
`8938`	`8946`	`}`
`8939`	`8947`	`previousWhitespace = token() === SyntaxKind.WhitespaceTrivia;`
`8940`		`- tok = nextTokenJSDoc();`
	`8948`	`+ if (state === JSDocState.BeginningOfLine) {`
	`8949`	`+ tok = nextTokenJSDocInitialIndent();`
	`8950`	`+ }`
	`8951`	`+ else {`
	`8952`	`+ tok = nextTokenJSDoc();`
	`8953`	`+ }`
`8941`	`8954`	`}`
`8942`	`8955`
`8943`	`8956`	`removeLeadingNewlines(comments);`