Add FSC option for retrieving raw lexer tokens

laenas · laenas · commit 4ff375e0ff62 · 2020-12-18T23:00:06.000+01:00
diff --git a/src/fsharp/CompilerConfig.fs b/src/fsharp/CompilerConfig.fs
@@ -266,6 +266,11 @@ type LStatus =
     | Unprocessed
     | Processed
 
+type TokenizeOption =
+    | AndCompile
+    | Only
+    | Unfiltered
+
 type PackageManagerLine =
     { Directive: Directive
       LineStatus: LStatus
@@ -362,7 +367,7 @@ type TcConfigBuilder =
       mutable importAllReferencesOnly: bool
       mutable simulateException: string option
       mutable printAst: bool
-      mutable tokenizeOnly: bool
+      mutable tokenize: TokenizeOption
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -526,7 +531,7 @@ type TcConfigBuilder =
           importAllReferencesOnly = false
           simulateException = None
           printAst = false
-          tokenizeOnly = false
+          tokenize = TokenizeOption.AndCompile
           testInteractionParser = false
           reportNumDecls = false
           printSignature = false
@@ -923,7 +928,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) =
     member x.simulateException = data.simulateException
     member x.printAst = data.printAst
     member x.targetFrameworkVersion = targetFrameworkVersionValue
-    member x.tokenizeOnly = data.tokenizeOnly
+    member x.tokenize = data.tokenize
     member x.testInteractionParser = data.testInteractionParser
     member x.reportNumDecls = data.reportNumDecls
     member x.printSignature = data.printSignature
diff --git a/src/fsharp/CompilerConfig.fsi b/src/fsharp/CompilerConfig.fsi
@@ -119,6 +119,11 @@ type LStatus =
     | Unprocessed
     | Processed
 
+type TokenizeOption =
+    | AndCompile
+    | Only
+    | Unfiltered
+
 type PackageManagerLine =
     { Directive: Directive
       LineStatus: LStatus
@@ -180,7 +185,7 @@ type TcConfigBuilder =
       mutable importAllReferencesOnly: bool
       mutable simulateException: string option
       mutable printAst: bool
-      mutable tokenizeOnly: bool
+      mutable tokenize: TokenizeOption
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -358,7 +363,7 @@ type TcConfig =
     member importAllReferencesOnly: bool
     member simulateException: string option
     member printAst: bool
-    member tokenizeOnly: bool
+    member tokenize: TokenizeOption
     member testInteractionParser: bool
     member reportNumDecls: bool
     member printSignature: bool
diff --git a/src/fsharp/CompilerOptions.fs b/src/fsharp/CompilerOptions.fs
@@ -1137,8 +1137,13 @@ let internalFlags (tcConfigB:TcConfigBuilder) =
     
     CompilerOption
        ("tokenize", tagNone,
-        OptionUnit (fun () -> tcConfigB.tokenizeOnly <- true),
+        OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Only),
         Some(InternalCommandLineOption("--tokenize", rangeCmdArgs)), None)
+
+    CompilerOption
+       ("tokenize-unfiltered", tagNone,
+        OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Unfiltered),
+        Some(InternalCommandLineOption("--tokenize-unfiltered", rangeCmdArgs)), None)
     
     CompilerOption
        ("testInteractionParser", tagNone,
diff --git a/src/fsharp/ParseAndCheckInputs.fs b/src/fsharp/ParseAndCheckInputs.fs
@@ -286,21 +286,23 @@ let ParseInput (lexer, errorLogger: ErrorLogger, lexbuf: UnicodeLexing.Lexbuf, d
         let filteringErrorLogger = GetErrorLoggerFilteringByScopedPragmas(false, scopedPragmas, errorLogger)
         delayLogger.CommitDelayedDiagnostics filteringErrorLogger
 
+type Tokenizer = unit -> Parser.token
+
 // Show all tokens in the stream, for testing purposes
-let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
+let ShowAllTokensAndExit (shortFilename, tokenizer: Tokenizer, lexbuf: LexBuffer<char>) =
     while true do 
         printf "tokenize - getting one token from %s\n" shortFilename
-        let t = tokenizer.GetToken()
+        let t = tokenizer ()
         printf "tokenize - got %s @ %a\n" (Parser.token_to_string t) outputRange lexbuf.LexemeRange
         match t with
         | Parser.EOF _ -> exit 0 
         | _ -> ()
         if lexbuf.IsPastEndOfStream then printf "!!! at end of stream\n"
 
 // Test one of the parser entry points, just for testing purposes 
-let TestInteractionParserAndExit (tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
+let TestInteractionParserAndExit (tokenizer: Tokenizer, lexbuf: LexBuffer<char>) =
     while true do 
-        match (Parser.interaction (fun _ -> tokenizer.GetToken()) lexbuf) with
+        match (Parser.interaction (fun _ -> tokenizer ()) lexbuf) with
         | IDefns(l, m) -> printfn "Parsed OK, got %d defs @ %a" l.Length outputRange m
         | IHash (_, m) -> printfn "Parsed OK, got hash @ %a" outputRange m
     exit 0
@@ -341,18 +343,22 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp
             Lexhelp.usingLexbufForParsing (lexbuf, filename) (fun lexbuf ->
                 
                 // Set up the LexFilter over the token stream
-                let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf)
+                let tokenizer,tokenizeOnly =
+                    match tcConfig.tokenize with
+                    | Unfiltered -> (fun () -> Lexer.token lexargs skipWhitespaceTokens lexbuf), true
+                    | Only ->       LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, true
+                    | _ ->          LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, false
 
                 // If '--tokenize' then show the tokens now and exit
-                if tcConfig.tokenizeOnly then 
+                if tokenizeOnly then
                     ShowAllTokensAndExit(shortFilename, tokenizer, lexbuf)
 
                 // Test hook for one of the parser entry points
                 if tcConfig.testInteractionParser then 
                     TestInteractionParserAndExit (tokenizer, lexbuf)
 
                 // Parse the input
-                let res = ParseInput((fun _ -> tokenizer.GetToken()), errorLogger, lexbuf, None, filename, isLastCompiland)
+                let res = ParseInput((fun _ -> tokenizer ()), errorLogger, lexbuf, None, filename, isLastCompiland)
 
                 // Report the statistics for testing purposes
                 if tcConfig.reportNumDecls then 
