diff --git a/src/absil/illex.fsl b/src/absil/illex.fsl index 849ebd17b8e..e1f923c3582 100644 --- a/src/absil/illex.fsl +++ b/src/absil/illex.fsl @@ -16,7 +16,7 @@ open FSharp.Compiler.AbstractIL.Internal.AsciiParser open FSharp.Compiler.AbstractIL.Internal.AsciiConstants -let lexeme (lexbuf : LexBuffer) = new System.String(lexbuf.Lexeme) +let lexeme (lexbuf : LexBuffer<_>) = LexBuffer<_>.LexemeString (lexbuf) let unexpectedChar _lexbuf = raise Parsing.RecoverableParseError ;; diff --git a/src/absil/illib.fs b/src/absil/illib.fs index 68b7c8d40a8..c4ded950d0d 100644 --- a/src/absil/illib.fs +++ b/src/absil/illib.fs @@ -571,6 +571,9 @@ module String = else None let getLines (str: string) = +#if FABLE_COMPILER + System.Text.RegularExpressions.Regex.Split(str, "\r\n|\r|\n"); +#else use reader = new StringReader(str) [| let line = ref (reader.ReadLine()) diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index d34092940e2..63ebe2f2b35 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -13,12 +13,16 @@ open System.IO open Internal.Utilities.Text.Lexing -type Lexbuf = LexBuffer +type Lexbuf = LexBuffer let StringAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, s:string) : Lexbuf = +#if FABLE_COMPILER + LexBuffer<_>.FromString (supportsFeature, s) +#else LexBuffer<_>.FromChars (supportsFeature, s.ToCharArray()) +#endif -let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferFiller: char[] * int * int -> int) : Lexbuf = +let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferFiller: LexBufferChar[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction(supportsFeature, bufferFiller) let SourceTextAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, sourceText) = @@ -69,5 +73,9 @@ let UnicodeFileAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, file else reraise() let source = getSource 0 +#if FABLE_COMPILER + let lexbuf = LexBuffer<_>.FromString (supportsFeature, source) +#else let lexbuf = LexBuffer<_>.FromChars(supportsFeature, source.ToCharArray()) +#endif lexbuf diff --git a/src/fsharp/UnicodeLexing.fsi b/src/fsharp/UnicodeLexing.fsi index 2478c7f857b..a6947f38f5b 100644 --- a/src/fsharp/UnicodeLexing.fsi +++ b/src/fsharp/UnicodeLexing.fsi @@ -7,8 +7,8 @@ open FSharp.Compiler.Text open Microsoft.FSharp.Text open Internal.Utilities.Text.Lexing -type Lexbuf = LexBuffer +type Lexbuf = LexBuffer val internal StringAsLexbuf: (Features.LanguageFeature -> bool) * string -> Lexbuf -val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (char [] * int * int -> int) -> Lexbuf +val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (LexBufferChar[] * int * int -> int) -> Lexbuf val public UnicodeFileAsLexbuf: (Features.LanguageFeature -> bool) * string * int option * (*retryLocked*) bool -> Lexbuf val public SourceTextAsLexbuf: (Features.LanguageFeature -> bool) * ISourceText -> Lexbuf diff --git a/src/fsharp/fsi/fsi.fs b/src/fsharp/fsi/fsi.fs index 99bffe0b73c..3be860c8d81 100644 --- a/src/fsharp/fsi/fsi.fs +++ b/src/fsharp/fsi/fsi.fs @@ -1619,7 +1619,7 @@ type internal FsiStdinLexerProvider let LexbufFromLineReader (fsiStdinSyphon: FsiStdinSyphon) readf = UnicodeLexing.FunctionAsLexbuf - (isFeatureSupported, (fun (buf: char[], start, len) -> + (isFeatureSupported, (fun (buf, start, len) -> //fprintf fsiConsoleOutput.Out "Calling ReadLine\n" let inputOption = try Some(readf()) with :? EndOfStreamException -> None inputOption |> Option.iter (fun t -> fsiStdinSyphon.Add (t + "\n")) @@ -1633,7 +1633,11 @@ type internal FsiStdinLexerProvider if ninput > len then fprintf fsiConsoleOutput.Error "%s" (FSIstrings.SR.fsiLineTooLong()) let ntrimmed = min len ninput for i = 0 to ntrimmed-1 do +#if FABLE_COMPILER + buf.[i+start] <- uint16 input.[i] +#else buf.[i+start] <- input.[i] +#endif ntrimmed )) diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index a3a0003f57e..b77ea661acf 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -147,7 +147,11 @@ let shouldStartFile args lexbuf (m:range) err tok = else tok let evalIfDefExpression startPos isFeatureSupported args (lookup:string->bool) (lexed:string) = +#if FABLE_COMPILER + let lexbuf = LexBuffer<_>.FromString (isFeatureSupported, lexed) +#else let lexbuf = LexBuffer.FromChars (isFeatureSupported, lexed.ToCharArray ()) +#endif lexbuf.StartPos <- startPos lexbuf.EndPos <- startPos let tokenStream = FSharp.Compiler.PPLexer.tokenstream args diff --git a/src/fsharp/service/ServiceLexing.fsi b/src/fsharp/service/ServiceLexing.fsi index 85e18f171c4..ee502ffbdfc 100755 --- a/src/fsharp/service/ServiceLexing.fsi +++ b/src/fsharp/service/ServiceLexing.fsi @@ -3,6 +3,7 @@ namespace FSharp.Compiler.SourceCodeServices open FSharp.Compiler +open Internal.Utilities.Text.Lexing type Position = int * int type Range = Position * Position @@ -237,7 +238,7 @@ type FSharpLineTokenizer = type FSharpSourceTokenizer = new : conditionalDefines:string list * fileName:string option -> FSharpSourceTokenizer member CreateLineTokenizer : lineText:string -> FSharpLineTokenizer - member CreateBufferTokenizer : bufferFiller:(char[] * int * int -> int) -> FSharpLineTokenizer + member CreateBufferTokenizer : bufferFiller:(LexBufferChar[] * int * int -> int) -> FSharpLineTokenizer module internal TestExpose = val TokenInfo : Parser.token -> (FSharpTokenColorKind * FSharpTokenCharKind * FSharpTokenTriggerClass) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 1f772d6e87e..8d30b3cd793 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -31,6 +31,9 @@ type ISourceText = type StringText(str: string) = let getLines (str: string) = +#if FABLE_COMPILER + System.Text.RegularExpressions.Regex.Split(str, "\r\n|\r|\n"); +#else use reader = new StringReader(str) [| let mutable line = reader.ReadLine() @@ -42,6 +45,7 @@ type StringText(str: string) = // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak yield String.Empty |] +#endif let getLines = // This requires allocating and getting all the lines. @@ -169,6 +173,12 @@ namespace Internal.Utilities.Text.Lexing 0, 0) +#if FABLE_COMPILER + type internal LexBufferChar = uint16 +#else + type internal LexBufferChar = char +#endif + type internal LexBufferFiller<'Char> = (LexBuffer<'Char> -> unit) and [] @@ -217,6 +227,8 @@ namespace Internal.Utilities.Text.Lexing and set b = endPos <- b member lexbuf.Lexeme = Array.sub buffer bufferScanStart lexemeLength + member lexbuf.LexemeChar(n) = buffer.[n+bufferScanStart] + member lexbuf.BufferLocalStore = (context :> IDictionary<_,_>) member lexbuf.LexemeLength with get() : int = lexemeLength and set v = lexemeLength <- v member lexbuf.Buffer with get() : 'Char[] = buffer and set v = buffer <- v @@ -225,8 +237,14 @@ namespace Internal.Utilities.Text.Lexing member lexbuf.BufferScanStart with get() : int = bufferScanStart and set v = bufferScanStart <- v member lexbuf.BufferAcceptAction with get() = bufferAcceptAction and set v = bufferAcceptAction <- v member lexbuf.RefillBuffer () = filler lexbuf - static member LexemeString(lexbuf:LexBuffer) = - new System.String(lexbuf.Buffer,lexbuf.BufferScanStart,lexbuf.LexemeLength) + + static member LexemeString (lexbuf: LexBuffer) = +#if FABLE_COMPILER + let chars = Array.init lexbuf.LexemeLength (lexbuf.LexemeChar >> char) + new System.String(chars) +#else + new System.String(lexbuf.Buffer, lexbuf.BufferScanStart, lexbuf.LexemeLength) +#endif member lexbuf.IsPastEndOfStream with get() = eof @@ -266,9 +284,13 @@ namespace Internal.Utilities.Text.Lexing LexBuffer<'Char>.FromArrayNoCopy(supportsFeature, buffer) // Important: This method takes ownership of the array - static member FromChars (supportsFeature:LanguageFeature -> bool, arr:char[]) = LexBuffer.FromArrayNoCopy (supportsFeature, arr) + static member FromChars (supportsFeature:LanguageFeature -> bool, arr:LexBufferChar[]) = LexBuffer.FromArrayNoCopy (supportsFeature, arr) static member FromSourceText (supportsFeature: LanguageFeature -> bool, sourceText: ISourceText) = +#if FABLE_COMPILER + let arr = Array.init sourceText.Length (fun i -> uint16 (sourceText.Item i)) + LexBuffer.FromArrayNoCopy (supportsFeature, arr) +#else let mutable currentSourceIndex = 0 LexBuffer.FromFunction(supportsFeature, fun (chars, start, length) -> let lengthToCopy = @@ -283,16 +305,25 @@ namespace Internal.Utilities.Text.Lexing currentSourceIndex <- currentSourceIndex + lengthToCopy lengthToCopy ) +#endif + + static member FromString (supportsFeature: LanguageFeature -> bool, s: string) = +#if FABLE_COMPILER + let arr = Array.init s.Length (fun i -> uint16 s.[i]) + LexBuffer.FromArrayNoCopy (supportsFeature, arr) +#else + LexBuffer.FromArrayNoCopy (supportsFeature, s.ToCharArray()) +#endif module GenericImplFragments = - let startInterpret(lexBuffer:LexBuffer) = + let startInterpret(lexBuffer:LexBuffer) = lexBuffer.BufferScanStart <- lexBuffer.BufferScanStart + lexBuffer.LexemeLength; lexBuffer.BufferMaxScanLength <- lexBuffer.BufferMaxScanLength - lexBuffer.LexemeLength; lexBuffer.BufferScanLength <- 0; lexBuffer.LexemeLength <- 0; lexBuffer.BufferAcceptAction <- -1; - let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer,scanUntilSentinel,endOfScan,state,eofPos) = + let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer,scanUntilSentinel,endOfScan,state,eofPos) = // end of file occurs if we couldn't extend the buffer if lexBuffer.BufferScanLength = lexBuffer.BufferMaxScanLength then let snew = int trans.[state].[eofPos] // == EOF @@ -306,9 +337,9 @@ namespace Internal.Utilities.Text.Lexing else scanUntilSentinel lexBuffer state - let onAccept (lexBuffer:LexBuffer,a) = - lexBuffer.LexemeLength <- lexBuffer.BufferScanLength; - lexBuffer.BufferAcceptAction <- a; + let onAccept (lexBuffer:LexBuffer, a) = + lexBuffer.LexemeLength <- lexBuffer.BufferScanLength + lexBuffer.BufferAcceptAction <- a open GenericImplFragments @@ -333,12 +364,16 @@ namespace Internal.Utilities.Text.Lexing // ways let baseForUnicodeCategories = numLowUnicodeChars+numSpecificUnicodeChars*2 let unicodeCategory = +#if FABLE_COMPILER + System.Char.GetUnicodeCategory(char inp) +#else System.Char.GetUnicodeCategory(inp) +#endif //System.Console.WriteLine("inp = {0}, unicodeCategory = {1}", [| box inp; box unicodeCategory |]); int trans.[state].[baseForUnicodeCategories + int32 unicodeCategory] else // This is the specific unicode character - let c = char (int trans.[state].[baseForSpecificUnicodeChars+i*2]) + let c = trans.[state].[baseForSpecificUnicodeChars+i*2] //System.Console.WriteLine("c = {0}, inp = {1}, i = {2}", [| box c; box inp; box i |]); // OK, have we found the entry for a specific unicode character? if c = inp @@ -360,7 +395,7 @@ namespace Internal.Utilities.Text.Lexing afterRefill (trans,sentinel,lexBuffer,scanUntilSentinel,lexBuffer.EndOfScan,state,eofPos) else // read a character - end the scan if there are no further transitions - let inp = lexBuffer.Buffer.[lexBuffer.BufferScanPos] + let inp = uint16 lexBuffer.Buffer.[lexBuffer.BufferScanPos] // Find the new state let snew = lookupUnicodeCharacters state inp @@ -378,7 +413,7 @@ namespace Internal.Utilities.Text.Lexing // 30 entries, one for each UnicodeCategory // 1 entry for EOF - member tables.Interpret(initialState,lexBuffer : LexBuffer) = + member tables.Interpret(initialState, lexBuffer: LexBuffer) = startInterpret(lexBuffer) scanUntilSentinel lexBuffer initialState diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index bf9eb3171c1..4abd9e79ae4 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -85,6 +85,12 @@ type internal Position = static member FirstLine : fileIdx:int -> Position +#if FABLE_COMPILER +type internal LexBufferChar = uint16 +#else +type internal LexBufferChar = char +#endif + [] /// Input buffers consumed by lexers generated by fslex.exe. /// The type must be generic to match the code generated by FsLex and FsYacc (if you would like to @@ -100,7 +106,7 @@ type internal LexBuffer<'Char> = member Lexeme: 'Char [] /// Fast helper to turn the matched characters into a string, avoiding an intermediate array. - static member LexemeString : LexBuffer -> string + static member LexemeString : LexBuffer -> string /// Dynamically typed, non-lexically scoped parameter table. member BufferLocalStore : IDictionary @@ -113,13 +119,16 @@ type internal LexBuffer<'Char> = /// Create a lex buffer suitable for Unicode lexing that reads characters from the given array. /// Important: does take ownership of the array. - static member FromChars: (LanguageFeature -> bool) * char[] -> LexBuffer + static member FromChars: (LanguageFeature -> bool) * LexBufferChar[] -> LexBuffer + + /// Create a lex buffer suitable for Unicode lexing that reads characters from the given string. + static member FromString: (LanguageFeature -> bool) * string -> LexBuffer /// Create a lex buffer that reads character or byte inputs by using the given function. static member FromFunction: (LanguageFeature -> bool) * ('Char[] * int * int -> int) -> LexBuffer<'Char> /// Create a lex buffer backed by source text. - static member FromSourceText : (LanguageFeature -> bool) * ISourceText -> LexBuffer + static member FromSourceText : (LanguageFeature -> bool) * ISourceText -> LexBuffer /// The type of tables for an unicode lexer generated by fslex.exe. [] @@ -129,5 +138,5 @@ type internal UnicodeTables = static member Create : uint16[][] * uint16[] -> UnicodeTables /// Interpret tables for a unicode lexer generated by fslex.exe. - member Interpret: initialState:int * LexBuffer -> int + member Interpret: initialState:int * LexBuffer -> int diff --git a/src/utils/prim-parsing.fs b/src/utils/prim-parsing.fs index 9ce01bff907..4cbfee630b6 100644 --- a/src/utils/prim-parsing.fs +++ b/src/utils/prim-parsing.fs @@ -14,7 +14,7 @@ exception RecoverableParseError exception Accept of obj [] -type internal IParseState(ruleStartPoss:Position[], ruleEndPoss:Position[], lhsPos:Position[], ruleValues:obj[], lexbuf:LexBuffer) = +type internal IParseState(ruleStartPoss:Position[], ruleEndPoss:Position[], lhsPos:Position[], ruleValues:obj[], lexbuf:LexBuffer) = member p.LexBuffer = lexbuf member p.InputRange n = ruleStartPoss.[n-1], ruleEndPoss.[n-1] diff --git a/src/utils/prim-parsing.fsi b/src/utils/prim-parsing.fsi index 7ec3e7a0a80..762c90796ab 100644 --- a/src/utils/prim-parsing.fsi +++ b/src/utils/prim-parsing.fsi @@ -37,7 +37,7 @@ type internal IParseState = member RaiseError<'b> : unit -> 'b /// Return the LexBuffer for this parser instance. - member LexBuffer : LexBuffer + member LexBuffer : LexBuffer [] @@ -118,7 +118,7 @@ type internal Tables<'tok> = /// Interpret the parser table taking input from the given lexer, using the given lex buffer, and the given start state. /// Returns an object indicating the final synthesized value for the parse. - member Interpret : lexer:(LexBuffer -> 'tok) * lexbuf:LexBuffer * startState:int -> obj + member Interpret : lexer:(LexBuffer -> 'tok) * lexbuf:LexBuffer * startState:int -> obj /// Indicates an accept action has occurred. exception internal Accept of obj