nullness implementation squashed

DEVGUIDE.md

@@ -113,6 +113,17 @@ If your changes involve modifying the list of language keywords in any way, (e.g
 
 This only works on Windows/.NETStandard framework, so changing this from any other platform requires editing and syncing all of the XLF files manually.
 
+## Updating baselines in tests
+
+Some tests use "baseline" files.  There is sometimes a way to update these baselines en-masse in your local build,
+useful when some change affects many baselines.  For example, in the 'fsharpqa' tests the baselines
+are updated using scripts or utilities that allow the following environment variable to be set:
+
+```
+set TEST_UPDATE_BSL=1
+```
+
+
 ## Developing the F# tools for Visual Studio
 
 As you would expect, doing this requires both Windows and Visual Studio are installed.

FSharp.Profiles.props

@@ -1,6 +1,9 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!-- Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information. -->
 <Project>
+  <PropertyGroup Condition="'$(Configuration)' == 'Proto'">
+    <DefineConstants>BUILDING_WITH_LKG;$(DefineConstants)</DefineConstants>
+  </PropertyGroup>
 
   <PropertyGroup Condition="$(TargetFramework.StartsWith('net4'))">
     <DefineConstants Condition="'$(MonoPackaging)' == 'true'">$(DefineConstants);CROSS_PLATFORM_COMPILER</DefineConstants>

fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj

@@ -11,6 +11,7 @@
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies> <!-- Tests won't run without this, at least on OSX, see https://github.com/NuGet/Home/issues/4837#issuecomment-354536302 -->
+    <DefineConstants>$(DefineConstants);USES_FSHARP_CORE_45_PACKAGE</DefineConstants>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="$(FSharpSourcesRoot)\..\tests\service\FsUnit.fs">

fcs/FSharp.Compiler.Service/AssemblyInfo.fs

@@ -53,10 +53,10 @@ open System.Runtime.InteropServices
 [<assembly:System.Runtime.CompilerServices.InternalsVisibleTo("FSharp.Compiler.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010077d32e043d184cf8cebf177201ec6fad091581a3a639a0534f1c4ebb3ab847a6b6636990224a04cf4bd1aec51ecec44cf0c8922eb5bb2ee65ec3fb9baa87e141042c96ce414f98af33508c7e24dab5b068aa802f6693881537ee0efcb5d3f1c9aaf8215ac42e92ba9a5a02574d6890d07464cb2f338b043b1c4ffe98efe069ee")>]
 
 // Until dotnet sdk can version assemblies, use this
-#if BUILD_FROM_SOURCE
-[<assembly: System.Reflection.AssemblyInformationalVersion("4.4.1.0")>]
-[<assembly: System.Reflection.AssemblyVersion("4.4.1.0")>]
-[<assembly: System.Reflection.AssemblyFileVersion("2017.06.27.0")>]
-#endif
+//#if BUILD_FROM_SOURCE
+//[<assembly: System.Reflection.AssemblyInformationalVersion("4.4.1.0")>]
+//[<assembly: System.Reflection.AssemblyVersion("4.4.1.0")>]
+//[<assembly: System.Reflection.AssemblyFileVersion("2017.06.27.0")>]
+//#endif
 
 do()

fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -7,6 +7,7 @@
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <DefineConstants>$(DefineConstants);COMPILER_SERVICE_AS_DLL</DefineConstants>
     <DefineConstants>$(DefineConstants);COMPILER</DefineConstants>
+    <DefineConstants>$(DefineConstants);BUILD_FROM_SOURCE</DefineConstants>
     <DefineConstants>$(DefineConstants);ENABLE_MONO_SUPPORT</DefineConstants>
     <DefineConstants>$(DefineConstants);NO_STRONG_NAMES</DefineConstants>
     <DefineConstants>$(DefineConstants);LOCALIZATION_FSCOMP</DefineConstants>

src/absil/bytes.fs

@@ -391,10 +391,15 @@ type ByteMemory with
     static member FromArray bytes =
         ByteArrayMemory.FromArray(bytes, 0, bytes.Length)
 
+    static member Empty with get() = ByteMemory.FromArray [| |]
+
 type internal ByteStream = 
     { bytes: ReadOnlyByteMemory
       mutable pos: int 
       max: int }
+
+    member b.IsEOF = (b.pos >= b.max)
+
     member b.ReadByte() = 
         if b.pos >= b.max then failwith "end of stream"
         let res = b.bytes.[b.pos]

src/absil/bytes.fsi

@@ -104,6 +104,9 @@ type ByteMemory with
     /// Creates a ByteMemory object that is backed by a byte array.
     static member FromArray: bytes: byte[] -> ByteMemory
 
+    /// Gets a ByteMemory object that is empty
+    static member Empty: ByteMemory
+
 /// Imperative buffers and streams of byte[]
 [<Sealed>]
 type internal ByteBuffer = 
@@ -125,6 +128,7 @@ type internal ByteBuffer =
 
 [<Sealed>]
 type internal ByteStream =
+    member IsEOF : bool
     member ReadByte : unit -> byte
     member ReadBytes : int -> ReadOnlyByteMemory
     member ReadUtf8String : int -> string

src/absil/illib.fs

@@ -20,6 +20,11 @@ let notlazy v = Lazy<_>.CreateFromValue v
 
 let inline isNil l = List.isEmpty l
 
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
+let inline (|NonNull|) x = match x with null -> raise (NullReferenceException()) | v -> v
+let inline nonNull<'T> (x: 'T) = x
+#endif
+
 /// Returns true if the list has less than 2 elements. Otherwise false.
 let inline isNilOrSingleton l =
     match l with
@@ -34,9 +39,6 @@ let inline isSingleton l =
     | _ -> false
 
 let inline isNonNull x = not (isNull x)
-
-let inline nonNull msg x = if isNull x then failwith ("null: " + msg) else x
-
 let inline (===) x y = LanguagePrimitives.PhysicalEquality x y
 
 /// Per the docs the threshold for the Large Object Heap is 85000 bytes: https://docs.microsoft.com/en-us/dotnet/standard/garbage-collection/large-object-heap#how-an-object-ends-up-on-the-large-object-heap-and-how-gc-handles-them
@@ -66,14 +68,17 @@ let reportTime =
 type InlineDelayInit<'T when 'T : not struct> = 
     new (f: unit -> 'T) = {store = Unchecked.defaultof<'T>; func = Func<_>(f) } 
     val mutable store : 'T
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
     val mutable func : Func<'T>
-
+#else
+    val mutable func : Func<'T> ?
+#endif
     member x.Value = 
         match x.func with 
         | null -> x.store 
         | _ -> 
         let res = LazyInitializer.EnsureInitialized(&x.store, x.func) 
-        x.func <- Unchecked.defaultof<_>
+        x.func <- null
         res
 
 //-------------------------------------------------------------------------
@@ -197,9 +202,7 @@ module Array =
     /// ~0.8x slower for ints
     let inline areEqual (xs: 'T []) (ys: 'T []) =
         match xs, ys with
-        | null, null -> true
         | [||], [||] -> true
-        | null, _ | _, null -> false
         | _ when xs.Length <> ys.Length -> false
         | _ ->
             let mutable break' = false
@@ -223,8 +226,7 @@ module Array =
     /// check if subArray is found in the wholeArray starting 
     /// at the provided index
     let inline isSubArray (subArray: 'T []) (wholeArray:'T []) index = 
-        if isNull subArray || isNull wholeArray then false
-        elif subArray.Length = 0 then true
+        if subArray.Length = 0 then true
         elif subArray.Length > wholeArray.Length then false
         elif subArray.Length = wholeArray.Length then areEqual subArray wholeArray else
         let rec loop subidx idx =
@@ -536,9 +538,6 @@ module String =
             String strArr
 
     let extractTrailingIndex (str: string) =
-        match str with
-        | null -> null, None
-        | _ ->
             let charr = str.ToCharArray() 
             Array.revInPlace charr
             let digits = Array.takeWhile Char.IsDigit charr
@@ -548,13 +547,9 @@ module String =
                | "" -> str, None
                | index -> str.Substring (0, str.Length - index.Length), Some (int index)
 
-    /// Remove all trailing and leading whitespace from the string
-    /// return null if the string is null
-    let trim (value: string) = if isNull value then null else value.Trim()
-
     /// Splits a string into substrings based on the strings in the array separators
     let split options (separator: string []) (value: string) = 
-        if isNull value then null else value.Split(separator, options)
+        value.Split(separator, options)
 
     let (|StartsWith|_|) pattern value =
         if String.IsNullOrWhiteSpace value then
@@ -1008,7 +1003,7 @@ type LazyWithContext<'T, 'ctxt> =
 
       /// This field holds either the function to run or a LazyWithContextFailure object recording the exception raised 
       /// from running the function. It is null if the thunk has been evaluated successfully.
-      mutable funcOrException: obj
+      mutable funcOrException: obj 
 
       /// A helper to ensure we rethrow the "original" exception
       findOriginalException : exn -> exn }
@@ -1331,14 +1326,32 @@ module Shim =
             member __.IsPathRootedShim (path: string) = Path.IsPathRooted path
 
             member __.IsInvalidPathShim(path: string) = 
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
                 let isInvalidPath(p: string) = 
-                    String.IsNullOrEmpty p || p.IndexOfAny(Path.GetInvalidPathChars()) <> -1
-
+#else
+                let isInvalidPath(p: string?) = 
+#endif
+                    match p with 
+                    | null | "" -> true
+                    | NonNull p -> p.IndexOfAny(Path.GetInvalidPathChars()) <> -1
+
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
                 let isInvalidFilename(p: string) = 
-                    String.IsNullOrEmpty p || p.IndexOfAny(Path.GetInvalidFileNameChars()) <> -1
-
+#else
+                let isInvalidFilename(p: string?) = 
+#endif
+                    match p with 
+                    | null | "" -> true
+                    | NonNull p -> p.IndexOfAny(Path.GetInvalidFileNameChars()) <> -1
+
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
                 let isInvalidDirectory(d: string) = 
-                    d=null || d.IndexOfAny(Path.GetInvalidPathChars()) <> -1
+#else
+                let isInvalidDirectory(d: string?) = 
+#endif
+                    match d with 
+                    | null -> true
+                    | NonNull d -> d.IndexOfAny(Path.GetInvalidPathChars()) <> -1
 
                 isInvalidPath path || 
                 let directory = Path.GetDirectoryName path

src/absil/ilread.fs

@@ -683,17 +683,24 @@ type GenericParamsIdx = GenericParamsIdx of int * TypeOrMethodDefTag * int
 
 let mkCacheInt32 lowMem _inbase _nm _sz =
     if lowMem then (fun f x -> f x) else
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
     let cache = ref null 
+#else
+    let cache : Dictionary<_,_>? ref = ref null // TODO NULLNESS: this explicit annotation should not be needed 
+#endif
     let count = ref 0
 #if STATISTICS
     addReport (fun oc -> if !count <> 0 then oc.WriteLine ((_inbase + string !count + " "+ _nm + " cache hits"): string))
 #endif
     fun f (idx: int32) ->
         let cache = 
             match !cache with
-            | null -> cache := new Dictionary<int32, _>(11)
-            | _ -> ()
-            !cache
+            | null -> 
+                let c = new Dictionary<int32, _>(11)
+                cache :=  c
+                c
+            | NonNull c -> c 
+
         match cache.TryGetValue idx with
         | true, res ->
             incr count 
@@ -705,18 +712,25 @@ let mkCacheInt32 lowMem _inbase _nm _sz =
 
 let mkCacheGeneric lowMem _inbase _nm _sz =
     if lowMem then (fun f x -> f x) else
+#if BUILDING_WITH_LKG || BUILD_FROM_SOURCE
     let cache = ref null 
+#else
+    let cache : Dictionary<_,_>? ref = ref null // TODO NULLNESS: this explicit annotation should not be needed
+#endif
     let count = ref 0
 #if STATISTICS
     addReport (fun oc -> if !count <> 0 then oc.WriteLine ((_inbase + string !count + " " + _nm + " cache hits"): string))
 #endif
     fun f (idx :'T) ->
         let cache = 
             match !cache with
-            | null -> cache := new Dictionary<_, _>(11 (* sz: int *) ) 
-            | _ -> ()
-            !cache
-        match cache.TryGetValue idx with
+            | null -> 
+                let c = new Dictionary<_, _>(11) 
+                cache := c
+                c
+            | NonNull c -> c
+
+        match cache.TryGetValue(idx) with
         | true, v ->
             incr count
             v
@@ -3030,7 +3044,7 @@ and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: Bin
                     let bytes = 
                         let bytes = pevEager.Slice(offsetOfBytesFromStartOfPhysicalPEFile, resourceLength)
                         // If we are trying to reduce memory, create a memory mapped file based on the contents.
-                        if canReduceMemory then
+                        if canReduceMemory && resourceLength > 0 then
                             ByteMemory.CreateMemoryMappedFile bytes
                         else
                             ByteMemory.FromArray(bytes.ToArray())