Aot for static linked fsharpcore test and fix duplicated embedded typ…

…es issue. (#15168)

* Add aot

* tests

* Add component test

* logfile

* improve filter

* feedback

.gitignore

@@ -126,3 +126,4 @@ nCrunchTemp_*
 
 tests/FSharp.Compiler.Service.Tests/FSharp.CompilerService.SurfaceArea.netstandard.actual
 *.vsp
+/tests/AheadOfTime/Trimming/output.txt

azure-pipelines.yml

@@ -650,19 +650,13 @@ stages:
             displayName: Report dotnet SDK versions
           - script: .\Build.cmd $(_kind) -pack -c $(_BuildConfig)
             displayName: Initial build and prepare packages.
-          - script:  dotnet publish -c Release -f:net472 -bl:"./artifacts/log/Release/AheadOfTime/SelfContained_Trimming_net472.binlog"
-            displayName: Build and publish a trim test package.
-            workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime/SelfContained_Trimming_Test
-          - script:  dotnet publish -c Release  -f:net7.0 -bl:"./artifacts/log/Release/AheadOfTime/SelfContained_Trimming_net7.0.binlog"
-            displayName: Build and publish a trim test package.
-            workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime/SelfContained_Trimming_Test
-          - script: .\check.cmd
-            displayName: Check the state of the trimmed app.
-            workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime/SelfContained_Trimming_Test
+          - script: $(Build.SourcesDirectory)/tests/AheadOfTime/Trimming/check.cmd
+            displayName: Build, trim, publish and check the state of the trimmed app.
+            workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime/Trimming
           - task: PublishPipelineArtifact@1
             displayName: Publish Trim Tests Logs
             inputs:
-              targetPath: './artifacts/log/Release/AheadOfTime'
+              targetPath: './artifacts/log/Release/AheadOfTime/Trimming/'
               artifactName: 'Trim Test Logs Attempt $(System.JobAttempt) Logs $(_kind)'
             continueOnError: true
             condition: always()

eng/Build.ps1

@@ -61,6 +61,7 @@ param (
     [switch]$testAll,
     [switch]$testAllButIntegration,
     [switch]$testpack,
+    [switch]$testAOT,
     [string]$officialSkipTests = "false",
     [switch]$noVisualStudio,
     [switch]$sourceBuild,
@@ -109,6 +110,7 @@ function Print-Usage() {
     Write-Host "  -testScripting                Run Scripting tests"
     Write-Host "  -testVs                       Run F# editor unit tests"
     Write-Host "  -testpack                     Verify built packages"
+    Write-Host "  -testAOT                      Run AOT/Trimming tests"
     Write-Host "  -officialSkipTests <bool>     Set to 'true' to skip running tests"
     Write-Host ""
     Write-Host "Advanced settings:"
@@ -148,6 +150,7 @@ function Process-Arguments() {
         $script:testFSharpQA = $True
         $script:testIntegration = $True
         $script:testVs = $True
+        $script:testAOT = $True
     }
 
     if ($testAllButIntegration) {
@@ -156,6 +159,7 @@ function Process-Arguments() {
         $script:testFSharpQA = $True
         $script:testIntegration = $False
         $script:testVs = $True
+        $script:testAOT = $True
     }
 
     if ([System.Boolean]::Parse($script:officialSkipTests)) {
@@ -171,6 +175,7 @@ function Process-Arguments() {
         $script:testIntegration = $False
         $script:testVs = $False
         $script:testpack = $False
+        $script:testAOT = $False
         $script:verifypackageshipstatus = $True
     }
 
@@ -202,6 +207,10 @@ function Process-Arguments() {
         $script:verifypackageshipstatus = $True;
     }
 
+    if ($testAOT) {
+        $script:pack = $True;
+    }
+
     foreach ($property in $properties) {
         if (!$property.StartsWith("/p:", "InvariantCultureIgnoreCase")) {
             Write-Host "Invalid argument: $property"
@@ -637,11 +646,17 @@ try {
         TestUsingXUnit -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" -targetFramework $desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj"
         TestUsingNUnit -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\VisualFSharp.UnitTests\"
     }
-    
+
     if ($testIntegration) {
         TestUsingXUnit -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.IntegrationTests\"
     }
 
+    if ($testAOT) {
+        Push-Location "$RepoRoot\tests\AheadOfTime\Trimming"
+        ./check.cmd
+        Pop-Location
+    }
+
     # verify nupkgs have access to the source code
     $nupkgtestFailed = $false
     if ($testpack) {

src/Compiler/AbstractIL/ilmorph.fs

@@ -256,8 +256,21 @@ let morphILMethodDefs f (m: ILMethodDefs) = mkILMethods (List.map f (m.AsList())
 let morphILFieldDefs f (fdefs: ILFieldDefs) =
     mkILFields (List.map f (fdefs.AsList()))
 
-let morphILTypeDefs f (tdefs: ILTypeDefs) =
-    mkILTypeDefsFromArray (Array.map f (tdefs.AsArray()))
+let morphILTypeDefs isInKnownSet f (tdefs: ILTypeDefs) =
+    let filtered (tdefs: ILTypeDef array) =
+        // The key ensures that items in the Known Set are not duplicated everything else may be.
+        let mkKey (i, (td: ILTypeDef)) =
+            if isInKnownSet td.Name then
+                struct (0, td.Name)
+            else
+                struct (i + 1, td.Name)
+
+        tdefs
+        |> Array.indexed
+        |> Array.distinctBy mkKey
+        |> Array.map (fun (_, td) -> td)
+
+    mkILTypeDefsFromArray (Array.map f (filtered (tdefs.AsArray())))
 
 let morphILLocals f locals = List.map (morphILLocal f) locals
 
@@ -349,7 +362,7 @@ let edefs_ty2ty f (edefs: ILEventDefs) =
 let mimpls_ty2ty f (mimpls: ILMethodImplDefs) =
     mkILMethodImpls (mimpls.AsList() |> List.map (mimpl_ty2ty f))
 
-let rec tdef_ty2ty_ilmbody2ilmbody_mdefs2mdefs enc fs (tdef: ILTypeDef) =
+let rec tdef_ty2ty_ilmbody2ilmbody_mdefs2mdefs isInKnownSet enc fs (tdef: ILTypeDef) =
     let fTyInCtxt, fMethodDefs = fs
     let fTyInCtxtR = fTyInCtxt (Some(enc, tdef)) None
     let mdefsR = fMethodDefs (enc, tdef) tdef.Methods
@@ -360,16 +373,16 @@ let rec tdef_ty2ty_ilmbody2ilmbody_mdefs2mdefs enc fs (tdef: ILTypeDef) =
         genericParams = gparams_ty2ty fTyInCtxtR tdef.GenericParams,
         extends = Option.map fTyInCtxtR tdef.Extends,
         methods = mdefsR,
-        nestedTypes = tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs (enc @ [ tdef ]) fs tdef.NestedTypes,
+        nestedTypes = tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs isInKnownSet (enc @ [ tdef ]) fs tdef.NestedTypes,
         fields = fdefsR,
         methodImpls = mimpls_ty2ty fTyInCtxtR tdef.MethodImpls,
         events = edefs_ty2ty fTyInCtxtR tdef.Events,
         properties = pdefs_ty2ty fTyInCtxtR tdef.Properties,
         customAttrs = cattrs_ty2ty fTyInCtxtR tdef.CustomAttrs
     )
 
-and tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs enc fs tdefs =
-    morphILTypeDefs (tdef_ty2ty_ilmbody2ilmbody_mdefs2mdefs enc fs) tdefs
+and tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs isInKnownSet enc fs tdefs =
+    morphILTypeDefs isInKnownSet (tdef_ty2ty_ilmbody2ilmbody_mdefs2mdefs isInKnownSet enc fs) tdefs
 
 // --------------------------------------------------------------------
 // Derived versions of the above, e.g. with defaults added
@@ -381,20 +394,21 @@ let manifest_ty2ty f (m: ILAssemblyManifest) =
     }
 
 let morphILTypeInILModule_ilmbody2ilmbody_mdefs2mdefs
+    isInKnownSet
     (fTyInCtxt: ILModuleDef -> (ILTypeDef list * ILTypeDef) option -> ILMethodDef option -> ILType -> ILType, fMethodDefs)
     modul
     =
 
     let ftdefs =
-        tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs [] (fTyInCtxt modul, fMethodDefs modul)
+        tdefs_ty2ty_ilmbody2ilmbody_mdefs2mdefs isInKnownSet [] (fTyInCtxt modul, fMethodDefs modul)
 
     { modul with
         TypeDefs = ftdefs modul.TypeDefs
         CustomAttrsStored = storeILCustomAttrs (cattrs_ty2ty (fTyInCtxt modul None None) modul.CustomAttrs)
         Manifest = Option.map (manifest_ty2ty (fTyInCtxt modul None None)) modul.Manifest
     }
 
-let morphILInstrsAndILTypesInILModule fs modul =
+let morphILInstrsAndILTypesInILModule isInKnownSet fs modul =
     let fCode, fTyInCtxt = fs
 
     let fMethBody modCtxt tdefCtxt mdefCtxt =
@@ -403,20 +417,20 @@ let morphILInstrsAndILTypesInILModule fs modul =
     let fMethodDefs modCtxt tdefCtxt =
         mdefs_ty2ty_ilmbody2ilmbody (fTyInCtxt modCtxt (Some tdefCtxt), fMethBody modCtxt tdefCtxt)
 
-    morphILTypeInILModule_ilmbody2ilmbody_mdefs2mdefs (fTyInCtxt, fMethodDefs) modul
+    morphILTypeInILModule_ilmbody2ilmbody_mdefs2mdefs isInKnownSet (fTyInCtxt, fMethodDefs) modul
 
 let morphILInstrsInILCode f ilcode = code_instr2instrs f ilcode
 
-let morphILTypeInILModule fTyInCtxt modul =
+let morphILTypeInILModule isInKnownSet fTyInCtxt modul =
     let finstr modCtxt tdefCtxt mdefCtxt =
         let fTy = fTyInCtxt modCtxt (Some tdefCtxt) mdefCtxt
         morphILTypesInILInstr ((fun _instrCtxt -> fTy), (fun _instrCtxt _formalCtxt -> fTy))
 
-    morphILInstrsAndILTypesInILModule (finstr, fTyInCtxt) modul
+    morphILInstrsAndILTypesInILModule isInKnownSet (finstr, fTyInCtxt) modul
 
-let morphILTypeRefsInILModuleMemoized f modul =
+let morphILTypeRefsInILModuleMemoized isInKnownSet f modul =
     let fTy = Tables.memoize (morphILTypeRefsInILType f)
-    morphILTypeInILModule (fun _ _ _ ty -> fTy ty) modul
+    morphILTypeInILModule isInKnownSet (fun _ _ _ ty -> fTy ty) modul
 
-let morphILScopeRefsInILModuleMemoized f modul =
-    morphILTypeRefsInILModuleMemoized (morphILScopeRefsInILTypeRef f) modul
+let morphILScopeRefsInILModuleMemoized isInKnownSet f modul =
+    morphILTypeRefsInILModuleMemoized isInKnownSet (morphILScopeRefsInILTypeRef f) modul

src/Compiler/AbstractIL/ilmorph.fsi

@@ -17,9 +17,9 @@ val morphILScopeRefsInILTypeRef: (ILScopeRef -> ILScopeRef) -> ILTypeRef -> ILTy
 val morphILTypeRefsInILType: (ILTypeRef -> ILTypeRef) -> ILType -> ILType
 
 /// Morph all type references throughout an entire module.
-val morphILTypeRefsInILModuleMemoized: (ILTypeRef -> ILTypeRef) -> ILModuleDef -> ILModuleDef
+val morphILTypeRefsInILModuleMemoized: (string -> bool) -> (ILTypeRef -> ILTypeRef) -> ILModuleDef -> ILModuleDef
 
-val morphILScopeRefsInILModuleMemoized: (ILScopeRef -> ILScopeRef) -> ILModuleDef -> ILModuleDef
+val morphILScopeRefsInILModuleMemoized: (string -> bool) -> (ILScopeRef -> ILScopeRef) -> ILModuleDef -> ILModuleDef
 
 val morphILInstrsInILCode: (ILInstr -> ILInstr list) -> ILCode -> ILCode
 

src/Compiler/Driver/StaticLinking.fs

@@ -16,6 +16,7 @@ open FSharp.Compiler.CompilerOptions
 open FSharp.Compiler.DiagnosticsLogger
 open FSharp.Compiler.IO
 open FSharp.Compiler.OptimizeInputs
+open FSharp.Compiler.TcGlobals
 open FSharp.Compiler.Text.Range
 open FSharp.Compiler.TypedTree
 open FSharp.Compiler.TypedTreeBasics
@@ -254,7 +255,7 @@ let StaticLinkILModules
                     NativeResources = savedNativeResources
                 }
 
-            Morphs.morphILTypeRefsInILModuleMemoized typeForwarding.TypeForwardILTypeRef main
+            Morphs.morphILTypeRefsInILModuleMemoized TcGlobals.IsInEmbeddableKnownSet typeForwarding.TypeForwardILTypeRef main
 
         ilxMainModule, rewriteExternalRefsToLocalRefs
 
@@ -495,7 +496,9 @@ let rec implantTypeDef ilGlobals isNested (tdefs: ILTypeDefs) (enc: string list)
 // Compute a static linker. This only captures tcImports (a large data structure) if
 // static linking is enabled. Normally this is not the case, which lets us collect tcImports
 // prior to this point.
-let StaticLink (ctok, tcConfig: TcConfig, tcImports: TcImports, ilGlobals: ILGlobals) =
+let StaticLink (ctok, tcConfig: TcConfig, tcImports: TcImports, tcGlobals: TcGlobals) =
+
+    let ilGlobals = tcGlobals.ilg
 
 #if !NO_TYPEPROVIDERS
     let providerGeneratedAssemblies =
@@ -549,7 +552,7 @@ let StaticLink (ctok, tcConfig: TcConfig, tcImports: TcImports, ilGlobals: ILGlo
 
                     let ilModule =
                         ilModule
-                        |> Morphs.morphILTypeRefsInILModuleMemoized (fun tref ->
+                        |> Morphs.morphILTypeRefsInILModuleMemoized TcGlobals.IsInEmbeddableKnownSet (fun tref ->
                             if debugStaticLinking then
                                 printfn "deciding whether to rewrite type ref %A" tref.QualifiedName
 
@@ -718,6 +721,7 @@ let StaticLink (ctok, tcConfig: TcConfig, tcImports: TcImports, ilGlobals: ILGlo
                     NormalizeAssemblyRefs(ctok, ilGlobals, tcImports)
 
                 Morphs.morphILTypeRefsInILModuleMemoized
+                    TcGlobals.IsInEmbeddableKnownSet
                     (Morphs.morphILScopeRefsInILTypeRef (
                         validateTargetPlatform
                         >> rewriteExternalRefsToLocalRefs

src/Compiler/Driver/StaticLinking.fsi

@@ -7,10 +7,11 @@ open Internal.Utilities.Library
 open FSharp.Compiler.AbstractIL.IL
 open FSharp.Compiler.CompilerConfig
 open FSharp.Compiler.CompilerImports
+open FSharp.Compiler.TcGlobals
 
 // Compute a static linker. This only captures tcImports (a large data structure) if
 // static linking is enabled. Normally this is not the case, which lets us collect tcImports
 // prior to this point.
 val StaticLink:
-    ctok: CompilationThreadToken * tcConfig: TcConfig * tcImports: TcImports * ilGlobals: ILGlobals ->
+    ctok: CompilationThreadToken * tcConfig: TcConfig * tcImports: TcImports * tcGlobals: TcGlobals ->
         (ILModuleDef -> ILModuleDef)

src/Compiler/Driver/fsc.fs

@@ -930,13 +930,11 @@ let main4
     | None -> ()
     | Some f -> f tcImports
 
-    // Compute a static linker, it gets called later.
-    let ilGlobals = tcGlobals.ilg
-
     if tcConfig.standalone && generatedCcu.UsesFSharp20PlusQuotations then
         error (Error(FSComp.SR.fscQuotationLiteralsStaticLinking0 (), rangeStartup))
 
-    let staticLinker = StaticLink(ctok, tcConfig, tcImports, ilGlobals)
+    // Compute a static linker, it gets called later.
+    let staticLinker = StaticLink(ctok, tcConfig, tcImports, tcGlobals)
 
     ReportTime tcConfig "TAST -> IL"
     use _ = UseBuildPhase BuildPhase.IlxGen

src/Compiler/Interactive/fsi.fs

@@ -1814,7 +1814,8 @@ type internal FsiDynamicCompiler
 
         // Rewrite references to local types to their respective dynamic assemblies
         let ilxMainModule =
-            ilxMainModule |> Morphs.morphILTypeRefsInILModuleMemoized emEnv.MapTypeRef
+            ilxMainModule
+            |> Morphs.morphILTypeRefsInILModuleMemoized TcGlobals.IsInEmbeddableKnownSet emEnv.MapTypeRef
 
         let opts =
             {
@@ -1943,7 +1944,10 @@ type internal FsiDynamicCompiler
         ReportTime tcConfig "Assembly refs Normalised"
 
         let ilxMainModule =
-            Morphs.morphILScopeRefsInILModuleMemoized (NormalizeAssemblyRefs(ctok, ilGlobals, tcImports)) ilxMainModule
+            Morphs.morphILScopeRefsInILModuleMemoized
+                TcGlobals.IsInEmbeddableKnownSet
+                (NormalizeAssemblyRefs(ctok, ilGlobals, tcImports))
+                ilxMainModule
 
         diagnosticsLogger.AbortOnError(fsiConsoleOutput)
 

src/Compiler/TypedTree/TcGlobals.fs

@@ -332,28 +332,39 @@ type TcGlobals(
       let tcref = mkNonLocalTyconRef2 ccu (Array.ofList path) typeName
       AttribInfo(tref, tcref)
 
-  let findOrEmbedSysPublicAttribute nm =
-        let sysAttrib = findPublicSysAttrib nm
-        if sysAttrib.TyconRef.CanDeref then
-            sysAttrib
-        else
-            let attrRef = ILTypeRef.Create(ILScopeRef.Local, [], nm)
-            let attrTycon =
-                Construct.NewTycon(
-                    Some (CompPath(ILScopeRef.Local, [])),
-                    attrRef.Name,
-                    range0,
-                    taccessInternal,
-                    taccessInternal,
-                    TyparKind.Type,
-                    LazyWithContext.NotLazy [],
-                    FSharp.Compiler.Xml.XmlDoc.Empty,
-                    false,
-                    false,
-                    false,
-                    MaybeLazy.Strict(Construct.NewEmptyModuleOrNamespaceType ModuleOrType)
-                )
-            AttribInfo(attrRef, mkLocalTyconRef attrTycon)
+  // Well known set of generated embeddable attribute names
+  static let isInEmbeddableKnownSet name =
+      match name with
+      | "System.Runtime.CompilerServices.IsReadOnlyAttribute"
+      | "System.Diagnostics.CodeAnalysis.DynamicDependencyAttribute"
+      | "System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes" -> true
+      | _ -> false
+
+  let findOrEmbedSysPublicType nm =
+
+      assert (isInEmbeddableKnownSet nm)                        //Ensure that the named type is in known set of embedded types
+
+      let sysAttrib = findPublicSysAttrib nm
+      if sysAttrib.TyconRef.CanDeref then
+          sysAttrib
+      else
+          let attrRef = ILTypeRef.Create(ILScopeRef.Local, [], nm)
+          let attrTycon =
+             Construct.NewTycon(
+                 Some (CompPath(ILScopeRef.Local, [])),
+                 attrRef.Name,
+                 range0,
+                 taccessInternal,
+                  taccessInternal,
+                  TyparKind.Type,
+                  LazyWithContext.NotLazy [],
+                  FSharp.Compiler.Xml.XmlDoc.Empty,
+                  false,
+                  false,
+                  false,
+                  MaybeLazy.Strict(Construct.NewEmptyModuleOrNamespaceType ModuleOrType)
+              )
+          AttribInfo(attrRef, mkLocalTyconRef attrTycon)
 
   let mkSysNonGenericTy path n = mkNonGenericTy(findSysTyconRef path n)
   let tryMkSysNonGenericTy path n = tryFindSysTyconRef path n |> Option.map mkNonGenericTy
@@ -1051,6 +1062,8 @@ type TcGlobals(
 
   member _.ilg = ilg
 
+  static member IsInEmbeddableKnownSet name = isInEmbeddableKnownSet name
+
   member _.embeddedTypeDefs = embeddedILTypeDefs.Values |> Seq.toList
 
   member _.tryRemoveEmbeddedILTypeDefs () = [
@@ -1421,9 +1434,9 @@ type TcGlobals(
 
   // We use 'findSysAttrib' here because lookup on attribute is done by name comparison, and can proceed
   // even if the type is not found in a system assembly.
-  member val attrib_IsReadOnlyAttribute = findOrEmbedSysPublicAttribute "System.Runtime.CompilerServices.IsReadOnlyAttribute"
-  member val attrib_DynamicDependencyAttribute = findOrEmbedSysPublicAttribute "System.Diagnostics.CodeAnalysis.DynamicDependencyAttribute"
-  member val enum_DynamicallyAccessedMemberTypes = findOrEmbedSysPublicAttribute "System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes"
+  member val attrib_IsReadOnlyAttribute = findOrEmbedSysPublicType "System.Runtime.CompilerServices.IsReadOnlyAttribute"
+  member val attrib_DynamicDependencyAttribute = findOrEmbedSysPublicType "System.Diagnostics.CodeAnalysis.DynamicDependencyAttribute"
+  member val enum_DynamicallyAccessedMemberTypes = findOrEmbedSysPublicType "System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes"
 
   member val attrib_SystemObsolete = findSysAttrib "System.ObsoleteAttribute"
   member val attrib_DllImportAttribute = tryFindSysAttrib "System.Runtime.InteropServices.DllImportAttribute"