full implementation of engine v2

src/Features/Core/Portable/Diagnostics/Analyzers/IDocumentDiagnosticAnalyzer.cs

@@ -3,7 +3,6 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.CodeAnalysis.LanguageServices;
 
 namespace Microsoft.CodeAnalysis.Diagnostics
 {
@@ -12,10 +11,14 @@ namespace Microsoft.CodeAnalysis.Diagnostics
     /// </summary>
     internal abstract class DocumentDiagnosticAnalyzer : DiagnosticAnalyzer
     {
+        // REVIEW: why DocumentDiagnosticAnalyzer doesn't have span based analysis?
         public abstract Task AnalyzeSyntaxAsync(Document document, Action<Diagnostic> addDiagnostic, CancellationToken cancellationToken);
         public abstract Task AnalyzeSemanticsAsync(Document document, Action<Diagnostic> addDiagnostic, CancellationToken cancellationToken);
 
-        public override void Initialize(AnalysisContext context)
+        /// <summary>
+        /// it is not allowed one to implement both DocumentDiagnosticAnalzyer and DiagnosticAnalyzer
+        /// </summary>
+        public sealed override void Initialize(AnalysisContext context)
         {
         }
     }

src/Features/Core/Portable/Diagnostics/Analyzers/IProjectDiagnosticAnalyzer.cs

@@ -3,7 +3,6 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.CodeAnalysis.LanguageServices;
 
 namespace Microsoft.CodeAnalysis.Diagnostics
 {
@@ -14,7 +13,10 @@ internal abstract class ProjectDiagnosticAnalyzer : DiagnosticAnalyzer
     {
         public abstract Task AnalyzeProjectAsync(Project project, Action<Diagnostic> addDiagnostic, CancellationToken cancellationToken);
 
-        public override void Initialize(AnalysisContext context)
+        /// <summary>
+        /// it is not allowed one to implement both ProjectDiagnosticAnalzyer and DiagnosticAnalyzer
+        /// </summary>
+        public sealed override void Initialize(AnalysisContext context)
         {
         }
     }

src/Features/Core/Portable/Diagnostics/BaseDiagnosticIncrementalAnalyzer.cs

@@ -199,22 +199,8 @@ protected BaseDiagnosticIncrementalAnalyzer(DiagnosticAnalyzerService owner, Wor
         /// It is up to each incremental analyzer how they will merge this information with live diagnostic info.
         /// 
         /// this API doesn't have cancellationToken since it can't be cancelled.
-        /// 
-        /// given diagnostics are project wide diagnostics that doesn't contain a source location.
-        /// </summary>
-        public abstract Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Project project, ImmutableArray<DiagnosticData> diagnostics);
-
-        /// <summary>
-        /// Callback from build listener.
-        /// 
-        /// Given diagnostics are errors host got from explicit build.
-        /// It is up to each incremental analyzer how they will merge this information with live diagnostic info
-        /// 
-        /// this API doesn't have cancellationToken since it can't be cancelled.
-        /// 
-        /// given diagnostics are ones that has a source location.
         /// </summary>
-        public abstract Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Document document, ImmutableArray<DiagnosticData> diagnostics);
+        public abstract Task SynchronizeWithBuildAsync(Workspace workspace, ImmutableDictionary<ProjectId, ImmutableArray<DiagnosticData>> diagnostics);
         #endregion
 
         internal DiagnosticAnalyzerService Owner { get; }

src/Features/Core/Portable/Diagnostics/DiagnosticAnalyzerService_BuildSynchronization.cs

@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-using System;
-using System.Collections.Concurrent;
 using System.Collections.Immutable;
 using System.Threading.Tasks;
 using Roslyn.Utilities;
@@ -10,81 +8,20 @@ namespace Microsoft.CodeAnalysis.Diagnostics
 {
     internal partial class DiagnosticAnalyzerService
     {
-        /// <summary>
-        /// Start new Batch build diagnostics update token.
-        /// </summary>
-        public IDisposable BeginBatchBuildDiagnosticsUpdate(Solution solution)
-        {
-            return new BatchUpdateToken(solution);
-        }
-
         /// <summary>
         /// Synchronize build errors with live error.
         /// 
         /// no cancellationToken since this can't be cancelled
         /// </summary>
-        public Task SynchronizeWithBuildAsync(IDisposable batchUpdateToken, Project project, ImmutableArray<DiagnosticData> diagnostics)
+        public Task SynchronizeWithBuildAsync(Workspace workspace, ImmutableDictionary<ProjectId, ImmutableArray<DiagnosticData>> diagnostics)
         {
-            var token = (BatchUpdateToken)batchUpdateToken;
-            token.CheckProjectInSnapshot(project);
-
-            BaseDiagnosticIncrementalAnalyzer analyzer;
-            if (_map.TryGetValue(project.Solution.Workspace, out analyzer))
-            {
-                return analyzer.SynchronizeWithBuildAsync(token, project, diagnostics);
-            }
-
-            return SpecializedTasks.EmptyTask;
-        }
-
-        /// <summary>
-        /// Synchronize build errors with live error
-        /// 
-        /// no cancellationToken since this can't be cancelled
-        /// </summary>
-        public Task SynchronizeWithBuildAsync(IDisposable batchUpdateToken, Document document, ImmutableArray<DiagnosticData> diagnostics)
-        {
-            var token = (BatchUpdateToken)batchUpdateToken;
-            token.CheckDocumentInSnapshot(document);
-
             BaseDiagnosticIncrementalAnalyzer analyzer;
-            if (_map.TryGetValue(document.Project.Solution.Workspace, out analyzer))
+            if (_map.TryGetValue(workspace, out analyzer))
             {
-                return analyzer.SynchronizeWithBuildAsync(token, document, diagnostics);
+                return analyzer.SynchronizeWithBuildAsync(workspace, diagnostics);
             }
 
             return SpecializedTasks.EmptyTask;
         }
-
-        public class BatchUpdateToken : IDisposable
-        {
-            public readonly ConcurrentDictionary<object, object> _cache = new ConcurrentDictionary<object, object>(concurrencyLevel: 2, capacity: 1);
-            private readonly Solution _solution;
-
-            public BatchUpdateToken(Solution solution)
-            {
-                _solution = solution;
-            }
-
-            public object GetCache(object key, Func<object, object> cacheCreator)
-            {
-                return _cache.GetOrAdd(key, cacheCreator);
-            }
-
-            public void CheckDocumentInSnapshot(Document document)
-            {
-                Contract.ThrowIfFalse(_solution.GetDocument(document.Id) == document);
-            }
-
-            public void CheckProjectInSnapshot(Project project)
-            {
-                Contract.ThrowIfFalse(_solution.GetProject(project.Id) == project);
-            }
-
-            public void Dispose()
-            {
-                _cache.Clear();
-            }
-        }
     }
 }

src/Features/Core/Portable/Diagnostics/DiagnosticAnalyzerService_IncrementalAnalyzer.cs

@@ -180,14 +180,9 @@ public override bool ContainsDiagnostics(Workspace workspace, ProjectId projectI
             #endregion
 
             #region build synchronization
-            public override Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Project project, ImmutableArray<DiagnosticData> diagnostics)
+            public override Task SynchronizeWithBuildAsync(Workspace workspace, ImmutableDictionary<ProjectId, ImmutableArray<DiagnosticData>> diagnostics)
             {
-                return Analyzer.SynchronizeWithBuildAsync(token, project, diagnostics);
-            }
-
-            public override Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Document document, ImmutableArray<DiagnosticData> diagnostics)
-            {
-                return Analyzer.SynchronizeWithBuildAsync(token, document, diagnostics);
+                return Analyzer.SynchronizeWithBuildAsync(workspace, diagnostics);
             }
             #endregion
 

src/Features/Core/Portable/Diagnostics/DiagnosticAnalyzerService_UpdateSource.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Immutable;
 using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Shared.TestHooks;
 using Roslyn.Utilities;
 
@@ -69,6 +70,22 @@ internal void RaiseBulkDiagnosticsUpdated(Action<Action<DiagnosticsUpdatedArgs>>
             }
         }
 
+        internal void RaiseBulkDiagnosticsUpdated(Func<Action<DiagnosticsUpdatedArgs>, Task> eventActionAsync)
+        {
+            // all diagnostics events are serialized.
+            var ev = _eventMap.GetEventHandlers<EventHandler<DiagnosticsUpdatedArgs>>(DiagnosticsUpdatedEventName);
+            if (ev.HasHandlers)
+            {
+                // we do this bulk update to reduce number of tasks (with captured data) enqueued.
+                // we saw some "out of memory" due to us having long list of pending tasks in memory. 
+                // this is to reduce for such case to happen.
+                Action<DiagnosticsUpdatedArgs> raiseEvents = args => ev.RaiseEvent(handler => handler(this, args));
+
+                var asyncToken = Listener.BeginAsyncOperation(nameof(RaiseDiagnosticsUpdated));
+                _eventQueue.ScheduleTask(() => eventActionAsync(raiseEvents)).CompletesAsyncOperation(asyncToken);
+            }
+        }
+
         bool IDiagnosticUpdateSource.SupportGetDiagnostics { get { return true; } }
 
         ImmutableArray<DiagnosticData> IDiagnosticUpdateSource.GetDiagnostics(Workspace workspace, ProjectId projectId, DocumentId documentId, object id, bool includeSuppressedDiagnostics, CancellationToken cancellationToken)

src/Features/Core/Portable/Diagnostics/EngineV1/DiagnosticAnalyzerDriver.cs

@@ -291,12 +291,15 @@ public async Task<ImmutableArray<Diagnostic>> GetProjectDiagnosticsAsync(Diagnos
 
                 using (var diagnostics = SharedPools.Default<List<Diagnostic>>().GetPooledObject())
                 {
-                    if (_project.SupportsCompilation)
+                    var projectAnalyzer = analyzer as ProjectDiagnosticAnalyzer;
+                    if (projectAnalyzer != null)
                     {
-                        await this.GetCompilationDiagnosticsAsync(analyzer, diagnostics.Object).ConfigureAwait(false);
+                        await this.GetProjectDiagnosticsWorkerAsync(projectAnalyzer, diagnostics.Object).ConfigureAwait(false);
+                        return diagnostics.Object.ToImmutableArray();
                     }
 
-                    await this.GetProjectDiagnosticsWorkerAsync(analyzer, diagnostics.Object).ConfigureAwait(false);
+                    Contract.ThrowIfFalse(_project.SupportsCompilation);
+                    await this.GetCompilationDiagnosticsAsync(analyzer, diagnostics.Object).ConfigureAwait(false);
 
                     return diagnostics.Object.ToImmutableArray();
                 }
@@ -307,29 +310,17 @@ public async Task<ImmutableArray<Diagnostic>> GetProjectDiagnosticsAsync(Diagnos
             }
         }
 
-        private async Task GetProjectDiagnosticsWorkerAsync(DiagnosticAnalyzer analyzer, List<Diagnostic> diagnostics)
+        private async Task GetProjectDiagnosticsWorkerAsync(ProjectDiagnosticAnalyzer analyzer, List<Diagnostic> diagnostics)
         {
+
             try
             {
-                var projectAnalyzer = analyzer as ProjectDiagnosticAnalyzer;
-                if (projectAnalyzer == null)
-                {
-                    return;
-                }
-
-                try
-                {
-                    await projectAnalyzer.AnalyzeProjectAsync(_project, diagnostics.Add, _cancellationToken).ConfigureAwait(false);
-                }
-                catch (Exception e) when (!IsCanceled(e, _cancellationToken))
-                {
-                    var compilation = await _project.GetCompilationAsync(_cancellationToken).ConfigureAwait(false);
-                    OnAnalyzerException(e, analyzer, compilation);
-                }
+                await analyzer.AnalyzeProjectAsync(_project, diagnostics.Add, _cancellationToken).ConfigureAwait(false);
             }
-            catch (Exception e) when (FatalError.ReportUnlessCanceled(e))
+            catch (Exception e) when (!IsCanceled(e, _cancellationToken))
             {
-                throw ExceptionUtilities.Unreachable;
+                var compilation = await _project.GetCompilationAsync(_cancellationToken).ConfigureAwait(false);
+                OnAnalyzerException(e, analyzer, compilation);
             }
         }
 

src/Features/Core/Portable/Diagnostics/EngineV1/DiagnosticIncrementalAnalyzer.StateManager.cs

@@ -68,16 +68,6 @@ public IEnumerable<StateSet> GetStateSets(Project project)
                 return GetStateSets(project.Id).Where(s => s.Language == project.Language);
             }
 
-            /// <summary>
-            /// Return <see cref="StateSet"/>s that are added as the given <see cref="Project"/>'s AnalyzerReferences.
-            /// This will never create new <see cref="StateSet"/> but will return ones already created.
-            /// </summary>
-            public ImmutableArray<StateSet> GetBuildOnlyStateSets(object cache, Project project)
-            {
-                var stateSetCache = (IDictionary<Project, ImmutableArray<StateSet>>)cache;
-                return stateSetCache.GetOrAdd(project, CreateBuildOnlyProjectStateSet);
-            }
-
             /// <summary>
             /// Return <see cref="StateSet"/>s for the given <see cref="Project"/>. 
             /// This will either return already created <see cref="StateSet"/>s for the specific snapshot of <see cref="Project"/> or
@@ -126,7 +116,11 @@ public void RemoveStateSet(ProjectId projectId)
                 _projectStates.RemoveStateSet(projectId);
             }
 
-            private ImmutableArray<StateSet> CreateBuildOnlyProjectStateSet(Project project)
+            /// <summary>
+            /// Return <see cref="StateSet"/>s that are added as the given <see cref="Project"/>'s AnalyzerReferences.
+            /// This will never create new <see cref="StateSet"/> but will return ones already created.
+            /// </summary>
+            public ImmutableArray<StateSet> CreateBuildOnlyProjectStateSet(Project project)
             {
                 var referenceIdentities = project.AnalyzerReferences.Select(r => _analyzerManager.GetAnalyzerReferenceIdentity(r)).ToSet();
                 var stateSetMap = GetStateSets(project).ToDictionary(s => s.Analyzer, s => s);

src/Features/Core/Portable/Diagnostics/EngineV1/DiagnosticIncrementalAnalyzer_BuildSynchronization.cs

@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Globalization;
@@ -14,21 +12,42 @@ namespace Microsoft.CodeAnalysis.Diagnostics.EngineV1
 {
     internal partial class DiagnosticIncrementalAnalyzer
     {
-        private readonly static Func<object, object> s_cacheCreator = _ => new ConcurrentDictionary<Project, ImmutableArray<StateSet>>(concurrencyLevel: 2, capacity: 10);
-
-        public override async Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Project project, ImmutableArray<DiagnosticData> diagnostics)
+        public override async Task SynchronizeWithBuildAsync(Workspace workspace, ImmutableDictionary<ProjectId, ImmutableArray<DiagnosticData>> map)
         {
-            if (!PreferBuildErrors(project.Solution.Workspace))
+            if (!PreferBuildErrors(workspace))
             {
                 // prefer live errors over build errors
                 return;
             }
 
+            var solution = workspace.CurrentSolution;
+            foreach (var projectEntry in map)
+            {
+                var project = solution.GetProject(projectEntry.Key);
+                if (project == null)
+                {
+                    continue;
+                }
+
+                var stateSets = _stateManager.CreateBuildOnlyProjectStateSet(project);
+                var lookup = projectEntry.Value.ToLookup(d => d.DocumentId);
+
+                // do project one first
+                await SynchronizeWithBuildAsync(project, stateSets, lookup[null]).ConfigureAwait(false);
+
+                foreach (var document in project.Documents)
+                {
+                    await SynchronizeWithBuildAsync(document, stateSets, lookup[document.Id]).ConfigureAwait(false);
+                }
+            }
+        }
+
+        private async Task SynchronizeWithBuildAsync(Project project, IEnumerable<StateSet> stateSets, IEnumerable<DiagnosticData> diagnostics)
+        {
             using (var poolObject = SharedPools.Default<HashSet<string>>().GetPooledObject())
             {
                 var lookup = CreateDiagnosticIdLookup(diagnostics);
-
-                foreach (var stateSet in _stateManager.GetBuildOnlyStateSets(token.GetCache(_stateManager, s_cacheCreator), project))
+                foreach (var stateSet in stateSets)
                 {
                     var descriptors = HostAnalyzerManager.GetDiagnosticDescriptors(stateSet.Analyzer);
                     var liveDiagnostics = ConvertToLiveDiagnostics(lookup, descriptors, poolObject.Object);
@@ -47,14 +66,9 @@ public override async Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.B
             }
         }
 
-        public override async Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.BatchUpdateToken token, Document document, ImmutableArray<DiagnosticData> diagnostics)
+        private async Task SynchronizeWithBuildAsync(Document document, IEnumerable<StateSet> stateSets, IEnumerable<DiagnosticData> diagnostics)
         {
             var workspace = document.Project.Solution.Workspace;
-            if (!PreferBuildErrors(workspace))
-            {
-                // prefer live errors over build errors
-                return;
-            }
 
             // check whether, for opened documents, we want to prefer live diagnostics
             if (PreferLiveErrorsOnOpenedFiles(workspace) && workspace.IsDocumentOpen(document.Id))
@@ -68,7 +82,7 @@ public override async Task SynchronizeWithBuildAsync(DiagnosticAnalyzerService.B
             {
                 var lookup = CreateDiagnosticIdLookup(diagnostics);
 
-                foreach (var stateSet in _stateManager.GetBuildOnlyStateSets(token.GetCache(_stateManager, s_cacheCreator), document.Project))
+                foreach (var stateSet in stateSets)
                 {
                     // we are using Default so that things like LB can't use cached information
                     var textVersion = VersionStamp.Default;
@@ -141,13 +155,8 @@ private ImmutableArray<DiagnosticData> MergeDiagnostics(ImmutableArray<Diagnosti
             return builder == null ? ImmutableArray<DiagnosticData>.Empty : builder.ToImmutable();
         }
 
-        private static ILookup<string, DiagnosticData> CreateDiagnosticIdLookup(ImmutableArray<DiagnosticData> diagnostics)
+        private static ILookup<string, DiagnosticData> CreateDiagnosticIdLookup(IEnumerable<DiagnosticData> diagnostics)
         {
-            if (diagnostics.Length == 0)
-            {
-                return null;
-            }
-
             return diagnostics.ToLookup(d => d.Id);
         }
 

src/Features/Core/Portable/Diagnostics/EngineV1/DiagnosticIncrementalAnalyzer_GetDiagnostics.cs

@@ -236,7 +236,7 @@ protected void AppendDiagnostics(IEnumerable<DiagnosticData> items)
                 }
             }
 
-            protected virtual ImmutableArray<DiagnosticData> GetDiagnosticData()
+            protected ImmutableArray<DiagnosticData> GetDiagnosticData()
             {
                 return _builder != null ? _builder.ToImmutableArray() : ImmutableArray<DiagnosticData>.Empty;
             }