Add Run Now support for schema snapshots

#902

DBADash/Messaging/CollectionMessage.cs

@@ -60,11 +60,17 @@ public override async Task<DataSet> Process(CollectionConfig cfg, Guid handle)
 
             var (standardCollections, customCollections) = ParseCollectionTypes(src, cfg);
 
+            if (standardCollections.Contains(CollectionType.SchemaSnapshot))
+            {
+                // Written to destinations as usual.  Could be additional delay to process.
+                // It's done DB at a time to limit the size of the data being processed.
+                await SchemaSnapshotDB.GenerateSchemaSnapshots(cfg, src);
+            }
+
             var collector = new DBCollector(src, cfg.ServiceName, true);
             collector.Collect(standardCollections.ToArray());
             collector.Collect(customCollections);
 
-
             if (CollectAgent.S3Path != null)
             {
                 op.Complete();
@@ -95,11 +101,11 @@ public override async Task<DataSet> Process(CollectionConfig cfg, Guid handle)
                 {
                     standardCollections.Add(CollectionType.DriversWMI);
                 }
-                else if (string.Equals(type,"QueryPlans", StringComparison.OrdinalIgnoreCase) || string.Equals(type, "QueryText", StringComparison.OrdinalIgnoreCase))
+                else if (string.Equals(type, "QueryPlans", StringComparison.OrdinalIgnoreCase) || string.Equals(type, "QueryText", StringComparison.OrdinalIgnoreCase))
                 {
                     throw new ArgumentException("QueryPlan and QueryText are collected as part of the RunningQueries collection");
                 }
-                else if (string.Equals(type,"SlowQueriesStats", StringComparison.OrdinalIgnoreCase))
+                else if (string.Equals(type, "SlowQueriesStats", StringComparison.OrdinalIgnoreCase))
                 {
                     throw new ArgumentException("SlowQueriesStats is collected as part of the SlowQueries collection");
                 }

DBADash/SchemaSnapshotDB.cs

@@ -3,10 +3,12 @@
 using Microsoft.SqlServer.Management.Smo.Broker;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
+using Serilog;
 using SerilogTimings;
 using System;
 using System.Data;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace DBADash
 {
@@ -35,10 +37,8 @@ public enum SchemaSnapshotDBObjectTypes
         Trigger
     }
 
-
     public class SchemaSnapshotDBOptions
     {
-
         public bool DriAll = true;
         public bool Triggers = true;
         public bool FullTextIndexes = true;
@@ -56,7 +56,6 @@ public static SchemaSnapshotDBObjectTypes[] DefaultSchemaSnapshotDBObjectTypes()
             return new SchemaSnapshotDBObjectTypes[] { SchemaSnapshotDBObjectTypes.Database, SchemaSnapshotDBObjectTypes.Aggregate, SchemaSnapshotDBObjectTypes.Assembly, SchemaSnapshotDBObjectTypes.DDLTrigger, SchemaSnapshotDBObjectTypes.Schema, SchemaSnapshotDBObjectTypes.StoredProcedures, SchemaSnapshotDBObjectTypes.Synonym, SchemaSnapshotDBObjectTypes.Tables, SchemaSnapshotDBObjectTypes.UserDefinedDataType, SchemaSnapshotDBObjectTypes.UserDefinedFunction, SchemaSnapshotDBObjectTypes.UserDefinedTableType, SchemaSnapshotDBObjectTypes.UserDefinedType, SchemaSnapshotDBObjectTypes.View, SchemaSnapshotDBObjectTypes.XMLSchema, SchemaSnapshotDBObjectTypes.Roles, SchemaSnapshotDBObjectTypes.ApplicationRole, SchemaSnapshotDBObjectTypes.Sequence, SchemaSnapshotDBObjectTypes.ServiceBroker, SchemaSnapshotDBObjectTypes.Trigger };
         }
 
-
         public ScriptingOptions ScriptOptions()
         {
             var so = new ScriptingOptions
@@ -73,18 +72,16 @@ public ScriptingOptions ScriptOptions()
             };
             return so;
         }
-
     }
 
     public class SchemaSnapshotDB : SMOBaseClass
     {
         public SchemaSnapshotDB(DBADashConnection source, SchemaSnapshotDBOptions options) : base(source, options)
         {
-
         }
+
         public SchemaSnapshotDB(DBADashConnection source) : base(source)
         {
-
         }
 
         private static DataTable RgDTSchema()
@@ -162,9 +159,7 @@ public DataTable ResourceGovernorConfiguration()
                     row["max_outstanding_io_per_volume"] = rg.ServerVersion.Major >= 12 ? rg.MaxOutstandingIOPerVolume : 0; // Supported 2014 and later
                     row["script"] = StringCollectionToString(sc);
                     dtRG.Rows.Add(row);
-
                 }
-
             }
             return dtRG;
         }
@@ -182,7 +177,6 @@ public static string ObjectDDL(string connectionString, string objectName)
 
         public DataTable SnapshotDB(string DBName)
         {
-
             using (var cn = new Microsoft.Data.SqlClient.SqlConnection(ConnectionString))
             using (var opSS = Operation.Begin("Schema snapshot {DBame}", DBName))
             {
@@ -365,12 +359,10 @@ public DataTable SnapshotDB(string DBName)
                                 op.Complete();
                             }
                         }
-
                     }
                 }
                 opSS.Complete();
                 return dtSchema;
-
             }
         }
 
@@ -478,7 +470,7 @@ private void AddServiceBroker(Database db, DataTable dtSchema)
                 r["ObjectDateModified"] = "1900-01-01";
                 dtSchema.Rows.Add(r);
             }
-            if (db.Parent.VersionMajor >= 10) // Broker priorities supported on SQL 2008 
+            if (db.Parent.VersionMajor >= 10) // Broker priorities supported on SQL 2008
             {
                 foreach (BrokerPriority p in db.ServiceBroker.Priorities)
                 {
@@ -520,7 +512,6 @@ private void AddSeq(Database db, DataTable dtSchema)
             }
         }
 
-
         private void AddAppRole(Database db, DataTable dtSchema)
         {
             foreach (ApplicationRole ar in db.ApplicationRoles)
@@ -668,7 +659,6 @@ private void AddSchema(Database db, DataTable dtSchema)
 
         private void AddSynonyms(Database db, DataTable dtSchema)
         {
-
             foreach (Synonym s in db.Synonyms)
             {
                 var r = dtSchema.NewRow();
@@ -715,7 +705,6 @@ private void AddDDLTriggers(Database db, DataTable dtSchema)
 
         private void AddUserDefinedTableType(Database db, DataTable dtSchema)
         {
-
             db.PrefetchObjects(typeof(UserDefinedTableType), ScriptingOptions);
             foreach (UserDefinedTableType t in db.UserDefinedTableTypes)
             {
@@ -799,6 +788,7 @@ private void AddUDFs(Database db, DataTable dtSchema)
                         case UserDefinedFunctionType.Inline:
                             r["ObjectType"] = "IF";
                             break;
+
                         case UserDefinedFunctionType.Scalar:
                             if (f.ImplementationType == ImplementationType.SqlClr)
                             {
@@ -809,6 +799,7 @@ private void AddUDFs(Database db, DataTable dtSchema)
                                 r["ObjectType"] = "FN";
                             }
                             break;
+
                         case UserDefinedFunctionType.Table:
                             if (f.ImplementationType == ImplementationType.SqlClr)
                             {
@@ -819,6 +810,7 @@ private void AddUDFs(Database db, DataTable dtSchema)
                                 r["ObjectType"] = "TF";
                             }
                             break;
+
                         default:
                             r["ObjectType"] = "??";
                             break;
@@ -985,8 +977,54 @@ private void AddTriggers(TriggerCollection triggers, string schema, DataTable dt
             }
         }
 
+        public static async Task GenerateSchemaSnapshots(CollectionConfig Config, DBADashSource src)
+        {
+            var connectionString = src.GetSource();
+            SqlConnectionStringBuilder builder = new(connectionString);
+
+            var collector = new DBCollector(src, Config.ServiceName);
+            var dsSnapshot = collector.Data;
+            var dbs = src.SchemaSnapshotDBs.Split(',');
+
+            await using var cn = new SqlConnection(connectionString);
+            var ss = new SchemaSnapshotDB(src.SourceConnection, Config.SchemaSnapshotOptions);
+            var instance = new Server(new Microsoft.SqlServer.Management.Common.ServerConnection(cn));
+
+            if (instance.ServerType == Microsoft.SqlServer.Management.Common.DatabaseEngineType.SqlAzureDatabase && builder.InitialCatalog is null or "master" or "")
+            {
+                return;
+            }
+            Log.Information("DB Snapshots from instance {source}", src.SourceConnection.ConnectionForPrint);
+
+            foreach (Database db in instance.Databases)
+            {
+                if (!db.IsUpdateable || !db.IsAccessible || db.IsSystemObject != false) continue;
 
+                var include = dbs.Any(strDB => strDB == "*" || string.Equals(strDB, db.Name, StringComparison.OrdinalIgnoreCase)) &&
+                              !dbs.Any(strDB => strDB.StartsWith('-') && string.Equals(strDB[1..], db.Name, StringComparison.OrdinalIgnoreCase));
+                if (!include) continue;
 
+                Log.Information("DB Snapshot {db} from instance {instance}", db.Name, builder.DataSource);
+                var StartTime = DateTime.UtcNow;
+                try
+                {
+                    var dt = ss.SnapshotDB(db.Name);
+                    var EndTime = DateTime.UtcNow;
+                    dt.TableName = "Snapshot_" + db.Name;
+                    dt.ExtendedProperties.Add("StartTimeBin", StartTime.ToBinary().ToString());
+                    dt.ExtendedProperties.Add("EndTimeBin", EndTime.ToBinary().ToString());
+                    dt.ExtendedProperties.Add("SnapshotOptions", JsonConvert.SerializeObject(Config.SchemaSnapshotOptions));
+                    dsSnapshot.Tables.Add(dt);
 
+                    var fileName = DBADashSource.GenerateFileName(src.SourceConnection.ConnectionForFileName);
+                    await DestinationHandling.WriteAllDestinations(dsSnapshot, src, fileName, Config);
+                    dsSnapshot.Tables.Remove(dt);
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, "Error creating schema snapshot {db}", db.Name);
+                }
+            }
+        }
     }
-}
+}

DBADashDB/dbo/Stored Procedures/DDLSnapshot_Add.sql

@@ -1,4 +1,4 @@
-CREATE   PROC [dbo].[DDLSnapshot_Add](
+CREATE   PROC dbo.DDLSnapshot_Add(
 	@ss dbo.DDLSnapshot READONLY,
 	@ConnectionId SYSNAME=NULL,
 	@InstanceID INT=NULL,
@@ -11,6 +11,7 @@
 )
 AS
 SET XACT_ABORT ON
+DECLARE @Ref VARCHAR(30)='SchemaSnapshot'
 DECLARE @Count INT=0
 DECLARE @DatabaseId INT
 DECLARE @ValidatedSnapshotDate DATETIME2(3)
@@ -194,4 +195,7 @@ BEGIN;
 		INSERT INTO dbo.DDLSnapshotsLog(DatabaseID,SnapshotDate,ValidatedSnapshot,EndDate,Duration)
 		SELECT @DatabaseId,@SnapshotDate,@ValidatedSnapshotDate,@EndTime,DATEDIFF(ms,@StartTime,@EndTime)
 	COMMIT
-END
+END
+EXEC dbo.CollectionDates_Upd @InstanceID = @InstanceID,  
+										@Reference = @Ref,
+										@SnapshotDate = @SnapshotDate

DBADashGUI/Messaging/CollectionMessaging.cs

@@ -26,19 +26,19 @@ internal static class CollectionMessaging
         private const int PendingRequestsThreshold = 10;
         private static readonly Dictionary<(string instanceName, string type), DateTime> LastTriggeredTimes = new();
         private static readonly object LockObject = new();
-        private const int CollectionDialogLifetime = 120;
+        private const int CollectionDialogLifetime = 600;
 
-        public static async Task TriggerCollection(string connectionID, CollectionType type,int collectAgentID, int importAgentID, ISetStatus control)
+        public static async Task TriggerCollection(string connectionID, CollectionType type, int collectAgentID, int importAgentID, ISetStatus control)
         {
-            await TriggerCollection(connectionID, new List<string>() { Enum.GetName(type) }, collectAgentID,importAgentID, control);
+            await TriggerCollection(connectionID, new List<string>() { Enum.GetName(type) }, collectAgentID, importAgentID, control);
         }
 
-        public static async Task TriggerCollection(string connectionID, string type, int collectAgentID,int importAgentID, ISetStatus control)
+        public static async Task TriggerCollection(string connectionID, string type, int collectAgentID, int importAgentID, ISetStatus control)
         {
-            await TriggerCollection(connectionID, new List<string>() { type }, collectAgentID,importAgentID, control);
+            await TriggerCollection(connectionID, new List<string>() { type }, collectAgentID, importAgentID, control);
         }
 
-        public static async Task TriggerCollection(string connectionID, List<string> types, int collectAgentID,int importAgentID,
+        public static async Task TriggerCollection(string connectionID, List<string> types, int collectAgentID, int importAgentID,
             ISetStatus control)
         {
             if (PendingRequests >= PendingRequestsThreshold)
@@ -63,9 +63,9 @@ public static async Task TriggerCollection(string connectionID, List<string> typ
 
             var typesString = string.Join(", ", types.Select(s => s.ToString()));
             var messageBase = $"{typesString} collection for {connectionID}: ";
-            var collectAgent = DBADashAgent.GetDBADashAgent(Common.ConnectionString,collectAgentID);
-            var importAgent = DBADashAgent.GetDBADashAgent(Common.ConnectionString,importAgentID);
-            var x = new CollectionMessage(types, connectionID) {CollectAgent = collectAgent,ImportAgent = importAgent };
+            var collectAgent = DBADashAgent.GetDBADashAgent(Common.ConnectionString, collectAgentID);
+            var importAgent = DBADashAgent.GetDBADashAgent(Common.ConnectionString, importAgentID);
+            var x = new CollectionMessage(types, connectionID) { CollectAgent = collectAgent, ImportAgent = importAgent };
 
             var payload = x.Serialize();
             var messageGroup = Guid.NewGuid();

DBADashService/SchemaSnapshotJob.cs

@@ -11,74 +11,14 @@ namespace DBADashService
 {
     public class SchemaSnapshotJob : IJob
     {
-        public Task Execute(IJobExecutionContext context)
+        public async Task Execute(IJobExecutionContext context)
         {
-            JobDataMap dataMap = context.JobDetail.JobDataMap;
+            var sourceString = context.JobDetail.JobDataMap.GetString("CFG");
+            if (sourceString == null) return;
 
-            var cfg = JsonConvert.DeserializeObject<DBADashSource>(dataMap.GetString("CFG"));
-            var schemaSnapshotDBs = dataMap.GetString("SchemaSnapshotDBs");
-            string connectionString = cfg.GetSource();
-            SqlConnectionStringBuilder builder = new(connectionString);
+            var src = JsonConvert.DeserializeObject<DBADashSource>(sourceString);
 
-            var collector = new DBCollector(cfg, SchedulerServiceConfig.Config.ServiceName);
-            var dsSnapshot = collector.Data;
-            var dbs = schemaSnapshotDBs.Split(',');
-
-            using var cn = new SqlConnection(connectionString);
-            var ss = new SchemaSnapshotDB(cfg.SourceConnection, SchedulerServiceConfig.Config.SchemaSnapshotOptions);
-            var instance = new Microsoft.SqlServer.Management.Smo.Server(new Microsoft.SqlServer.Management.Common.ServerConnection(cn));
-
-            if (instance.ServerType == Microsoft.SqlServer.Management.Common.DatabaseEngineType.SqlAzureDatabase && (builder.InitialCatalog == null || builder.InitialCatalog == "master" || builder.InitialCatalog == ""))
-            {
-                return Task.CompletedTask;
-            }
-            Log.Information("DB Snapshots from instance {source}", cfg.SourceConnection.ConnectionForPrint);
-            foreach (Database db in instance.Databases)
-            {
-                bool include = false;
-                if (db.IsUpdateable && db.IsAccessible && db.IsSystemObject == false)
-                {
-                    foreach (string strDB in dbs)
-                    {
-                        if (strDB.StartsWith("-"))
-                        {
-                            if (db.Name == strDB[1..])
-                            {
-                                include = false;
-                                break;
-                            }
-                        }
-                        if (strDB == db.Name || strDB == "*")
-                        {
-                            include = true;
-                        }
-                    }
-                    if (include)
-                    {
-                        Log.Information("DB Snapshot {db} from instance {instance}", db.Name, builder.DataSource);
-                        DateTime StartTime = DateTime.UtcNow;
-                        try
-                        {
-                            var dt = ss.SnapshotDB(db.Name);
-                            DateTime EndTime = DateTime.UtcNow;
-                            dt.TableName = "Snapshot_" + db.Name;
-                            dt.ExtendedProperties.Add("StartTimeBin", StartTime.ToBinary().ToString());
-                            dt.ExtendedProperties.Add("EndTimeBin", EndTime.ToBinary().ToString());
-                            dt.ExtendedProperties.Add("SnapshotOptions", JsonConvert.SerializeObject(SchedulerServiceConfig.Config.SchemaSnapshotOptions));
-                            dsSnapshot.Tables.Add(dt);
-
-                            string fileName = DBADashSource.GenerateFileName(cfg.SourceConnection.ConnectionForFileName);
-                            DestinationHandling.WriteAllDestinations(dsSnapshot, cfg, fileName, SchedulerServiceConfig.Config).Wait();
-                            dsSnapshot.Tables.Remove(dt);
-                        }
-                        catch (Exception ex)
-                        {
-                            Log.Error(ex, "Error creating schema snapshot {db}", db.Name);
-                        }
-                    }
-                }
-            }
-            return Task.CompletedTask;
+            await SchemaSnapshotDB.GenerateSchemaSnapshots(SchedulerServiceConfig.Config, src);
         }
     }
 }