diff --git a/DNN Platform/Connectors/Azure/Components/AzureConnector.cs b/DNN Platform/Connectors/Azure/Components/AzureConnector.cs index 5bc257ff53c..0118b913b90 100644 --- a/DNN Platform/Connectors/Azure/Components/AzureConnector.cs +++ b/DNN Platform/Connectors/Azure/Components/AzureConnector.cs @@ -214,6 +214,10 @@ public bool SaveConfig(int portalId, IDictionary values, ref boo { folderMapping.MappingName = DisplayName; } + if (!folderMapping.FolderMappingSettings.ContainsKey(Constants.SyncBatchSize)) + { + folderMapping.FolderMappingSettings[Constants.SyncBatchSize] = Constants.DefaultSyncBatchSize.ToString(); + } FolderMappingController.Instance.UpdateFolderMapping(folderMapping); diff --git a/DNN Platform/Connectors/Azure/Components/Constants.cs b/DNN Platform/Connectors/Azure/Components/Constants.cs index 47b15bc3494..b557d4c7a0c 100644 --- a/DNN Platform/Connectors/Azure/Components/Constants.cs +++ b/DNN Platform/Connectors/Azure/Components/Constants.cs @@ -15,6 +15,8 @@ internal class Constants public const string AzureContainerName = "Container"; public const string DirectLink = "DirectLink"; public const string UseHttps = "UseHttps"; + public const string SyncBatchSize = "SyncBatchSize"; + public const int DefaultSyncBatchSize = 2048; public const string LocalResourceFile = "~/DesktopModules/Connectors/Azure/App_LocalResources/SharedResources.resx"; diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/App_LocalResources/Settings.ascx.resx b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/App_LocalResources/Settings.ascx.resx index 9c665f3c788..2740086b0f0 100644 --- a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/App_LocalResources/Settings.ascx.resx +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/App_LocalResources/Settings.ascx.resx @@ -200,5 +200,14 @@ Custom Domain - + + + Syhchronization batch size must be a positive integer number + + + Sets the maximum number of items that can be fetched from Azure storage on each request during folder synchronization. The lower the number, the more round trips will be necessary to fetch all items. Large numbers will lead to timeouts in case of large repositories. + + + Synchronization Batch Size + \ No newline at end of file diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/AzureFolderProvider.cs b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/AzureFolderProvider.cs index f4c65c6da25..38e96b10791 100644 --- a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/AzureFolderProvider.cs +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/AzureFolderProvider.cs @@ -28,17 +28,6 @@ namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider /// public class AzureFolderProvider : BaseRemoteStorageProvider { - #region Constants - - private const string AccountName = "AccountName"; - private const string AccountKey = "AccountKey"; - private const string Container = "Container"; - private const string UseHttps = "UseHttps"; - private const string DirectLink = "DirectLink"; - private const string CustomDomain = "CustomDomain"; - private const string PlaceHolderFileName = "dotnetnuke.placeholder.donotdelete"; - - #endregion public AzureFolderProvider() { @@ -70,10 +59,10 @@ private static void CheckSettings(FolderMappingInfo folderMapping) { var settings = folderMapping.FolderMappingSettings; - if (string.IsNullOrEmpty((string)settings[AccountName]) || - string.IsNullOrEmpty((string)settings[AccountKey]) || - string.IsNullOrEmpty((string)settings[Container]) || - string.IsNullOrEmpty((string)settings[UseHttps])) + if (string.IsNullOrEmpty((string)settings[Constants.AccountName]) || + string.IsNullOrEmpty((string)settings[Constants.AccountKey]) || + string.IsNullOrEmpty((string)settings[Constants.Container]) || + string.IsNullOrEmpty((string)settings[Constants.UseHttps])) { throw new Exception("Settings cannot be found."); } @@ -84,10 +73,10 @@ private CloudBlobContainer GetContainer(FolderMappingInfo folderMapping) CheckSettings(folderMapping); - var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountName); - var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountKey); - var container = GetSetting(folderMapping, Container); - var useHttps = GetBooleanSetting(folderMapping, UseHttps); + var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountName); + var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountKey); + var container = GetSetting(folderMapping, Constants.Container); + var useHttps = GetBooleanSetting(folderMapping, Constants.UseHttps); var sc = new StorageCredentials(accountName, accountKey); var csa = new CloudStorageAccount(sc, useHttps); @@ -121,7 +110,7 @@ protected override void DeleteFileInternal(FolderMappingInfo folderMapping, stri protected override void DeleteFolderInternal(FolderMappingInfo folderMapping, IFolderInfo folder) { - DeleteFileInternal(folderMapping, folder.MappedPath + PlaceHolderFileName); + DeleteFileInternal(folderMapping, folder.MappedPath + Constants.PlaceHolderFileName); } protected override Stream GetFileStreamInternal(FolderMappingInfo folderMapping, string uri) @@ -147,6 +136,7 @@ protected override IList GetObjectList(FolderMappingInfo fol c => { var container = GetContainer(folderMapping); + var synchBatchSize = Convert.ToInt32(GetSetting(folderMapping, Constants.SyncBatchSize) ?? Constants.DefaultSyncBatchSize.ToString()); BlobContinuationToken continuationToken = null; BlobResultSegment resultSegment = null; @@ -156,7 +146,7 @@ protected override IList GetObjectList(FolderMappingInfo fol { //This overload allows control of the page size. You can return all remaining results by passing null for the maxResults parameter, //or by calling a different overload. - resultSegment = container.ListBlobsSegmented("", true, BlobListingDetails.All, 10, continuationToken, null, null); + resultSegment = container.ListBlobsSegmented("", true, BlobListingDetails.All, synchBatchSize, continuationToken, null, null); foreach (var blobItem in resultSegment.Results) { list.Add(new AzureRemoteStorageItem {Blob = new AzureBlob(blobItem as CloudBlob)}); @@ -228,7 +218,7 @@ public override void AddFolder(string folderPath, FolderMappingInfo folderMappin Requires.NotNull("folderPath", folderPath); Requires.NotNull("folderMapping", folderMapping); - UpdateFileInternal(new MemoryStream(), folderMapping, mappedPath + PlaceHolderFileName); + UpdateFileInternal(new MemoryStream(), folderMapping, mappedPath + Constants.PlaceHolderFileName); } /// @@ -239,7 +229,7 @@ public override string GetFileUrl(IFileInfo file) Requires.NotNull("file", file); var folderMapping = FolderMappingController.Instance.GetFolderMapping(file.PortalId, file.FolderMappingID); - var directLink = string.IsNullOrEmpty(GetSetting(folderMapping, DirectLink)) || GetSetting(folderMapping, DirectLink).ToLowerInvariant() == "true"; + var directLink = string.IsNullOrEmpty(GetSetting(folderMapping, Constants.DirectLink)) || GetSetting(folderMapping, Constants.DirectLink).ToLowerInvariant() == "true"; if (directLink) { @@ -249,7 +239,7 @@ public override string GetFileUrl(IFileInfo file) var container = GetContainer(folderMapping); var blob = container.GetBlobReference(uri); var absuri = blob.Uri.AbsoluteUri; - var customDomain = GetEncryptedSetting(folderMapping.FolderMappingSettings, CustomDomain); + var customDomain = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.CustomDomain); if (!string.IsNullOrEmpty(customDomain)) { @@ -318,9 +308,9 @@ public override string GetFolderProviderIconPath() public List GetAllContainers(FolderMappingInfo folderMapping) { List containers = new List(); - var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountName); - var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountKey); - var useHttps = GetBooleanSetting(folderMapping, UseHttps); + var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountName); + var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountKey); + var useHttps = GetBooleanSetting(folderMapping, Constants.UseHttps); var sc = new StorageCredentials(accountName, accountKey); var csa = new CloudStorageAccount(sc, useHttps); diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Constants.cs b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Constants.cs new file mode 100644 index 00000000000..af70b9ad792 --- /dev/null +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Constants.cs @@ -0,0 +1,23 @@ +#region Copyright +// DotNetNuke® - http://www.dotnetnuke.com +// Copyright (c) 2002-2018 +// by DotNetNuke Corporation +// All Rights Reserved +#endregion + +namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider +{ + internal class Constants + { + public const string AccountName = "AccountName"; + public const string AccountKey = "AccountKey"; + public const string Container = "Container"; + public const string UseHttps = "UseHttps"; + public const string DirectLink = "DirectLink"; + public const string CustomDomain = "CustomDomain"; + public const string FolderProviderType = "AzureFolderProvider"; + public const string PlaceHolderFileName = "dotnetnuke.placeholder.donotdelete"; + public const string SyncBatchSize = "SyncBatchSize"; + public const int DefaultSyncBatchSize = 2048; + } +} diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx index ecfab3774f5..4226468af5d 100644 --- a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx @@ -38,4 +38,10 @@ + +
+ + +
\ No newline at end of file diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.cs b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.cs index 5e523e818da..cb572cc370f 100644 --- a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.cs +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.cs @@ -12,7 +12,6 @@ using DotNetNuke.Instrumentation; using DotNetNuke.Services.FileSystem; using DotNetNuke.Services.Localization; -using Microsoft.WindowsAzure; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Auth; using Microsoft.WindowsAzure.Storage.Blob; @@ -25,18 +24,7 @@ namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider public partial class Settings : FolderMappingSettingsControlBase { private static readonly ILog Logger = LoggerSource.Instance.GetLogger(typeof (Settings)); - #region Constants - private const string AccountName = "AccountName"; - private const string AccountKey = "AccountKey"; - private const string Container = "Container"; - private const string UseHttps = "UseHttps"; - private const string DirectLink = "DirectLink"; - private const string CustomDomain = "CustomDomain"; - private const string FolderProviderType = "AzureFolderProvider"; - - #endregion - #region Overrided Methods /// @@ -45,24 +33,24 @@ public partial class Settings : FolderMappingSettingsControlBase /// The Hashtable containing the folder mapping settings. public override void LoadSettings(Hashtable folderMappingSettings) { - var folderProvider = FolderProvider.Instance(FolderProviderType); - if (folderMappingSettings.ContainsKey(AccountName)) + var folderProvider = FolderProvider.Instance(Constants.FolderProviderType); + if (folderMappingSettings.ContainsKey(Constants.AccountName)) { - tbAccountName.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, AccountName); + tbAccountName.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.AccountName); } - if (folderMappingSettings.ContainsKey(AccountKey)) + if (folderMappingSettings.ContainsKey(Constants.AccountKey)) { - tbAccountKey.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, AccountKey); + tbAccountKey.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.AccountKey); } if (tbAccountName.Text.Length > 0 && tbAccountKey.Text.Length > 0) { var bucketName = ""; - if (folderMappingSettings.ContainsKey(Container)) + if (folderMappingSettings.ContainsKey(Constants.Container)) { - bucketName = folderMappingSettings[Container].ToString(); + bucketName = folderMappingSettings[Constants.Container].ToString(); } LoadContainers(); @@ -75,16 +63,25 @@ public override void LoadSettings(Hashtable folderMappingSettings) } } - if (folderMappingSettings.ContainsKey(UseHttps)) + if (folderMappingSettings.ContainsKey(Constants.UseHttps)) { - chkUseHttps.Checked = bool.Parse(folderMappingSettings[UseHttps].ToString()); + chkUseHttps.Checked = bool.Parse(folderMappingSettings[Constants.UseHttps].ToString()); } - chkDirectLink.Checked = !folderMappingSettings.ContainsKey(DirectLink) || folderMappingSettings[DirectLink].ToString().ToLowerInvariant() == "true"; + chkDirectLink.Checked = !folderMappingSettings.ContainsKey(Constants.DirectLink) || folderMappingSettings[Constants.DirectLink].ToString().ToLowerInvariant() == "true"; + + if (folderMappingSettings.ContainsKey(Constants.CustomDomain)) + { + tbCustomDomain.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.CustomDomain); + } - if (folderMappingSettings.ContainsKey(CustomDomain)) + if (folderMappingSettings.ContainsKey(Constants.SyncBatchSize) && folderMappingSettings[Constants.SyncBatchSize] != null) { - tbCustomDomain.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, CustomDomain); + tbSyncBatchSize.Text = folderMappingSettings[Constants.SyncBatchSize].ToString(); + } + else + { + tbSyncBatchSize.Text = Constants.DefaultSyncBatchSize.ToString(); } } @@ -109,6 +106,7 @@ public override void UpdateSettings(int folderMappingID) var container = GetContainer(); var useHttps = GetUseHttps(); var customDomain = GetCustomDomain(); + var synchBatchSize = GetSynchBatchSize(); if (AreThereFolderMappingsWithSameSettings(folderMapping, accountName, container)) { @@ -118,12 +116,13 @@ public override void UpdateSettings(int folderMappingID) throw new Exception(); } - folderMapping.FolderMappingSettings[AccountName] = accountName; - folderMapping.FolderMappingSettings[AccountKey] = accountKey; - folderMapping.FolderMappingSettings[Container] = container; - folderMapping.FolderMappingSettings[UseHttps] = useHttps; - folderMapping.FolderMappingSettings[DirectLink] = chkDirectLink.Checked; - folderMapping.FolderMappingSettings[CustomDomain] = customDomain; + folderMapping.FolderMappingSettings[Constants.AccountName] = accountName; + folderMapping.FolderMappingSettings[Constants.AccountKey] = accountKey; + folderMapping.FolderMappingSettings[Constants.Container] = container; + folderMapping.FolderMappingSettings[Constants.UseHttps] = useHttps; + folderMapping.FolderMappingSettings[Constants.DirectLink] = chkDirectLink.Checked; + folderMapping.FolderMappingSettings[Constants.CustomDomain] = customDomain; + folderMapping.FolderMappingSettings[Constants.SyncBatchSize] = synchBatchSize; folderMappingController.UpdateFolderMapping(folderMapping); } @@ -139,8 +138,8 @@ private static bool AreThereFolderMappingsWithSameSettings(FolderMappingInfo fol return folderMappings .Where(fm => fm.FolderMappingID != folderMapping.FolderMappingID && fm.FolderProviderType == folderMapping.FolderProviderType) - .Any(fm => fm.FolderMappingSettings[AccountName].ToString().Equals(accountName, StringComparison.InvariantCulture) && - fm.FolderMappingSettings[Container].ToString().Equals(container, StringComparison.InvariantCultureIgnoreCase)); + .Any(fm => fm.FolderMappingSettings[Constants.AccountName].ToString().Equals(accountName, StringComparison.InvariantCulture) && + fm.FolderMappingSettings[Constants.Container].ToString().Equals(container, StringComparison.InvariantCultureIgnoreCase)); } private string GetUseHttps() @@ -246,17 +245,22 @@ private bool CreateContainer(string containerName) private string GetAccountKey() { - return FolderProvider.Instance(FolderProviderType).EncryptValue(tbAccountKey.Text); + return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbAccountKey.Text); } private string GetAccountName() { - return FolderProvider.Instance(FolderProviderType).EncryptValue(tbAccountName.Text); + return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbAccountName.Text); } private string GetCustomDomain() { - return FolderProvider.Instance(FolderProviderType).EncryptValue(tbCustomDomain.Text); + return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbCustomDomain.Text); + } + + private string GetSynchBatchSize() + { + return tbSyncBatchSize.Text; } private void LoadContainers() diff --git a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.designer.cs b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.designer.cs index 196235801fe..767f844584e 100644 --- a/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.designer.cs +++ b/DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.designer.cs @@ -200,5 +200,32 @@ public partial class Settings { /// To modify move field declaration from designer file to code-behind file. /// protected global::System.Web.UI.WebControls.RegularExpressionValidator rgBlobEndpoint; + + /// + /// plSyncBatchSize control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.UserControl plSyncBatchSize; + + /// + /// tbSyncBatchSize control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.TextBox tbSyncBatchSize; + + /// + /// rgInteger control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.RegularExpressionValidator rgInteger; } } diff --git a/DNN Platform/Providers/FolderProviders/DotNetNuke.Providers.FolderProviders.csproj b/DNN Platform/Providers/FolderProviders/DotNetNuke.Providers.FolderProviders.csproj index 06f54f8df41..3761b73e317 100644 --- a/DNN Platform/Providers/FolderProviders/DotNetNuke.Providers.FolderProviders.csproj +++ b/DNN Platform/Providers/FolderProviders/DotNetNuke.Providers.FolderProviders.csproj @@ -83,6 +83,7 @@ + Settings.ascx ASPXCodeBehind