Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Digital Assets: Azure folders are slow to open #2698

Merged
merged 3 commits into from
Apr 16, 2019
Merged

Digital Assets: Azure folders are slow to open #2698

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ea26f83
DNN-28922 Added SyncBatchSize setting to the UI to make it works to f…
mikebigun Apr 16, 2019
3ae84e3
DNN-28922 Added SyncBatchSize setting to Azure connector
mikebigun Apr 16, 2019
601d41f
Minor suggestions
daguiler Apr 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions DNN Platform/Connectors/Azure/Components/AzureConnector.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ public bool SaveConfig(int portalId, IDictionary<string, string> values, ref boo
{
folderMapping.MappingName = DisplayName;
}
if (!folderMapping.FolderMappingSettings.ContainsKey(Constants.SyncBatchSize))
{
folderMapping.FolderMappingSettings[Constants.SyncBatchSize] = Constants.DefaultSyncBatchSize.ToString();
}

FolderMappingController.Instance.UpdateFolderMapping(folderMapping);

Expand Down
2 changes: 2 additions & 0 deletions DNN Platform/Connectors/Azure/Components/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ internal class Constants
public const string AzureContainerName = "Container";
public const string DirectLink = "DirectLink";
public const string UseHttps = "UseHttps";
public const string SyncBatchSize = "SyncBatchSize";
public const int DefaultSyncBatchSize = 2048;

public const string LocalResourceFile =
"~/DesktopModules/Connectors/Azure/App_LocalResources/SharedResources.resx";
Expand Down
11 changes: 10 additions & 1 deletion ...tform/Providers/FolderProviders/AzureFolderProvider/App_LocalResources/Settings.ascx.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,5 +200,14 @@
</data>
<data name="plCustomDomain.Text" xml:space="preserve">
<value>Custom Domain</value>
</data>
</data>
<data name="tbSyncBatchSize.Error" xml:space="preserve">
<value>The syhchronization batch size should be numeric value</value>
daguiler marked this conversation as resolved.
Show resolved Hide resolved
</data>
<data name="plSyncBatchSize.Help" xml:space="preserve">
<value>Sets the maximum number of items that can be fetched from Azure storage on each request during folder synchronization. The lower the number, the more round trips will be necessary to fetch all items. Large numbers will lead to timeouts in case of large repositories.</value>
</data>
<data name="plSyncBatchSize.Text" xml:space="preserve">
<value>Synchronization Batch Size</value>
</data>
</root>
44 changes: 17 additions & 27 deletions DNN Platform/Providers/FolderProviders/AzureFolderProvider/AzureFolderProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,6 @@ namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider
///</summary>
public class AzureFolderProvider : BaseRemoteStorageProvider
{
#region Constants

private const string AccountName = "AccountName";
private const string AccountKey = "AccountKey";
private const string Container = "Container";
private const string UseHttps = "UseHttps";
private const string DirectLink = "DirectLink";
private const string CustomDomain = "CustomDomain";
private const string PlaceHolderFileName = "dotnetnuke.placeholder.donotdelete";

#endregion

public AzureFolderProvider()
{
Expand Down Expand Up @@ -70,10 +59,10 @@ private static void CheckSettings(FolderMappingInfo folderMapping)
{
var settings = folderMapping.FolderMappingSettings;

if (string.IsNullOrEmpty((string)settings[AccountName]) ||
string.IsNullOrEmpty((string)settings[AccountKey]) ||
string.IsNullOrEmpty((string)settings[Container]) ||
string.IsNullOrEmpty((string)settings[UseHttps]))
if (string.IsNullOrEmpty((string)settings[Constants.AccountName]) ||
string.IsNullOrEmpty((string)settings[Constants.AccountKey]) ||
string.IsNullOrEmpty((string)settings[Constants.Container]) ||
string.IsNullOrEmpty((string)settings[Constants.UseHttps]))
{
throw new Exception("Settings cannot be found.");
}
Expand All @@ -84,10 +73,10 @@ private CloudBlobContainer GetContainer(FolderMappingInfo folderMapping)

CheckSettings(folderMapping);

var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountName);
var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountKey);
var container = GetSetting(folderMapping, Container);
var useHttps = GetBooleanSetting(folderMapping, UseHttps);
var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountName);
var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountKey);
var container = GetSetting(folderMapping, Constants.Container);
var useHttps = GetBooleanSetting(folderMapping, Constants.UseHttps);

var sc = new StorageCredentials(accountName, accountKey);
var csa = new CloudStorageAccount(sc, useHttps);
Expand Down Expand Up @@ -121,7 +110,7 @@ protected override void DeleteFileInternal(FolderMappingInfo folderMapping, stri

protected override void DeleteFolderInternal(FolderMappingInfo folderMapping, IFolderInfo folder)
{
DeleteFileInternal(folderMapping, folder.MappedPath + PlaceHolderFileName);
DeleteFileInternal(folderMapping, folder.MappedPath + Constants.PlaceHolderFileName);
}

protected override Stream GetFileStreamInternal(FolderMappingInfo folderMapping, string uri)
Expand All @@ -147,6 +136,7 @@ protected override IList<IRemoteStorageItem> GetObjectList(FolderMappingInfo fol
c =>
{
var container = GetContainer(folderMapping);
var synchBatchSize = Convert.ToInt32(GetSetting(folderMapping, Constants.SyncBatchSize) ?? Constants.DefaultSyncBatchSize.ToString());

BlobContinuationToken continuationToken = null;
BlobResultSegment resultSegment = null;
Expand All @@ -156,7 +146,7 @@ protected override IList<IRemoteStorageItem> GetObjectList(FolderMappingInfo fol
{
//This overload allows control of the page size. You can return all remaining results by passing null for the maxResults parameter,
//or by calling a different overload.
resultSegment = container.ListBlobsSegmented("", true, BlobListingDetails.All, 10, continuationToken, null, null);
resultSegment = container.ListBlobsSegmented("", true, BlobListingDetails.All, synchBatchSize, continuationToken, null, null);
foreach (var blobItem in resultSegment.Results)
{
list.Add(new AzureRemoteStorageItem {Blob = new AzureBlob(blobItem as CloudBlob)});
Expand Down Expand Up @@ -228,7 +218,7 @@ public override void AddFolder(string folderPath, FolderMappingInfo folderMappin
Requires.NotNull("folderPath", folderPath);
Requires.NotNull("folderMapping", folderMapping);

UpdateFileInternal(new MemoryStream(), folderMapping, mappedPath + PlaceHolderFileName);
UpdateFileInternal(new MemoryStream(), folderMapping, mappedPath + Constants.PlaceHolderFileName);
}

/// <summary>
Expand All @@ -239,7 +229,7 @@ public override string GetFileUrl(IFileInfo file)
Requires.NotNull("file", file);

var folderMapping = FolderMappingController.Instance.GetFolderMapping(file.PortalId, file.FolderMappingID);
var directLink = string.IsNullOrEmpty(GetSetting(folderMapping, DirectLink)) || GetSetting(folderMapping, DirectLink).ToLowerInvariant() == "true";
var directLink = string.IsNullOrEmpty(GetSetting(folderMapping, Constants.DirectLink)) || GetSetting(folderMapping, Constants.DirectLink).ToLowerInvariant() == "true";

if (directLink)
{
Expand All @@ -249,7 +239,7 @@ public override string GetFileUrl(IFileInfo file)
var container = GetContainer(folderMapping);
var blob = container.GetBlobReference(uri);
var absuri = blob.Uri.AbsoluteUri;
var customDomain = GetEncryptedSetting(folderMapping.FolderMappingSettings, CustomDomain);
var customDomain = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.CustomDomain);

if (!string.IsNullOrEmpty(customDomain))
{
Expand Down Expand Up @@ -318,9 +308,9 @@ public override string GetFolderProviderIconPath()
public List<string> GetAllContainers(FolderMappingInfo folderMapping)
{
List<string> containers = new List<string>();
var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountName);
var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, AccountKey);
var useHttps = GetBooleanSetting(folderMapping, UseHttps);
var accountName = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountName);
var accountKey = GetEncryptedSetting(folderMapping.FolderMappingSettings, Constants.AccountKey);
var useHttps = GetBooleanSetting(folderMapping, Constants.UseHttps);

var sc = new StorageCredentials(accountName, accountKey);
var csa = new CloudStorageAccount(sc, useHttps);
Expand Down
23 changes: 23 additions & 0 deletions DNN Platform/Providers/FolderProviders/AzureFolderProvider/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#region Copyright
// DotNetNuke® - http://www.dotnetnuke.com
// Copyright (c) 2002-2018
// by DotNetNuke Corporation
// All Rights Reserved
#endregion

namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider
{
internal class Constants
{
public const string AccountName = "AccountName";
public const string AccountKey = "AccountKey";
public const string Container = "Container";
public const string UseHttps = "UseHttps";
public const string DirectLink = "DirectLink";
public const string CustomDomain = "CustomDomain";
public const string FolderProviderType = "AzureFolderProvider";
public const string PlaceHolderFileName = "dotnetnuke.placeholder.donotdelete";
public const string SyncBatchSize = "SyncBatchSize";
public const int DefaultSyncBatchSize = 2048;
}
}
6 changes: 6 additions & 0 deletions DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,10 @@
<asp:TextBox ID="tbCustomDomain" runat="server" MaxLength="200" Width="200" />
<asp:RegularExpressionValidator ID="rgBlobEndpoint" CssClass="dnnFormMessage dnnFormError"
runat="server" ControlToValidate="tbCustomDomain" Display="Dynamic" ResourceKey="tbCustomDomain.Error" ValidationExpression="^http[s]?://([a-zA-Z0-9\-]+\.)*([a-zA-Z]{3,61}|[a-zA-Z]{1,}\.[a-zA-Z]{2,3})$"></asp:RegularExpressionValidator>
</div>
<div class="dnnFormItem">
<dnn:Label ID="plSyncBatchSize" runat="server" />
<asp:TextBox ID="tbSyncBatchSize" runat="server" MaxLength="6" Width="200" />
<asp:RegularExpressionValidator ID="rgInteger" CssClass="dnnFormMessage dnnFormError"
runat="server" ControlToValidate="tbSyncBatchSize" Display="Dynamic" ResourceKey="tbSyncBatchSize.Error" ValidationExpression="^\d+$"></asp:RegularExpressionValidator>
</div>
74 changes: 39 additions & 35 deletions DNN Platform/Providers/FolderProviders/AzureFolderProvider/Settings.ascx.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
using DotNetNuke.Instrumentation;
using DotNetNuke.Services.FileSystem;
using DotNetNuke.Services.Localization;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.WindowsAzure.Storage.Blob;
Expand All @@ -25,18 +24,7 @@ namespace DotNetNuke.Providers.FolderProviders.AzureFolderProvider
public partial class Settings : FolderMappingSettingsControlBase
{
private static readonly ILog Logger = LoggerSource.Instance.GetLogger(typeof (Settings));
#region Constants

private const string AccountName = "AccountName";
private const string AccountKey = "AccountKey";
private const string Container = "Container";
private const string UseHttps = "UseHttps";
private const string DirectLink = "DirectLink";
private const string CustomDomain = "CustomDomain";
private const string FolderProviderType = "AzureFolderProvider";

#endregion

#region Overrided Methods

/// <summary>
Expand All @@ -45,24 +33,24 @@ public partial class Settings : FolderMappingSettingsControlBase
/// <param name="folderMappingSettings">The Hashtable containing the folder mapping settings.</param>
public override void LoadSettings(Hashtable folderMappingSettings)
{
var folderProvider = FolderProvider.Instance(FolderProviderType);
if (folderMappingSettings.ContainsKey(AccountName))
var folderProvider = FolderProvider.Instance(Constants.FolderProviderType);
if (folderMappingSettings.ContainsKey(Constants.AccountName))
{
tbAccountName.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, AccountName);
tbAccountName.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.AccountName);
daguiler marked this conversation as resolved.
Show resolved Hide resolved
}

if (folderMappingSettings.ContainsKey(AccountKey))
if (folderMappingSettings.ContainsKey(Constants.AccountKey))
{
tbAccountKey.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, AccountKey);
tbAccountKey.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.AccountKey);
}

if (tbAccountName.Text.Length > 0 && tbAccountKey.Text.Length > 0)
{
var bucketName = "";

if (folderMappingSettings.ContainsKey(Container))
if (folderMappingSettings.ContainsKey(Constants.Container))
{
bucketName = folderMappingSettings[Container].ToString();
bucketName = folderMappingSettings[Constants.Container].ToString();
}

LoadContainers();
Expand All @@ -75,16 +63,25 @@ public override void LoadSettings(Hashtable folderMappingSettings)
}
}

if (folderMappingSettings.ContainsKey(UseHttps))
if (folderMappingSettings.ContainsKey(Constants.UseHttps))
{
chkUseHttps.Checked = bool.Parse(folderMappingSettings[UseHttps].ToString());
chkUseHttps.Checked = bool.Parse(folderMappingSettings[Constants.UseHttps].ToString());
}

chkDirectLink.Checked = !folderMappingSettings.ContainsKey(DirectLink) || folderMappingSettings[DirectLink].ToString().ToLowerInvariant() == "true";
chkDirectLink.Checked = !folderMappingSettings.ContainsKey(Constants.DirectLink) || folderMappingSettings[Constants.DirectLink].ToString().ToLowerInvariant() == "true";
daguiler marked this conversation as resolved.
Show resolved Hide resolved

if (folderMappingSettings.ContainsKey(Constants.CustomDomain))
{
tbCustomDomain.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, Constants.CustomDomain);
}

if (folderMappingSettings.ContainsKey(CustomDomain))
if (folderMappingSettings.ContainsKey(Constants.SyncBatchSize) && folderMappingSettings[Constants.SyncBatchSize] != null)
{
tbCustomDomain.Text = folderProvider.GetEncryptedSetting(folderMappingSettings, CustomDomain);
tbSyncBatchSize.Text = folderMappingSettings[Constants.SyncBatchSize].ToString();
}
else
{
tbSyncBatchSize.Text = Constants.DefaultSyncBatchSize.ToString();
}
}

Expand All @@ -109,6 +106,7 @@ public override void UpdateSettings(int folderMappingID)
var container = GetContainer();
var useHttps = GetUseHttps();
var customDomain = GetCustomDomain();
var synchBatchSize = GetSynchBatchSize();

if (AreThereFolderMappingsWithSameSettings(folderMapping, accountName, container))
{
Expand All @@ -118,12 +116,13 @@ public override void UpdateSettings(int folderMappingID)
throw new Exception();
}

folderMapping.FolderMappingSettings[AccountName] = accountName;
folderMapping.FolderMappingSettings[AccountKey] = accountKey;
folderMapping.FolderMappingSettings[Container] = container;
folderMapping.FolderMappingSettings[UseHttps] = useHttps;
folderMapping.FolderMappingSettings[DirectLink] = chkDirectLink.Checked;
folderMapping.FolderMappingSettings[CustomDomain] = customDomain;
folderMapping.FolderMappingSettings[Constants.AccountName] = accountName;
folderMapping.FolderMappingSettings[Constants.AccountKey] = accountKey;
folderMapping.FolderMappingSettings[Constants.Container] = container;
folderMapping.FolderMappingSettings[Constants.UseHttps] = useHttps;
folderMapping.FolderMappingSettings[Constants.DirectLink] = chkDirectLink.Checked;
folderMapping.FolderMappingSettings[Constants.CustomDomain] = customDomain;
folderMapping.FolderMappingSettings[Constants.SyncBatchSize] = synchBatchSize;

folderMappingController.UpdateFolderMapping(folderMapping);
}
Expand All @@ -139,8 +138,8 @@ private static bool AreThereFolderMappingsWithSameSettings(FolderMappingInfo fol

return folderMappings
.Where(fm => fm.FolderMappingID != folderMapping.FolderMappingID && fm.FolderProviderType == folderMapping.FolderProviderType)
.Any(fm => fm.FolderMappingSettings[AccountName].ToString().Equals(accountName, StringComparison.InvariantCulture) &&
fm.FolderMappingSettings[Container].ToString().Equals(container, StringComparison.InvariantCultureIgnoreCase));
.Any(fm => fm.FolderMappingSettings[Constants.AccountName].ToString().Equals(accountName, StringComparison.InvariantCulture) &&
fm.FolderMappingSettings[Constants.Container].ToString().Equals(container, StringComparison.InvariantCultureIgnoreCase));
}

private string GetUseHttps()
Expand Down Expand Up @@ -246,17 +245,22 @@ private bool CreateContainer(string containerName)

private string GetAccountKey()
{
return FolderProvider.Instance(FolderProviderType).EncryptValue(tbAccountKey.Text);
return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbAccountKey.Text);
}

private string GetAccountName()
{
return FolderProvider.Instance(FolderProviderType).EncryptValue(tbAccountName.Text);
return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbAccountName.Text);
}

private string GetCustomDomain()
{
return FolderProvider.Instance(FolderProviderType).EncryptValue(tbCustomDomain.Text);
return FolderProvider.Instance(Constants.FolderProviderType).EncryptValue(tbCustomDomain.Text);
}

private string GetSynchBatchSize()
{
return tbSyncBatchSize.Text;
}

private void LoadContainers()
Expand Down
Loading