Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

For Azure, support multiple storage accounts and secondary endpoints (which are readonly) #13228

Closed
wants to merge 1 commit into from
Closed

For Azure, support multiple storage accounts and secondary endpoints (which are readonly) #13228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions plugins/cloud-azure/src/main/java/org/elasticsearch/cloud/azure/AzureModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,8 @@ public static boolean isSnapshotReady(Settings settings, ESLogger logger) {
return false;
}

if (isPropertyMissing(settings, Storage.ACCOUNT) ||
isPropertyMissing(settings, Storage.KEY)) {
if (isArrayMissing(settings, Storage.ACCOUNT) ||
isArrayMissing(settings, Storage.KEY)) {
logger.debug("azure repository is not set using [{}] and [{}] properties",
Storage.ACCOUNT,
Storage.KEY);
Expand All @@ -165,4 +165,10 @@ public static boolean isPropertyMissing(Settings settings, String name) throws E
return false;
}

public static boolean isArrayMissing(Settings settings, String name) throws ElasticsearchException {
if (settings.getAsArray(name) == null) {
return true;
}
return false;
}
}
12 changes: 6 additions & 6 deletions ...cloud-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public AzureBlobContainer(String repositoryName, BlobPath path, AzureBlobStore b
@Override
public boolean blobExists(String blobName) {
try {
return blobStore.client().blobExists(blobStore.container(), buildKey(blobName));
return blobStore.blobExists(blobStore.container(), buildKey(blobName));
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage());
}
Expand All @@ -71,7 +71,7 @@ public boolean blobExists(String blobName) {
@Override
public InputStream openInput(String blobName) throws IOException {
try {
return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new FileNotFoundException(e.getMessage());
Expand All @@ -85,7 +85,7 @@ public InputStream openInput(String blobName) throws IOException {
@Override
public OutputStream createOutput(String blobName) throws IOException {
try {
return new AzureOutputStream(blobStore.client().getOutputStream(blobStore.container(), buildKey(blobName)));
return new AzureOutputStream(blobStore.getOutputStream(blobStore.container(), buildKey(blobName)));
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new FileNotFoundException(e.getMessage());
Expand All @@ -101,7 +101,7 @@ public OutputStream createOutput(String blobName) throws IOException {
@Override
public void deleteBlob(String blobName) throws IOException {
try {
blobStore.client().deleteBlob(blobStore.container(), buildKey(blobName));
blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand All @@ -112,7 +112,7 @@ public void deleteBlob(String blobName) throws IOException {
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {

try {
return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand All @@ -127,7 +127,7 @@ public void move(String sourceBlobName, String targetBlobName) throws IOExceptio

logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore.container());

blobStore.client().moveBlob(blobStore.container(), source, target);
blobStore.moveBlob(blobStore.container(), source, target);
} catch (URISyntaxException e) {
logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand Down
78 changes: 73 additions & 5 deletions ...ins/cloud-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
package org.elasticsearch.cloud.azure.blobstore;

import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.LocationMode;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.component.AbstractComponent;
Expand All @@ -30,10 +32,17 @@
import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;

import java.io.InputStream;
import java.io.OutputStream;
import java.io.ByteArrayOutputStream;
import java.util.Locale;
import java.util.Map;

import java.net.URISyntaxException;

import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.CONTAINER;
import static org.elasticsearch.repositories.azure.AzureRepository.CONTAINER_DEFAULT;
import static org.elasticsearch.repositories.azure.AzureRepository.Repository;

/**
*
Expand All @@ -42,6 +51,8 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {

private final AzureStorageService client;

private final String accountName;
private final LocationMode locMode;
private final String container;
private final String repositoryName;

Expand All @@ -52,17 +63,24 @@ public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings
this.client = client;
this.container = repositorySettings.settings().get("container", settings.get(CONTAINER, CONTAINER_DEFAULT));
this.repositoryName = name.getName();

this.accountName = repositorySettings.settings().get(Repository.ACCOUNT, null);
// NOTE: null account means to use the first one specified in config

String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null);
if (modeStr == null) {
this.locMode = LocationMode.PRIMARY_ONLY;
}
else {
this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
}
}

@Override
public String toString() {
return container;
}

public AzureStorageService client() {
return client;
}

public String container() {
return container;
}
Expand All @@ -80,7 +98,7 @@ public void delete(BlobPath path) {
}

try {
client.deleteFiles(container, keyPath);
this.client.deleteFiles(this.accountName, this.locMode, container, keyPath);
} catch (URISyntaxException | StorageException e) {
logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
}
Expand All @@ -89,4 +107,54 @@ public void delete(BlobPath path) {
@Override
public void close() {
}

public boolean doesContainerExist(String container)
{
return this.client.doesContainerExist(this.accountName, this.locMode, container);
}

public void removeContainer(String container) throws URISyntaxException, StorageException
{
this.client.removeContainer(this.accountName, this.locMode, container);
}

public void createContainer(String container) throws URISyntaxException, StorageException
{
this.client.createContainer(this.accountName, this.locMode, container);
}

public void deleteFiles(String container, String path) throws URISyntaxException, StorageException
{
this.client.deleteFiles(this.accountName, this.locMode, container, path);
}

public boolean blobExists(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.blobExists(this.accountName, this.locMode, container, blob);
}

public void deleteBlob(String container, String blob) throws URISyntaxException, StorageException
{
this.client.deleteBlob(this.accountName, this.locMode, container, blob);
}

public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.getInputStream(this.accountName, this.locMode, container, blob);
}

public OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.getOutputStream(this.accountName, this.locMode, container, blob);
}

public Map<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException
{
return this.client.listBlobsByPrefix(this.accountName, this.locMode, container, keyPath, prefix);
}

public void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException
{
this.client.moveBlob(this.accountName, this.locMode, container, sourceBlob, targetBlob);
}
}
21 changes: 11 additions & 10 deletions .../cloud-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.cloud.azure.storage;

import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.LocationMode;
import org.elasticsearch.common.blobstore.BlobMetaData;

import java.io.InputStream;
Expand All @@ -42,23 +43,23 @@ static public final class Storage {
public static final String COMPRESS = "repositories.azure.compress";
}

boolean doesContainerExist(String container);
boolean doesContainerExist(String account, LocationMode mode, String container);

void removeContainer(String container) throws URISyntaxException, StorageException;
void removeContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException;

void createContainer(String container) throws URISyntaxException, StorageException;
void createContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException;

void deleteFiles(String container, String path) throws URISyntaxException, StorageException;
void deleteFiles(String account, LocationMode mode, String container, String path) throws URISyntaxException, StorageException;

boolean blobExists(String container, String blob) throws URISyntaxException, StorageException;
boolean blobExists(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;

void deleteBlob(String container, String blob) throws URISyntaxException, StorageException;
void deleteBlob(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;

InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException;
InputStream getInputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;

OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException;
OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;

Map<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
Map<String,BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException;

void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
}
Loading