Skip to content

Commit

Permalink
Add support for secondary azure storage account
Browse files Browse the repository at this point in the history 
Follow up for elastic#13228.

This commit adds support for a secondary storage account:

```yml
cloud:
    azure:
        storage:
            my_account1:
                account: your_azure_storage_account1
                key: your_azure_storage_key1
                default: true
            my_account2:
                account: your_azure_storage_account2
                key: your_azure_storage_key2
```

When creating a repository, you can choose which azure account you want to use for it:

```sh
curl -XPUT localhost:9200/_snapshot/my_backup1?pretty -d '{
  "type": "azure"
}'

curl -XPUT localhost:9200/_snapshot/my_backup2?pretty -d '{
  "type": "azure",
  "settings": {
    "account" : "my_account2",
    "location_mode": "secondary_only"
  }
}'
```

`location_mode` supports `primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
to `secondary_only`, it will force `read_only` to true.
  • Loading branch information
@dadoonet
craigwi authored and dadoonet committed Nov 18, 2015
1 parent b855e7d commit 79a4d9c
Show file tree
Hide file tree
Showing 16 changed files with 774 additions and 171 deletions.
74 changes: 44 additions & 30 deletions docs/plugins/repository-azure.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,37 @@ To enable Azure repositories, you have first to set your azure storage settings
cloud:
azure:
storage:
account: your_azure_storage_account
key: your_azure_storage_key
my_account:
account: your_azure_storage_account
key: your_azure_storage_key
----

For information, in previous version of the azure plugin, settings were:
Note that you can also define more than one account:

[source,yaml]
----
cloud:
azure:
storage_account: your_azure_storage_account
storage_key: your_azure_storage_key
storage:
my_account1:
account: your_azure_storage_account1
key: your_azure_storage_key1
default: true
my_account2:
account: your_azure_storage_account2
key: your_azure_storage_key2
----

`my_account1` is the default account which will be used by a repository unless you set an explicit one.


The Azure repository supports following settings:

`account`::

Azure account settings to use. Defaults to the only one if you set a single
account or to the one marked as `default` if you have more than one.

`container`::

Container name. Defaults to `elasticsearch-snapshots`
Expand All @@ -82,6 +97,11 @@ The Azure repository supports following settings:

Makes repository read-only. coming[2.1.0] Defaults to `false`.

`location_mode`::

`primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
to `secondary_only`, it will force `read_only` to true.

Some examples, using scripts:

[source,json]
Expand All @@ -103,14 +123,32 @@ PUT _snapshot/my_backup2
"compress": true
}
}
# With two accounts defined in elasticsearch.yml (my_account1 and my_account2)
PUT _snapshot/my_backup3
{
"type": "azure",
"settings": {
"account": "my_account1"
}
}
PUT _snapshot/my_backup4
{
"type": "azure",
"settings": {
"account": "my_account2",
"location_mode": "primary_only"
}
}
----
// AUTOSENSE

Example using Java:

[source,java]
----
client.admin().cluster().preparePutRepository("my_backup3")
client.admin().cluster().preparePutRepository("my_backup_java1")
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Storage.CONTAINER, "backup_container")
.put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB))
Expand All @@ -129,27 +167,3 @@ permitted in container names.
* All letters in a container name must be lowercase.
* Container names must be from 3 through 63 characters long.

[[repository-azure-testing]]
==== Testing Azure

Integrations tests in this plugin require working Azure configuration and therefore disabled by default.
To enable tests prepare a config file `elasticsearch.yml` with the following content:

[source,yaml]
----
cloud:
azure:
storage:
account: "YOUR-AZURE-STORAGE-NAME"
key: "YOUR-AZURE-STORAGE-KEY"
----

Replaces `account`, `key` with your settings. Please, note that the test will delete all snapshot/restore related
files in the specified bucket.

To run test:

[source,sh]
----
mvn -Dtests.azure=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test
----
1 change: 0 additions & 1 deletion ...ins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,4 @@ public static boolean isPropertyMissing(Settings settings, String name) throws E
}
return false;
}

}
41 changes: 2 additions & 39 deletions ...s/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,9 @@

package org.elasticsearch.cloud.azure;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
import org.elasticsearch.cloud.azure.storage.AzureStorageSettingsFilter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
Expand All @@ -43,18 +40,12 @@
*/
public class AzureRepositoryModule extends AbstractModule {
protected final ESLogger logger;
private Settings settings;

// pkg private so it is settable by tests
static Class<? extends AzureStorageService> storageServiceImpl = AzureStorageServiceImpl.class;

public static Class<? extends AzureStorageService> getStorageServiceImpl() {
return storageServiceImpl;
}

@Inject
public AzureRepositoryModule(Settings settings) {
this.settings = settings;
this.logger = Loggers.getLogger(getClass(), settings);
}

Expand All @@ -64,35 +55,7 @@ protected void configure() {
bind(AzureStorageSettingsFilter.class).asEagerSingleton();

// If we have settings for azure repository, let's start the azure storage service
if (isSnapshotReady(settings, logger)) {
logger.debug("starting azure repository service");
bind(AzureStorageService.class).to(storageServiceImpl).asEagerSingleton();
}
}

/**
* Check if we have repository azure settings available
* @return true if we can use snapshot and restore
*/
public static boolean isSnapshotReady(Settings settings, ESLogger logger) {
if (isPropertyMissing(settings, Storage.ACCOUNT) ||
isPropertyMissing(settings, Storage.KEY)) {
logger.debug("azure repository is not set using [{}] and [{}] properties",
Storage.ACCOUNT,
Storage.KEY);
return false;
}

logger.trace("all required properties for azure repository are set!");

return true;
}

public static boolean isPropertyMissing(Settings settings, String name) throws ElasticsearchException {
if (!Strings.hasText(settings.get(name))) {
return true;
}
return false;
logger.debug("starting azure repository service");
bind(AzureStorageService.class).to(storageServiceImpl).asEagerSingleton();
}

}
12 changes: 6 additions & 6 deletions ...itory-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public AzureBlobContainer(String repositoryName, BlobPath path, AzureBlobStore b
@Override
public boolean blobExists(String blobName) {
try {
return blobStore.client().blobExists(blobStore.container(), buildKey(blobName));
return blobStore.blobExists(blobStore.container(), buildKey(blobName));
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage());
}
Expand All @@ -73,7 +73,7 @@ public boolean blobExists(String blobName) {
@Override
public InputStream readBlob(String blobName) throws IOException {
try {
return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new FileNotFoundException(e.getMessage());
Expand All @@ -100,7 +100,7 @@ public void writeBlob(String blobName, BytesReference bytes) throws IOException

private OutputStream createOutput(String blobName) throws IOException {
try {
return new AzureOutputStream(blobStore.client().getOutputStream(blobStore.container(), buildKey(blobName)));
return new AzureOutputStream(blobStore.getOutputStream(blobStore.container(), buildKey(blobName)));
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new FileNotFoundException(e.getMessage());
Expand All @@ -116,7 +116,7 @@ private OutputStream createOutput(String blobName) throws IOException {
@Override
public void deleteBlob(String blobName) throws IOException {
try {
blobStore.client().deleteBlob(blobStore.container(), buildKey(blobName));
blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", blobName, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand All @@ -127,7 +127,7 @@ public void deleteBlob(String blobName) throws IOException {
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {

try {
return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand All @@ -142,7 +142,7 @@ public void move(String sourceBlobName, String targetBlobName) throws IOExceptio

logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore.container());

blobStore.client().moveBlob(blobStore.container(), source, target);
blobStore.moveBlob(blobStore.container(), source, target);
} catch (URISyntaxException e) {
logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
throw new IOException(e);
Expand Down
80 changes: 71 additions & 9 deletions ...epository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@

package org.elasticsearch.cloud.azure.blobstore;

import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.component.AbstractComponent;
Expand All @@ -30,39 +32,49 @@
import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;

import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Map;

import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.CONTAINER;
import static org.elasticsearch.repositories.azure.AzureRepository.CONTAINER_DEFAULT;
import static org.elasticsearch.repositories.azure.AzureRepository.Repository;

/**
*
*/
public class AzureBlobStore extends AbstractComponent implements BlobStore {

private final AzureStorageService client;

private final String accountName;
private final LocationMode locMode;
private final String container;
private final String repositoryName;

@Inject
public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings repositorySettings,
AzureStorageService client) throws URISyntaxException, StorageException {
super(settings);
this.client = client;
this.client = client.start();
this.container = repositorySettings.settings().get("container", settings.get(CONTAINER, CONTAINER_DEFAULT));
this.repositoryName = name.getName();

// NOTE: null account means to use the first one specified in config
this.accountName = repositorySettings.settings().get(Repository.ACCOUNT, null);

String modeStr = repositorySettings.settings().get(Repository.LOCATION_MODE, null);
if (modeStr == null) {
this.locMode = LocationMode.PRIMARY_ONLY;
} else {
this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
}
}

@Override
public String toString() {
return container;
}

public AzureStorageService client() {
return client;
}

public String container() {
return container;
}
Expand All @@ -80,7 +92,7 @@ public void delete(BlobPath path) {
}

try {
client.deleteFiles(container, keyPath);
this.client.deleteFiles(this.accountName, this.locMode, container, keyPath);
} catch (URISyntaxException | StorageException e) {
logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
}
Expand All @@ -89,4 +101,54 @@ public void delete(BlobPath path) {
@Override
public void close() {
}

public boolean doesContainerExist(String container)
{
return this.client.doesContainerExist(this.accountName, this.locMode, container);
}

public void removeContainer(String container) throws URISyntaxException, StorageException
{
this.client.removeContainer(this.accountName, this.locMode, container);
}

public void createContainer(String container) throws URISyntaxException, StorageException
{
this.client.createContainer(this.accountName, this.locMode, container);
}

public void deleteFiles(String container, String path) throws URISyntaxException, StorageException
{
this.client.deleteFiles(this.accountName, this.locMode, container, path);
}

public boolean blobExists(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.blobExists(this.accountName, this.locMode, container, blob);
}

public void deleteBlob(String container, String blob) throws URISyntaxException, StorageException
{
this.client.deleteBlob(this.accountName, this.locMode, container, blob);
}

public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.getInputStream(this.accountName, this.locMode, container, blob);
}

public OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException
{
return this.client.getOutputStream(this.accountName, this.locMode, container, blob);
}

public Map<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException
{
return this.client.listBlobsByPrefix(this.accountName, this.locMode, container, keyPath, prefix);
}

public void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException
{
this.client.moveBlob(this.accountName, this.locMode, container, sourceBlob, targetBlob);
}
}
Loading

0 comments on commit 79a4d9c

Please sign in to comment.