Add regex blob rename middleware

README.md

@@ -107,6 +107,7 @@ S3Proxy can modify its behavior based on middlewares:
 * [large object mocking](https://github.com/gaul/s3proxy/wiki/Middleware-large-object-mocking)
 * [read-only](https://github.com/gaul/s3proxy/wiki/Middleware-read-only)
 * [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend)
+* [regex rename blobs](https://github.com/gaul/s3proxy/wiki/Middleware-regex)
 
 ## Limitations
 

src/main/java/org/gaul/s3proxy/Main.java

@@ -35,6 +35,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableBiMap;
@@ -247,6 +248,13 @@ private static BlobStore parseMiddlewareProperties(BlobStore blobStore,
             blobStore = AliasBlobStore.newAliasBlobStore(blobStore, aliases);
         }
 
+        ImmutableList<Map.Entry<Pattern, String>> regexs = RegexBlobStore.parseRegexs(
+                properties);
+        if (!regexs.isEmpty()){
+            System.err.println("Using regex backend");
+            blobStore = RegexBlobStore.newRegexBlobStore(blobStore, regexs);
+        }
+
         ImmutableMap<String, Integer> shards =
                 ShardedBlobStore.parseBucketShards(properties);
         ImmutableMap<String, String> prefixes =

src/main/java/org/gaul/s3proxy/RegexBlobStore.java

@@ -0,0 +1,214 @@
+package org.gaul.s3proxy;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobAccess;
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.options.CopyOptions;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.blobstore.util.ForwardingBlobStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * This class implements a middleware to apply regex to blob names.
+ * The regex are configured as:
+ * s3proxy.regex-blobstore.match.<regex name> = <regex match
+ * expression>
+ * s3proxy.regex-blobstore.replace.<regex name> = <regex replace
+ * expression>
+ *
+ * You can add multiple regex, they will be applied from the beginning to the
+ * end,
+ * stopping as soon as the first regex matches.
+ */
+public class RegexBlobStore extends ForwardingBlobStore {
+    private final ImmutableList<Entry<Pattern, String>> regexs;
+    private static final Logger logger = LoggerFactory.getLogger(RegexBlobStore.class);
+
+    static BlobStore newRegexBlobStore(BlobStore delegate, ImmutableList<Entry<Pattern, String>> regexs) {
+        return new RegexBlobStore(delegate, regexs);
+    }
+
+    private RegexBlobStore(BlobStore blobStore, ImmutableList<Entry<Pattern, String>> regexs) {
+        super(blobStore);
+        this.regexs = requireNonNull(regexs);
+    }
+
+    public static ImmutableList<Map.Entry<Pattern, String>> parseRegexs(Properties properties) {
+
+        List<Entry<String, String>> config_regex = new ArrayList<>();
+        List<Entry<Pattern, String>> regexs = new ArrayList<>();
+
+        for (String key : properties.stringPropertyNames()) {
+            if (key.startsWith(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE)) {
+                String prop_key = key.substring(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE.length() + 1);
+                String value = properties.getProperty(key);
+
+                config_regex.add(new SimpleEntry<>(prop_key, value));
+            }
+        }
+
+        for (Entry<String, String> entry : config_regex) {
+            String key = entry.getKey();
+            if (key.startsWith(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH)) {
+                String regex_name = key.substring(S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH.length() + 1);
+                String regex = entry.getValue();
+                Pattern pattern = Pattern.compile(regex);
+
+                String replace = properties.getProperty(
+                        String.join(
+                                ".",
+                                S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
+                                S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE,
+                                regex_name));
+
+                checkArgument(
+                        replace != null,
+                        "Regex %s has no replace property associated",
+                        regex_name);
+
+                logger.info("Adding new regex with name {} replaces with {} to {}", regex_name, regex, replace);
+
+                regexs.add(new SimpleEntry<>(pattern, replace));
+            }
+        }
+
+        return ImmutableList.copyOf(regexs);
+    }
+
+    @Override
+    public boolean directoryExists(String container, String directory) {
+        return super.directoryExists(container, replaceBlobName(directory));
+    }
+
+    @Override
+    public void createDirectory(String container, String directory) {
+        super.createDirectory(container, replaceBlobName(directory));
+    }
+
+    @Override
+    public void deleteDirectory(String container, String directory) {
+        super.deleteDirectory(container, replaceBlobName(directory));
+    }
+
+    @Override
+    public boolean blobExists(String container, String name) {
+        return super.blobExists(container, replaceBlobName(name));
+    }
+
+    @Override
+    public String putBlob(String containerName, Blob blob) {
+        String name = blob.getMetadata().getName();
+        String newName = replaceBlobName(name);
+        blob.getMetadata().setName(newName);
+
+        logger.debug("Renaming blob name from {} to {}", name, newName);
+
+        return super.putBlob(containerName, blob);
+    }
+
+    @Override
+    public String putBlob(String containerName, Blob blob, PutOptions putOptions) {
+        String name = blob.getMetadata().getName();
+        String newName = replaceBlobName(name);
+        blob.getMetadata().setName(newName);
+
+        logger.debug("Renaming blob name from {} to {}", name, newName);
+
+        return super.putBlob(containerName, blob, putOptions);
+    }
+
+    @Override
+    public String copyBlob(String fromContainer, String fromName, String toContainer, String toName,
+            CopyOptions options) {
+        return super.copyBlob(fromContainer, replaceBlobName(fromName), toContainer, replaceBlobName(toName), options);
+    }
+
+    @Override
+    public BlobMetadata blobMetadata(String container, String name) {
+        return super.blobMetadata(container, replaceBlobName(name));
+    }
+
+    @Override
+    public Blob getBlob(String containerName, String name) {
+        return super.getBlob(containerName, replaceBlobName(name));
+    }
+
+    @Override
+    public void removeBlob(String container, String name) {
+        super.removeBlob(container, replaceBlobName(name));
+    }
+
+    @Override
+    public void removeBlobs(String container, Iterable<String> iterable) {
+        List<String> blobs = new ArrayList<>();
+        for (String name : iterable) {
+            blobs.add(replaceBlobName(name));
+        }
+        super.removeBlobs(container, blobs);
+    }
+
+    @Override
+    public BlobAccess getBlobAccess(String container, String name) {
+        return super.getBlobAccess(container, replaceBlobName(name));
+    }
+
+    @Override
+    public void setBlobAccess(String container, String name, BlobAccess access) {
+        super.setBlobAccess(container, replaceBlobName(name), access);
+    }
+
+    @Override
+    public void downloadBlob(String container, String name, File destination) {
+        super.downloadBlob(container, replaceBlobName(name), destination);
+    }
+
+    @Override
+    public void downloadBlob(String container, String name, File destination, ExecutorService executor) {
+        super.downloadBlob(container, replaceBlobName(name), destination, executor);
+    }
+
+    @Override
+    public InputStream streamBlob(String container, String name) {
+        return super.streamBlob(container, replaceBlobName(name));
+    }
+
+    @Override
+    public InputStream streamBlob(String container, String name, ExecutorService executor) {
+        return super.streamBlob(container, replaceBlobName(name), executor);
+    }
+
+    private String replaceBlobName(String name) {
+        String newName = name;
+
+        for (Map.Entry<Pattern, String> entry : this.regexs) {
+            Pattern pattern = entry.getKey();
+            Matcher match = pattern.matcher(name);
+
+            if (match.find()) {
+                return match.replaceAll(entry.getValue());
+            }
+
+        }
+
+        return newName;
+    }
+}

src/main/java/org/gaul/s3proxy/S3ProxyConstants.java

@@ -93,6 +93,13 @@ public final class S3ProxyConstants {
     /** Alias a backend bucket to an alternate name. */
     public static final String PROPERTY_ALIAS_BLOBSTORE =
             "s3proxy.alias-blobstore";
+    /** Alias a backend bucket to an alternate name. */
+    public static final String PROPERTY_REGEX_BLOBSTORE =
+            "s3proxy.regex-blobstore";
+    public static final String PROPERTY_REGEX_BLOBSTORE_MATCH =
+            "match";
+    public static final String PROPERTY_REGEX_BLOBSTORE_REPLACE =
+            "replace";
     /** Discard object data. */
     public static final String PROPERTY_NULL_BLOBSTORE =
             "s3proxy.null-blobstore";

src/test/java/org/gaul/s3proxy/RegexBlobStoreTest.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2014-2021 Andrew Gaul <andrew@gaul.org>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gaul.s3proxy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
+import com.google.inject.Module;
+
+import java.util.AbstractMap.SimpleEntry;
+
+import org.assertj.core.api.Assertions;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+
+public final class RegexBlobStoreTest {
+    private BlobStoreContext context;
+    private BlobStore delegate;
+    private String containerName;
+
+    @Before
+    public void setUp() throws Exception {
+        containerName = createRandomContainerName();
+
+        context = ContextBuilder
+                .newBuilder("transient")
+                .credentials("identity", "credential")
+                .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
+                .build(BlobStoreContext.class);
+        delegate = context.getBlobStore();
+        delegate.createContainerInLocation(null, containerName);
+
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (context != null) {
+            delegate.deleteContainer(containerName);
+            context.close();
+        }
+    }
+
+    @Test
+    public void testRemoveSomeCharsFromName() throws IOException {
+        ImmutableList.Builder<Map.Entry<Pattern, String>> regexBuilder = new ImmutableList.Builder<>();
+        regexBuilder.add(new SimpleEntry<Pattern, String>(Pattern.compile("[^a-zA-Z0-9/_.]"), "_"));
+        BlobStore regexBlobStore = RegexBlobStore.newRegexBlobStore(delegate, regexBuilder.build());
+
+        String initialBlobName = "test/remove:badchars-folder/blob.txt";
+        String targetBlobName = "test/remove_badchars_folder/blob.txt";
+        ByteSource content = TestUtils.randomByteSource().slice(0, 1024);
+        @SuppressWarnings("deprecation")
+        String contentHash = Hashing.md5().hashBytes(content.read()).toString();
+        Blob blob = regexBlobStore.blobBuilder(initialBlobName).payload(content).build();
+
+        String eTag = regexBlobStore.putBlob(containerName, blob);
+        assertThat(eTag).isEqualTo(contentHash);
+
+        BlobMetadata blobMetadata = regexBlobStore.blobMetadata(
+                containerName, targetBlobName);
+
+        assertThat(blobMetadata.getETag()).isEqualTo(contentHash);
+        blob = regexBlobStore.getBlob(containerName, targetBlobName);
+        try (InputStream actual = blob.getPayload().openStream();
+             InputStream expected = content.openStream()) {
+            assertThat(actual).hasContentEqualTo(expected);
+        }
+
+        blob = regexBlobStore.getBlob(containerName, initialBlobName);
+        try (InputStream actual = blob.getPayload().openStream();
+             InputStream expected = content.openStream()) {
+            assertThat(actual).hasContentEqualTo(expected);
+        }
+    }
+
+    @Test
+    public void testParseMatchWithoutReplace() {
+        Properties properties = new Properties();
+        properties.put(
+                String.format("%s.%s.sample1", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
+                        S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
+                "test");
+        properties.put(
+                String.format("%s.%s.sample2", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
+                        S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_MATCH),
+                "test");
+        properties.put(
+                String.format("%s.%s.sample1", S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE,
+                        S3ProxyConstants.PROPERTY_REGEX_BLOBSTORE_REPLACE),
+                "test");
+
+        try {
+            RegexBlobStore.parseRegexs(properties);
+            Assertions.failBecauseExceptionWasNotThrown(
+                    IllegalArgumentException.class);
+        } catch (IllegalArgumentException exc) {
+            assertThat(exc.getMessage()).isEqualTo(
+                    "Regex sample2 has no replace property associated");
+        }
+    }
+
+    private static String createRandomContainerName() {
+        return "container-" + new Random().nextInt(Integer.MAX_VALUE);
+    }
+}