diff --git a/dora/core/common/src/main/java/alluxio/conf/PropertyKey.java b/dora/core/common/src/main/java/alluxio/conf/PropertyKey.java index 3785636a8fb8..28592b3c21f7 100755 --- a/dora/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/dora/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -6423,6 +6423,13 @@ public String toString() { .setConsistencyCheckLevel(ConsistencyCheckLevel.IGNORE) .setScope(Scope.CLIENT) .build(); + public static final PropertyKey FUSE_UPDATE_CHECK_ENABLED = + booleanBuilder(Name.FUSE_UPDATE_CHECK_ENABLED) + .setDefaultValue(Boolean.parseBoolean(ProjectConstants.UPDATE_CHECK_ENABLED)) + .setDescription("Whether to check for update availability for alluxio-fuse") + .setConsistencyCheckLevel(ConsistencyCheckLevel.ENFORCE) + .setIsHidden(true) + .build(); public static final PropertyKey FUSE_USER_GROUP_TRANSLATION_ENABLED = booleanBuilder(Name.FUSE_USER_GROUP_TRANSLATION_ENABLED) .setDefaultValue(false) @@ -8490,6 +8497,8 @@ public static final class Name { "alluxio.fuse.stat.cache.refresh.interval"; public static final String FUSE_UMOUNT_TIMEOUT = "alluxio.fuse.umount.timeout"; + public static final String FUSE_UPDATE_CHECK_ENABLED = + "alluxio.fuse.update.check.enabled"; public static final String FUSE_USER_GROUP_TRANSLATION_ENABLED = "alluxio.fuse.user.group.translation.enabled"; public static final String FUSE_SPECIAL_COMMAND_ENABLED = diff --git a/dora/core/common/src/main/java/alluxio/util/CommonUtils.java b/dora/core/common/src/main/java/alluxio/util/CommonUtils.java index c78d0698f973..e796d00ea800 100644 --- a/dora/core/common/src/main/java/alluxio/util/CommonUtils.java +++ b/dora/core/common/src/main/java/alluxio/util/CommonUtils.java @@ -678,14 +678,15 @@ public static RuntimeException closeAndRethrowRuntimeException(Closer closer, Th /** Alluxio process types. */ public enum ProcessType { + CLIENT, + FUSE, JOB_MASTER, JOB_WORKER, - CLIENT, MASTER, PLUGIN, PROXY, SECURITY, - WORKER; + WORKER, } /** diff --git a/dora/core/common/src/main/java/alluxio/check/UpdateCheck.java b/dora/core/common/src/main/java/alluxio/util/UpdateCheckUtils.java similarity index 63% rename from dora/core/common/src/main/java/alluxio/check/UpdateCheck.java rename to dora/core/common/src/main/java/alluxio/util/UpdateCheckUtils.java index ba64aee2853d..ff861c987e34 100644 --- a/dora/core/common/src/main/java/alluxio/check/UpdateCheck.java +++ b/dora/core/common/src/main/java/alluxio/util/UpdateCheckUtils.java @@ -9,13 +9,10 @@ * See the NOTICE file distributed with this work for information regarding copyright ownership. */ -package alluxio.check; +package alluxio.util; import alluxio.ProjectConstants; import alluxio.exception.runtime.FailedPreconditionRuntimeException; -import alluxio.util.EnvironmentUtils; -import alluxio.util.FeatureUtils; -import alluxio.util.OSUtils; import com.amazonaws.util.EC2MetadataUtils; import com.google.common.annotations.VisibleForTesting; @@ -39,7 +36,7 @@ * Check for updates. */ @ThreadSafe -public final class UpdateCheck { +public final class UpdateCheckUtils { public static final String USER_AGENT_SEPARATOR = ";"; static final String PRODUCT_CODE_FORMAT = "ProductCode:%s"; @@ -49,51 +46,36 @@ public final class UpdateCheck { static final String CFT_KEY = "cft"; static final String DOCKER_KEY = "docker"; static final String EC2_KEY = "ec2"; - static final String EMBEDDED_KEY = "embedded"; static final String EMR_KEY = "emr"; static final String GCE_KEY = "gce"; static final String KUBERNETES_KEY = "kubernetes"; - // Feature - static final String BACKUP_DELEGATION_KEY = "backupDelegation"; - static final String DAILY_BACKUP_KEY = "dailyBackup"; - static final String MASTER_AUDIT_LOG_KEY = "masterAuditLog"; - static final String PERSIST_BLACK_LIST_KEY = "persistBlackList"; - static final String PAGE_STORE_KEY = "pageStore"; - static final String INODE_METASTORE_ROCKS_KEY = "inodeRocks"; - static final String BLOCK_METASTORE_ROCKS_KEY = "blockRocks"; - static final String UNSAFE_PERSIST_KEY = "unsafePersist"; - static final String ZOOKEEPER_KEY = "zookeeper"; - /** * @param id the id of the current Alluxio identity (e.g. cluster id, instance id) + * @param processType process type * @param additionalInfo additional information to send - * @param connectionRequestTimeout the connection request timeout for the HTTP request in ms - * @param connectTimeout the connection timeout for the HTTP request in ms - * @param socketTimeout the socket timeout for the HTTP request in ms * @return the latest Alluxio version string */ - public static String getLatestVersion(String id, List additionalInfo, - long connectionRequestTimeout, long connectTimeout, long socketTimeout) - throws IOException { + public static String getLatestVersion(String id, CommonUtils.ProcessType processType, + List additionalInfo) throws IOException { Preconditions.checkState(id != null && !id.isEmpty(), "id should not be null or empty"); Preconditions.checkNotNull(additionalInfo); // Create the GET request. Joiner joiner = Joiner.on("/"); - String path = joiner.join("v0", "version"); + String path = joiner.join("v1", "version"); String url = new URL(new URL(ProjectConstants.UPDATE_CHECK_HOST), path).toString(); HttpGet post = new HttpGet(url); - post.setHeader("User-Agent", getUserAgentString(id, additionalInfo)); + post.setHeader("User-Agent", getUserAgentString(id, processType, additionalInfo)); post.setHeader("Authorization", "Basic " + ProjectConstants.UPDATE_CHECK_MAGIC_NUMBER); // Fire off the version check request. HttpClient client = HttpClientBuilder.create() .setDefaultRequestConfig( RequestConfig.custom() - .setConnectionRequestTimeout((int) connectionRequestTimeout) - .setConnectTimeout((int) connectTimeout) - .setSocketTimeout((int) socketTimeout) + .setConnectionRequestTimeout(3000) + .setConnectTimeout(3000) + .setSocketTimeout(3000) .build()) .build(); HttpResponse response = client.execute(post); @@ -110,17 +92,19 @@ public static String getLatestVersion(String id, List additionalInfo, /** * @param id the id of the current Alluxio identity (e.g. cluster id, instance id) + * @param processType process type * @param additionalInfo additional information to add to result string * @return a string representation of the user's environment in the format * "Alluxio/{ALLUXIO_VERSION} (valueA; valueB)" */ @VisibleForTesting - public static String getUserAgentString(String id, List additionalInfo) { + public static String getUserAgentString(String id, CommonUtils.ProcessType processType, + List additionalInfo) { List info = new ArrayList<>(); info.add(id); addUserAgentEnvironments(info); - addUserAgentFeatures(info); info.addAll(additionalInfo); + info.add(String.format("processType:%s", processType.toString())); return String.format("Alluxio/%s (%s)", ProjectConstants.VERSION, Joiner.on(USER_AGENT_SEPARATOR + " ").skipNulls().join(info)); } @@ -146,38 +130,6 @@ public static void addUserAgentEnvironments(List info) { } } - /** - * Get the feature's information. - * - * @param info the list to add info to - */ - @VisibleForTesting - public static void addUserAgentFeatures(List info) { - addIfTrue(FeatureUtils.isEmbeddedJournal(), info, EMBEDDED_KEY); - addIfTrue(FeatureUtils.isInodeStoreRocks(), info, INODE_METASTORE_ROCKS_KEY); - addIfTrue(FeatureUtils.isBlockStoreRocks(), info, BLOCK_METASTORE_ROCKS_KEY); - addIfTrue(FeatureUtils.isZookeeperEnabled(), info, ZOOKEEPER_KEY); - addIfTrue(FeatureUtils.isBackupDelegationEnabled(), info, BACKUP_DELEGATION_KEY); - addIfTrue(FeatureUtils.isDailyBackupEnabled(), info, DAILY_BACKUP_KEY); - addIfTrue(!FeatureUtils.isPersistenceBlacklistEmpty(), info, PERSIST_BLACK_LIST_KEY); - addIfTrue(FeatureUtils.isUnsafeDirectPersistEnabled(), info, UNSAFE_PERSIST_KEY); - addIfTrue(FeatureUtils.isMasterAuditLoggingEnabled(), info, MASTER_AUDIT_LOG_KEY); - addIfTrue(FeatureUtils.isPageStoreEnabled(), info, PAGE_STORE_KEY); - } - - /** - * Add feature name if condition is true. - * - * @param valid true, if condition is valid - * @param features feature list - * @param featureName feature name - */ - public static void addIfTrue(boolean valid, List features, String featureName) { - if (valid) { - features.add(featureName); - } - } - /** * Adds the information of EC2 environment to given list. * @@ -214,5 +166,5 @@ private static void addEC2Info(List info) { } } - private UpdateCheck() {} // prevent instantiation + private UpdateCheckUtils() {} // prevent instantiation } diff --git a/dora/core/common/src/test/java/alluxio/check/UpdateCheckTest.java b/dora/core/common/src/test/java/alluxio/check/UpdateCheckTest.java deleted file mode 100644 index 2d47b28d758f..000000000000 --- a/dora/core/common/src/test/java/alluxio/check/UpdateCheckTest.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.check; - -import alluxio.ProjectConstants; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.master.journal.JournalType; -import alluxio.master.metastore.MetastoreType; -import alluxio.util.EnvironmentUtils; -import alluxio.util.OSUtils; -import alluxio.worker.block.BlockStoreType; - -import com.amazonaws.SdkClientException; -import com.amazonaws.util.EC2MetadataUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Unit tests for {@link UpdateCheck}. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({EnvironmentUtils.class, EC2MetadataUtils.class}) -public class UpdateCheckTest { - - @Before - public void before() { - PowerMockito.mockStatic(EnvironmentUtils.class); - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(false); - Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(false); - Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(false); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(""); - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(false); - Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(false); - Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(false); - PowerMockito.mockStatic(EC2MetadataUtils.class); - } - - @Test - public void userAgentEnvironmentStringEmpty() { - List info = new ArrayList<>(); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(1, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - } - - @Test - public void userAgentEnvironmentStringDocker() { - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(2, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.DOCKER_KEY, info.get(1)); - } - - @Test - public void userAgentEnvironmentStringK8s() { - Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); - Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(3, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.DOCKER_KEY, info.get(1)); - Assert.assertEquals(UpdateCheck.KUBERNETES_KEY, info.get(2)); - } - - @Test - public void userAgentEnvironmentStringGCP() { - Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()) - .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(2, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(UpdateCheck.GCE_KEY, info.get(1)); - } - - @Test - public void userAgentEnvironmentStringEC2AMI() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - // When no user data in this ec2, null is returned - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn(null); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(3, info.size()); - Assert.assertEquals(String.format(UpdateCheck.OS_FORMAT, OSUtils.OS_NAME), - info.get(0)); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(2)); - } - - @Test - public void userAgentEnvironmentStringEC2CFT() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("{ \"cft_configure\": {}}"); - - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(4, info.size()); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.CFT_KEY, info.get(2)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(3)); - } - - @Test - public void userAgentEnvironmentStringEC2EMR() { - String randomProductCode = "random123code"; - Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); - Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); - Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(true); - Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("emr_apps"); - List info = new ArrayList<>(); - UpdateCheck.addUserAgentEnvironments(info); - Assert.assertEquals(4, info.size()); - Assert.assertEquals(String.format(UpdateCheck.PRODUCT_CODE_FORMAT, randomProductCode), - info.get(1)); - Assert.assertEquals(UpdateCheck.EMR_KEY, info.get(2)); - Assert.assertEquals(UpdateCheck.EC2_KEY, info.get(3)); - } - - @Test - public void featureStringEmbeddedJournal() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.EMBEDDED_KEY)); - Configuration.set(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.EMBEDDED_KEY)); - } - - @Test - public void featureStringInodeMetastoreRocks() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_INODE_METASTORE, MetastoreType.ROCKS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.INODE_METASTORE_ROCKS_KEY)); - Configuration.set(PropertyKey.MASTER_INODE_METASTORE, MetastoreType.HEAP); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.INODE_METASTORE_ROCKS_KEY)); - } - - @Test - public void featureStringBlockMetastoreRocks() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_BLOCK_METASTORE, MetastoreType.ROCKS); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.BLOCK_METASTORE_ROCKS_KEY)); - Configuration.set(PropertyKey.MASTER_BLOCK_METASTORE, MetastoreType.HEAP); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.BLOCK_METASTORE_ROCKS_KEY)); - } - - @Test - public void featureStringZookeeper() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.ZOOKEEPER_KEY)); - Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.ZOOKEEPER_KEY)); - } - - @Test - public void featureStringBackupDelegation() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.BACKUP_DELEGATION_KEY)); - Configuration.set(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.BACKUP_DELEGATION_KEY)); - } - - @Test - public void featureStringDailyBackup() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.DAILY_BACKUP_KEY)); - Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.DAILY_BACKUP_KEY)); - } - - @Test - public void featureStringPersistneceBlacklist() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_PERSISTENCE_BLACKLIST, ".tmp"); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.PERSIST_BLACK_LIST_KEY)); - Configuration.unset(PropertyKey.MASTER_PERSISTENCE_BLACKLIST); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.PERSIST_BLACK_LIST_KEY)); - } - - @Test - public void featureStringUnsafePersist() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_UNSAFE_DIRECT_PERSIST_OBJECT_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.UNSAFE_PERSIST_KEY)); - Configuration.set(PropertyKey.MASTER_UNSAFE_DIRECT_PERSIST_OBJECT_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.UNSAFE_PERSIST_KEY)); - } - - @Test - public void featureStringMasterAuditLogging() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, true); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.MASTER_AUDIT_LOG_KEY)); - Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.MASTER_AUDIT_LOG_KEY)); - } - - @Test - public void featureStringPageStore() { - List info = new ArrayList<>(); - Configuration.set(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.PAGE); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertTrue(listContainsTarget(info, UpdateCheck.PAGE_STORE_KEY)); - Configuration.set(PropertyKey.WORKER_BLOCK_STORE_TYPE, BlockStoreType.FILE); - info.clear(); - UpdateCheck.addUserAgentFeatures(info); - Assert.assertFalse(listContainsTarget(info, UpdateCheck.PAGE_STORE_KEY)); - } - - @Test - public void userAgent() { - String userAgentString = UpdateCheck.getUserAgentString("cluster1", new ArrayList<>()); - Pattern pattern = Pattern.compile( - String.format("Alluxio\\/%s \\(cluster1(?:.+)[^;]\\)", ProjectConstants.VERSION)); - Matcher matcher = pattern.matcher(userAgentString); - Assert.assertTrue(matcher.matches()); - } - - /** - * Makes sure the list containing the target information. - * - * @param list the list to check - * @param target the target info - * @return true if list contains the target - */ - private boolean listContainsTarget(List list, String target) { - for (String str : list) { - if (str.equals(target)) { - return true; - } - } - return false; - } -} diff --git a/dora/core/common/src/test/java/alluxio/util/UpdateCheckUtilsTest.java b/dora/core/common/src/test/java/alluxio/util/UpdateCheckUtilsTest.java new file mode 100644 index 000000000000..ed0d3fe56b7c --- /dev/null +++ b/dora/core/common/src/test/java/alluxio/util/UpdateCheckUtilsTest.java @@ -0,0 +1,163 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.util; + +import alluxio.ProjectConstants; + +import com.amazonaws.SdkClientException; +import com.amazonaws.util.EC2MetadataUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Unit tests for {@link UpdateCheckUtils}. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({EnvironmentUtils.class, EC2MetadataUtils.class}) +public class UpdateCheckUtilsTest { + + @Before + public void before() { + PowerMockito.mockStatic(EnvironmentUtils.class); + Mockito.when(EnvironmentUtils.isDocker()).thenReturn(false); + Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(false); + Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(false); + Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(""); + Mockito.when(EnvironmentUtils.isEC2()).thenReturn(false); + Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(false); + Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(false); + PowerMockito.mockStatic(EC2MetadataUtils.class); + } + + @Test + public void userAgentEnvironmentStringEmpty() { + List info = new ArrayList<>(); + Mockito.when(EC2MetadataUtils.getUserData()) + .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(1, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.OS_FORMAT, OSUtils.OS_NAME), + info.get(0)); + } + + @Test + public void userAgentEnvironmentStringDocker() { + Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); + Mockito.when(EC2MetadataUtils.getUserData()) + .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(2, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.OS_FORMAT, OSUtils.OS_NAME), + info.get(0)); + Assert.assertEquals(UpdateCheckUtils.DOCKER_KEY, info.get(1)); + } + + @Test + public void userAgentEnvironmentStringK8s() { + Mockito.when(EnvironmentUtils.isDocker()).thenReturn(true); + Mockito.when(EnvironmentUtils.isKubernetes()).thenReturn(true); + Mockito.when(EC2MetadataUtils.getUserData()) + .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(3, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.OS_FORMAT, OSUtils.OS_NAME), + info.get(0)); + Assert.assertEquals(UpdateCheckUtils.DOCKER_KEY, info.get(1)); + Assert.assertEquals(UpdateCheckUtils.KUBERNETES_KEY, info.get(2)); + } + + @Test + public void userAgentEnvironmentStringGCP() { + Mockito.when(EnvironmentUtils.isGoogleComputeEngine()).thenReturn(true); + Mockito.when(EC2MetadataUtils.getUserData()) + .thenThrow(new SdkClientException("Unable to contact EC2 metadata service.")); + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(2, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.OS_FORMAT, OSUtils.OS_NAME), + info.get(0)); + Assert.assertEquals(UpdateCheckUtils.GCE_KEY, info.get(1)); + } + + @Test + public void userAgentEnvironmentStringEC2AMI() { + String randomProductCode = "random123code"; + Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); + Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); + // When no user data in this ec2, null is returned + Mockito.when(EC2MetadataUtils.getUserData()).thenReturn(null); + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(3, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.OS_FORMAT, OSUtils.OS_NAME), + info.get(0)); + Assert.assertEquals(String.format(UpdateCheckUtils.PRODUCT_CODE_FORMAT, randomProductCode), + info.get(1)); + Assert.assertEquals(UpdateCheckUtils.EC2_KEY, info.get(2)); + } + + @Test + public void userAgentEnvironmentStringEC2CFT() { + String randomProductCode = "random123code"; + Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); + Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); + Mockito.when(EnvironmentUtils.isCFT(Mockito.anyString())).thenReturn(true); + Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("{ \"cft_configure\": {}}"); + + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(4, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.PRODUCT_CODE_FORMAT, randomProductCode), + info.get(1)); + Assert.assertEquals(UpdateCheckUtils.CFT_KEY, info.get(2)); + Assert.assertEquals(UpdateCheckUtils.EC2_KEY, info.get(3)); + } + + @Test + public void userAgentEnvironmentStringEC2EMR() { + String randomProductCode = "random123code"; + Mockito.when(EnvironmentUtils.isEC2()).thenReturn(true); + Mockito.when(EnvironmentUtils.getEC2ProductCode()).thenReturn(randomProductCode); + Mockito.when(EnvironmentUtils.isEMR(Mockito.anyString())).thenReturn(true); + Mockito.when(EC2MetadataUtils.getUserData()).thenReturn("emr_apps"); + List info = new ArrayList<>(); + UpdateCheckUtils.addUserAgentEnvironments(info); + Assert.assertEquals(4, info.size()); + Assert.assertEquals(String.format(UpdateCheckUtils.PRODUCT_CODE_FORMAT, randomProductCode), + info.get(1)); + Assert.assertEquals(UpdateCheckUtils.EMR_KEY, info.get(2)); + Assert.assertEquals(UpdateCheckUtils.EC2_KEY, info.get(3)); + } + + @Test + public void userAgent() { + String userAgentString = UpdateCheckUtils.getUserAgentString("cluster1", + CommonUtils.ProcessType.MASTER, new ArrayList<>()); + Pattern pattern = Pattern.compile( + String.format("Alluxio\\/%s \\(cluster1(?:.+)[^;]\\)", ProjectConstants.VERSION)); + Matcher matcher = pattern.matcher(userAgentString); + Assert.assertTrue(matcher.matches()); + } +} diff --git a/dora/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java b/dora/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java index 32be555944d5..16c95753cb8e 100644 --- a/dora/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java +++ b/dora/core/server/master/src/main/java/alluxio/master/meta/DefaultMetaMaster.java @@ -365,7 +365,7 @@ public void start(Boolean isPrimary) throws IOException { if (updateCheck && !Configuration.getBoolean(PropertyKey.TEST_MODE)) { // never start update check thread if in test mode getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UPDATE_CHECK, - new UpdateChecker(this), + new MasterUpdateChecker(this), () -> new FixedIntervalSupplier( Configuration.getMs(PropertyKey.MASTER_UPDATE_CHECK_INTERVAL)), Configuration.global(), mMasterContext.getUserState())); diff --git a/dora/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java b/dora/core/server/master/src/main/java/alluxio/master/meta/MasterUpdateChecker.java similarity index 80% rename from dora/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java rename to dora/core/server/master/src/main/java/alluxio/master/meta/MasterUpdateChecker.java index 7bfdfb6e77c2..864c987d67c1 100644 --- a/dora/core/server/master/src/main/java/alluxio/master/meta/UpdateChecker.java +++ b/dora/core/server/master/src/main/java/alluxio/master/meta/MasterUpdateChecker.java @@ -12,8 +12,9 @@ package alluxio.master.meta; import alluxio.ProjectConstants; -import alluxio.check.UpdateCheck; import alluxio.heartbeat.HeartbeatExecutor; +import alluxio.util.CommonUtils; +import alluxio.util.UpdateCheckUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,18 +27,17 @@ * Periodically Alluxio version update check. */ @NotThreadSafe -public final class UpdateChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(UpdateChecker.class); +public final class MasterUpdateChecker implements HeartbeatExecutor { + private static final Logger LOG = LoggerFactory.getLogger(MasterUpdateChecker.class); private static final String NUM_WORKER_INFO_FORMAT = "numWorkers:%s"; - private final MetaMaster mMetaMaster; /** - * Creates a new instance of {@link UpdateChecker}. + * Creates a new instance of {@link MasterUpdateChecker}. * * @param metaMaster the meta master */ - public UpdateChecker(DefaultMetaMaster metaMaster) { + public MasterUpdateChecker(DefaultMetaMaster metaMaster) { mMetaMaster = metaMaster; } @@ -53,9 +53,8 @@ public void heartbeat(long timeLimitMs) { // TODO(lu) use -1 here since we cannot distinguish // no worker vs cluster not ready (still registering) cases clusterSize > 0 ? clusterSize : -1)); - String latestVersion = - UpdateCheck.getLatestVersion(mMetaMaster.getClusterID(), additionalInfo, - 3000, 3000, 3000); + String latestVersion = UpdateCheckUtils.getLatestVersion(mMetaMaster.getClusterID(), + CommonUtils.ProcessType.MASTER, additionalInfo); if (!ProjectConstants.VERSION.equals(latestVersion)) { LOG.info("The latest version (" + latestVersion + ") is not the same " + "as the current version (" + ProjectConstants.VERSION + "). To upgrade " diff --git a/dora/integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java b/dora/integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java index 3d2db239f35b..bb674932d604 100644 --- a/dora/integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java +++ b/dora/integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java @@ -11,8 +11,6 @@ package alluxio.fuse; -import static alluxio.fuse.options.FuseOptions.FUSE_UPDATE_CHECK_ENABLED; - import alluxio.Constants; import alluxio.ProjectConstants; import alluxio.RuntimeConstants; @@ -26,7 +24,7 @@ import alluxio.conf.Source; import alluxio.exception.runtime.FailedPreconditionRuntimeException; import alluxio.exception.runtime.InvalidArgumentRuntimeException; -import alluxio.fuse.meta.UpdateChecker; +import alluxio.fuse.meta.FuseUpdateChecker; import alluxio.fuse.options.FuseCliOptions; import alluxio.fuse.options.FuseOptions; import alluxio.fuse.options.MountOptions; @@ -98,10 +96,15 @@ protected void startCommon(AlluxioConfiguration conf, } startJvmMonitorProcess(); ExecutorService executor = null; - if (fuseOptions.updateCheckEnabled()) { + // updateCheck is false only if configurable and not enabled + boolean updateCheck = true; + if (Boolean.parseBoolean(ProjectConstants.UPDATE_CHECK_CONFIGURABLE)) { + updateCheck = Configuration.getBoolean(PropertyKey.FUSE_UPDATE_CHECK_ENABLED); + } + if (updateCheck && !Configuration.getBoolean(PropertyKey.TEST_MODE)) { executor = Executors.newSingleThreadExecutor(); executor.submit(new HeartbeatThread(HeartbeatContext.FUSE_UPDATE_CHECK, - UpdateChecker.create(fuseOptions), () -> new FixedIntervalSupplier(Constants.DAY_MS), + new FuseUpdateChecker(fuseOptions), () -> new FixedIntervalSupplier(Constants.DAY_MS), Configuration.global(), UserState.Factory.create(conf))); } try (FileSystem fs = createBaseFileSystem(fsContext, fuseOptions)) { @@ -296,10 +299,6 @@ protected static InstancedConfiguration parseCliOptionsAsConfig(FuseCliOptions c InstancedConfiguration conf = new InstancedConfiguration(new AlluxioProperties()); cli.getMountPoint() .ifPresent(mp -> conf.set(PropertyKey.FUSE_MOUNT_POINT, mp, Source.RUNTIME)); - cli.getUpdateCheck() - .ifPresent(updateCheckEnabled -> { - conf.set(FUSE_UPDATE_CHECK_ENABLED, updateCheckEnabled, Source.RUNTIME); - }); cli.getRootUfsUri() .ifPresent(ufsRootUri -> { conf.set(FuseOptions.FUSE_UFS_ROOT, ufsRootUri.toString(), Source.RUNTIME); diff --git a/dora/integration/fuse/src/main/java/alluxio/fuse/meta/FuseUpdateChecker.java b/dora/integration/fuse/src/main/java/alluxio/fuse/meta/FuseUpdateChecker.java new file mode 100644 index 000000000000..f1fa7a91500f --- /dev/null +++ b/dora/integration/fuse/src/main/java/alluxio/fuse/meta/FuseUpdateChecker.java @@ -0,0 +1,105 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.fuse.meta; + +import alluxio.ProjectConstants; +import alluxio.fuse.options.FuseOptions; +import alluxio.heartbeat.HeartbeatExecutor; +import alluxio.util.CommonUtils; +import alluxio.util.URIUtils; +import alluxio.util.UpdateCheckUtils; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * Periodically Alluxio version update check. + */ +@NotThreadSafe +public final class FuseUpdateChecker implements HeartbeatExecutor { + private static final Logger LOG = LoggerFactory.getLogger(FuseUpdateChecker.class); + static final String LOCAL_ALLUXIO_DATA_CACHE = "localAlluxioDataCache"; + static final String LOCAL_ALLUXIO_METADATA_CACHE = "localAlluxioMetadataCache"; + static final String LOCAL_KERNEL_DATA_CACHE = "localKernelDataCache"; + static final String LOCAL_FS = "local"; + private final String mInstanceId = UUID.randomUUID().toString(); + private final List mFuseInfo = new ArrayList<>(); + + /** + * Creates a {@link FuseUpdateChecker}. + * + * @param fuseOptions the fuse options + */ + public FuseUpdateChecker(FuseOptions fuseOptions) { + if (fuseOptions.getFileSystemOptions().isDataCacheEnabled()) { + mFuseInfo.add(LOCAL_ALLUXIO_DATA_CACHE); + } + if (fuseOptions.getFileSystemOptions().isMetadataCacheEnabled()) { + mFuseInfo.add(LOCAL_ALLUXIO_METADATA_CACHE); + } + if (!fuseOptions.getFuseMountOptions().contains("direct_io")) { + mFuseInfo.add(LOCAL_KERNEL_DATA_CACHE); + } + mFuseInfo.add(String.format("UnderlyingFileSystem:%s", getUnderlyingFileSystem(fuseOptions))); + } + + /** + * Heartbeat for the periodic update check. + */ + @Override + public void heartbeat(long timeLimitMs) { + try { + String latestVersion = UpdateCheckUtils.getLatestVersion(mInstanceId, + CommonUtils.ProcessType.FUSE, mFuseInfo); + if (!ProjectConstants.VERSION.equals(latestVersion)) { + LOG.info("The latest version (" + latestVersion + ") is not the same " + + "as the current version (" + ProjectConstants.VERSION + "). To upgrade " + + "visit https://www.alluxio.io/download/."); + } + } catch (Throwable t) { + LOG.debug("Unable to check for updates:", t); + } + } + + @Override + public void close() {} + + /** + * @return the underlying file system type + */ + private static String getUnderlyingFileSystem(FuseOptions fuseOptions) { + String ufsAddress = fuseOptions.getFileSystemOptions() + .getUfsFileSystemOptions().get().getUfsAddress(); + if (URIUtils.isLocalFilesystem(ufsAddress)) { + return LOCAL_FS; + } + String[] components = ufsAddress.split(":"); + if (components.length < 2) { + return "unknown"; + } + return components[0]; + } + + /** + * @return + */ + @VisibleForTesting + List getFuseInfo() { + return mFuseInfo; + } +} diff --git a/dora/integration/fuse/src/main/java/alluxio/fuse/meta/UpdateChecker.java b/dora/integration/fuse/src/main/java/alluxio/fuse/meta/UpdateChecker.java deleted file mode 100644 index bfcc6ca93f13..000000000000 --- a/dora/integration/fuse/src/main/java/alluxio/fuse/meta/UpdateChecker.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.fuse.meta; - -import alluxio.ProjectConstants; -import alluxio.check.UpdateCheck; -import alluxio.fuse.FuseConstants; -import alluxio.fuse.options.FuseOptions; -import alluxio.heartbeat.HeartbeatExecutor; -import alluxio.metrics.MetricsSystem; -import alluxio.util.URIUtils; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import javax.annotation.concurrent.NotThreadSafe; - -/** - * Periodically Alluxio version update check. - */ -@NotThreadSafe -public final class UpdateChecker implements HeartbeatExecutor { - private static final Logger LOG = LoggerFactory.getLogger(UpdateChecker.class); - static final String UNDERLYING_FS_FORMAT = "UnderlyingFileSystem:%s"; - static final String LOCAL_ALLUXIO_DATA_CACHE = "localAlluxioDataCache"; - static final String LOCAL_ALLUXIO_METADATA_CACHE = "localAlluxioMetadataCache"; - static final String LOCAL_KERNEL_DATA_CACHE = "localKernelDataCache"; - - static final String ALLUXIO_FS = "alluxio"; - static final String LOCAL_FS = "local"; - - private final String mInstanceId = UUID.randomUUID().toString(); - - private final Map mFuseOpsCounter; - private final List mUnchangeableFuseInfo; - - /** - * Creates a {@link UpdateChecker}. - * - * @param fuseOptions the fuse options - * @return the update checker - */ - public static UpdateChecker create(FuseOptions fuseOptions) { - List fuseInfo = new ArrayList<>(); - UpdateCheck.addIfTrue(isLocalAlluxioDataCacheEnabled(fuseOptions), - fuseInfo, LOCAL_ALLUXIO_DATA_CACHE); - UpdateCheck.addIfTrue(isLocalAlluxioMetadataCacheEnabled(fuseOptions), fuseInfo, - LOCAL_ALLUXIO_METADATA_CACHE); - UpdateCheck.addIfTrue(isLocalKernelDataCacheEnabled(fuseOptions), - fuseInfo, LOCAL_KERNEL_DATA_CACHE); - fuseInfo.add(String.format("UnderlyingFileSystem:%s", getUnderlyingFileSystem(fuseOptions))); - return new UpdateChecker(Collections.unmodifiableList(fuseInfo), - FuseConstants.getFuseMethodNames().stream() - .collect(Collectors.toMap(methodName -> methodName, methodName -> 0L))); - } - - private UpdateChecker(List unchangeableFuseInfo, Map fuseOpsCounter) { - mUnchangeableFuseInfo = unchangeableFuseInfo; - mFuseOpsCounter = fuseOpsCounter; - } - - /** - * Heartbeat for the periodic update check. - */ - @Override - public void heartbeat(long timeLimitMs) { - try { - String latestVersion = - UpdateCheck.getLatestVersion(mInstanceId, getFuseCheckInfo(), - 3000, 3000, 3000); - if (!ProjectConstants.VERSION.equals(latestVersion)) { - LOG.info("The latest version (" + latestVersion + ") is not the same " - + "as the current version (" + ProjectConstants.VERSION + "). To upgrade " - + "visit https://www.alluxio.io/download/."); - } - } catch (Throwable t) { - LOG.debug("Unable to check for updates:", t); - } - } - - @Override - public void close() {} - - @VisibleForTesting - List getFuseCheckInfo() { - List info = new ArrayList<>(mUnchangeableFuseInfo); - for (String fuseOps : mFuseOpsCounter.keySet()) { - mFuseOpsCounter.computeIfPresent(fuseOps, (key, value) -> { - long newCount = MetricsSystem.timer(key).getCount(); - if (newCount > value) { - info.add(fuseOps); - } - return newCount; - }); - } - return info; - } - - /** - * @return - */ - @VisibleForTesting - List getUnchangeableFuseInfo() { - return mUnchangeableFuseInfo; - } - - /** - * @return true, if local Alluxio data cache is enabled - */ - private static boolean isLocalAlluxioDataCacheEnabled(FuseOptions fuseOptions) { - return fuseOptions.getFileSystemOptions().isDataCacheEnabled(); - } - - /** - * @return true, if local Alluxio metadata cache is enabled - */ - private static boolean isLocalAlluxioMetadataCacheEnabled(FuseOptions fuseOptions) { - return fuseOptions.getFileSystemOptions().isMetadataCacheEnabled(); - } - - /** - * @return true, if local kernel data cache is enabled - */ - private static boolean isLocalKernelDataCacheEnabled(FuseOptions fuseOptions) { - return !fuseOptions.getFuseMountOptions().contains("direct_io"); - } - - /** - * @return the underlying file system type - */ - private static String getUnderlyingFileSystem(FuseOptions fuseOptions) { - if (!fuseOptions.getFileSystemOptions().getUfsFileSystemOptions().isPresent()) { - return ALLUXIO_FS; - } - String ufsAddress = fuseOptions.getFileSystemOptions() - .getUfsFileSystemOptions().get().getUfsAddress(); - if (URIUtils.isLocalFilesystem(ufsAddress)) { - return LOCAL_FS; - } - String[] components = ufsAddress.split(":"); - if (components.length < 2) { - return "unknown"; - } - return components[0]; - } -} diff --git a/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseCliOptions.java b/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseCliOptions.java index 9d34532d0993..297c14264049 100644 --- a/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseCliOptions.java +++ b/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseCliOptions.java @@ -57,18 +57,6 @@ public class FuseCliOptions { @Nullable protected MountCliOptions mMountCliOptions = new MountCliOptions(); - @Parameter( - names = {"--update-check"}, - description = "Enables or disables the FUSE version update check. " - + "Disabled by default when connecting to Alluxio system cache or Dora cache. " - + "Enabled by default when connecting an under storage directly.", - arity = 0, - required = false, - hidden = true - ) - @Nullable - protected Boolean mUpdateCheck = null; - @Parameter( names = {"-h", "--help"}, description = "Display this help message", @@ -112,13 +100,6 @@ public Optional getRootUfsUri() { return Optional.ofNullable(mRootUfsUri); } - /** - * @return if update check is enabled - */ - public Optional getUpdateCheck() { - return Optional.ofNullable(mUpdateCheck); - } - /** * @return if user specified {@code --help} */ @@ -156,12 +137,11 @@ public boolean equals(Object o) { return mHelp == that.mHelp && Objects.equals(mMountPoint, that.mMountPoint) && Objects.equals(mRootUfsUri, that.mRootUfsUri) - && Objects.equals(mMountCliOptions, that.mMountCliOptions) - && Objects.equals(mUpdateCheck, that.mUpdateCheck); + && Objects.equals(mMountCliOptions, that.mMountCliOptions); } @Override public int hashCode() { - return Objects.hash(mMountPoint, mRootUfsUri, mMountCliOptions, mUpdateCheck, mHelp); + return Objects.hash(mMountPoint, mRootUfsUri, mMountCliOptions, mHelp); } } diff --git a/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseOptions.java b/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseOptions.java index 9f9ce5730e71..6e23e13b8745 100644 --- a/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseOptions.java +++ b/dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseOptions.java @@ -30,12 +30,6 @@ */ public class FuseOptions { private static final Logger LOG = LoggerFactory.getLogger(FuseOptions.class); - public static final PropertyKey FUSE_UPDATE_CHECK_ENABLED = - PropertyKey.Builder.booleanBuilder("fuse.update.check.enabled") - .setIsBuiltIn(false) - .setDefaultValue(false) - .buildUnregistered(); - /** * The UFS root that Fuse mounts. * In standalone Fuse SDK, this is different from {@link PropertyKey#DORA_CLIENT_UFS_ROOT}. @@ -47,7 +41,6 @@ public class FuseOptions { private final FileSystemOptions mFileSystemOptions; private final Set mFuseMountOptions; - private final boolean mUpdateCheckEnabled; private final boolean mSpecialCommandEnabled; /** @@ -55,14 +48,12 @@ public class FuseOptions { * * @param fileSystemOptions the file system options * @param fuseMountOptions the FUSE mount options - * @param updateCheckEnabled whether to enable update check * @param specialCommandEnabled whether fuse special commands are enabled */ protected FuseOptions(FileSystemOptions fileSystemOptions, - Set fuseMountOptions, boolean updateCheckEnabled, boolean specialCommandEnabled) { + Set fuseMountOptions, boolean specialCommandEnabled) { mFileSystemOptions = Preconditions.checkNotNull(fileSystemOptions); mFuseMountOptions = Preconditions.checkNotNull(fuseMountOptions); - mUpdateCheckEnabled = updateCheckEnabled; mSpecialCommandEnabled = specialCommandEnabled; } @@ -80,13 +71,6 @@ public Set getFuseMountOptions() { return mFuseMountOptions; } - /** - * @return true if update check is enabled - */ - public boolean updateCheckEnabled() { - return mUpdateCheckEnabled; - } - /** * @return true if fuse special command is enabled */ @@ -100,7 +84,6 @@ public boolean specialCommandEnabled() { public static class Builder { private FileSystemOptions mFileSystemOptions; private Set mFuseMountOptions; - private boolean mUpdateCheckEnabled; private boolean mSpecialCommandEnabled; /** @@ -117,17 +100,6 @@ public Builder() { } public static Builder fromConfig(AlluxioConfiguration conf) { FuseOptions.Builder builder = new FuseOptions.Builder(); - // Set update check - final boolean updateCheckEnabled; - if (!conf.isSetByUser(FUSE_UPDATE_CHECK_ENABLED)) { - // Standalone FUSE SDK without dora distributed cache - updateCheckEnabled = !conf.getBoolean(PropertyKey.DORA_ENABLED) - && conf.isSetByUser(PropertyKey.DORA_CLIENT_UFS_ROOT); - } else { - updateCheckEnabled = conf.getBoolean(FUSE_UPDATE_CHECK_ENABLED); - } - builder.setUpdateCheckEnabled(updateCheckEnabled); - // Set mount options HashSet mountOptions = new HashSet<>(conf.getList(PropertyKey.FUSE_MOUNT_OPTIONS)); LibfuseVersion version = AlluxioFuseUtils.getLibfuseVersion(conf); @@ -212,24 +184,6 @@ public Builder setFuseMountOptions(Set fuseMountOptions) { return this; } - /** - * @return if update check is enabled - */ - public boolean isUpdateCheckEnabled() { - return mUpdateCheckEnabled; - } - - /** - * Enables or disables update check. - * - * @param updateCheckEnabled - * @return this builder - */ - public Builder setUpdateCheckEnabled(boolean updateCheckEnabled) { - mUpdateCheckEnabled = updateCheckEnabled; - return this; - } - /** * @return if fuse special commands are enabled */ @@ -254,7 +208,7 @@ public Builder setSpecialCommandEnabled(boolean specialCommandEnabled) { * @return fuse options */ public FuseOptions build() { - return new FuseOptions(mFileSystemOptions, mFuseMountOptions, mUpdateCheckEnabled, + return new FuseOptions(mFileSystemOptions, mFuseMountOptions, mSpecialCommandEnabled); } } diff --git a/dora/integration/fuse/src/test/java/alluxio/fuse/meta/FuseUpdateCheckerTest.java b/dora/integration/fuse/src/test/java/alluxio/fuse/meta/FuseUpdateCheckerTest.java new file mode 100644 index 000000000000..470a74ccaddb --- /dev/null +++ b/dora/integration/fuse/src/test/java/alluxio/fuse/meta/FuseUpdateCheckerTest.java @@ -0,0 +1,109 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.fuse.meta; + +import alluxio.client.file.options.FileSystemOptions; +import alluxio.client.file.options.UfsFileSystemOptions; +import alluxio.conf.Configuration; +import alluxio.conf.InstancedConfiguration; +import alluxio.conf.PropertyKey; +import alluxio.fuse.options.FuseOptions; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.util.List; + +/** + * Tests for {@link FuseUpdateChecker}. + */ +public class FuseUpdateCheckerTest { + private final InstancedConfiguration mConf = Configuration.copyGlobal(); + + @Test + public void UnderFileSystemLocal() { + try (FuseUpdateChecker checker = getUpdateCheckerWithUfs("/home/ec2-user/testFolder")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), + FuseUpdateChecker.LOCAL_FS)); + } + } + + @Test + public void UnderFileSystemS3() { + try (FuseUpdateChecker checker = getUpdateCheckerWithUfs("s3://alluxio-test/")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), "s3")); + } + } + + @Test + public void UnderFileSystemS3A() { + try (FuseUpdateChecker checker = getUpdateCheckerWithUfs("s3a://alluxio-test/")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), "s3a")); + } + } + + @Test + public void UnderFileSystemHdfs() { + try (FuseUpdateChecker checker = getUpdateCheckerWithUfs("hdfs://namenode:port/testFolder")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), "hdfs")); + } + } + + @Test + public void localKernelDataCacheDisabled() { + Assume.assumeTrue(Configuration.getInt(PropertyKey.FUSE_JNIFUSE_LIBFUSE_VERSION) == 2); + try (FuseUpdateChecker checker = getUpdateCheckerWithMountOptions("direct_io")) { + Assert.assertFalse(containsTargetInfo(checker.getFuseInfo(), + FuseUpdateChecker.LOCAL_KERNEL_DATA_CACHE)); + } + } + + @Test + public void localKernelDataCacheEnabled() { + try (FuseUpdateChecker checker = getUpdateCheckerWithMountOptions("kernel_cache")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), + FuseUpdateChecker.LOCAL_KERNEL_DATA_CACHE)); + } + try (FuseUpdateChecker checker = getUpdateCheckerWithMountOptions("auto_cache")) { + Assert.assertTrue(containsTargetInfo(checker.getFuseInfo(), + FuseUpdateChecker.LOCAL_KERNEL_DATA_CACHE)); + } + } + + private FuseUpdateChecker getUpdateCheckerWithUfs(String ufsAddress) { + final FileSystemOptions fileSystemOptions = + FileSystemOptions.Builder + .fromConf(mConf) + .setUfsFileSystemOptions(new UfsFileSystemOptions(ufsAddress)) + .build(); + return new FuseUpdateChecker( + FuseOptions.Builder.fromConfig(Configuration.global()) + .setFileSystemOptions(fileSystemOptions) + .build()); + } + + private FuseUpdateChecker getUpdateCheckerWithMountOptions(String mountOptions) { + mConf.set(PropertyKey.FUSE_MOUNT_OPTIONS, + PropertyKey.FUSE_MOUNT_OPTIONS.formatValue(mountOptions)); + return new FuseUpdateChecker(FuseOptions.Builder.fromConfig(mConf).build()); + } + + private boolean containsTargetInfo(List list, String targetInfo) { + for (String info : list) { + if (info.contains(targetInfo)) { + return true; + } + } + return false; + } +} diff --git a/dora/integration/fuse/src/test/java/alluxio/fuse/meta/UpdateCheckerTest.java b/dora/integration/fuse/src/test/java/alluxio/fuse/meta/UpdateCheckerTest.java deleted file mode 100644 index 1e34c6ab0751..000000000000 --- a/dora/integration/fuse/src/test/java/alluxio/fuse/meta/UpdateCheckerTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.fuse.meta; - -import alluxio.client.file.options.FileSystemOptions; -import alluxio.client.file.options.UfsFileSystemOptions; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.fuse.FuseConstants; -import alluxio.fuse.options.FuseOptions; -import alluxio.metrics.MetricsSystem; - -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Test; - -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * Tests for {@link UpdateChecker}. - */ -public class UpdateCheckerTest { - - @Test - public void UnderFileSystemLocal() { - try (UpdateChecker checker = getUpdateCheckerWithUfs("/home/ec2-user/testFolder")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), - UpdateChecker.LOCAL_FS)); - } - } - - @Test - public void UnderFileSystemS3() { - try (UpdateChecker checker = getUpdateCheckerWithUfs("s3://alluxio-test/")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), "s3")); - } - } - - @Test - public void UnderFileSystemS3A() { - try (UpdateChecker checker = getUpdateCheckerWithUfs("s3a://alluxio-test/")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), "s3a")); - } - } - - @Test - public void UnderFileSystemHdfs() { - try (UpdateChecker checker = getUpdateCheckerWithUfs("hdfs://namenode:port/testFolder")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), "hdfs")); - } - } - - @Test - public void localKernelDataCacheDisabled() { - Assume.assumeTrue(Configuration.getInt(PropertyKey.FUSE_JNIFUSE_LIBFUSE_VERSION) == 2); - try (UpdateChecker checker = getUpdateCheckerWithMountOptions("direct_io")) { - Assert.assertFalse(containsTargetInfo(checker.getUnchangeableFuseInfo(), - UpdateChecker.LOCAL_KERNEL_DATA_CACHE)); - } - } - - @Test - public void localKernelDataCacheEnabled() { - try (UpdateChecker checker = getUpdateCheckerWithMountOptions("kernel_cache")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), - UpdateChecker.LOCAL_KERNEL_DATA_CACHE)); - } - try (UpdateChecker checker = getUpdateCheckerWithMountOptions("auto_cache")) { - Assert.assertTrue(containsTargetInfo(checker.getUnchangeableFuseInfo(), - UpdateChecker.LOCAL_KERNEL_DATA_CACHE)); - } - } - - @Test - public void FuseOpsCalled() { - try (UpdateChecker checker = UpdateChecker.create( - FuseOptions.Builder.fromConfig(Configuration.global()).build())) { - MetricsSystem.timer(FuseConstants.FUSE_READ).update(5, TimeUnit.MILLISECONDS); - Assert.assertTrue(containsTargetInfo(checker.getFuseCheckInfo(), FuseConstants.FUSE_READ)); - MetricsSystem.timer(FuseConstants.FUSE_WRITE).update(5, TimeUnit.MILLISECONDS); - List checkInfo = checker.getFuseCheckInfo(); - Assert.assertFalse(containsTargetInfo(checkInfo, FuseConstants.FUSE_READ)); - Assert.assertTrue(containsTargetInfo(checkInfo, FuseConstants.FUSE_WRITE)); - } - } - - private UpdateChecker getUpdateCheckerWithUfs(String ufsAddress) { - final FileSystemOptions fileSystemOptions = - FileSystemOptions.Builder - .fromConf(Configuration.global()) - .setUfsFileSystemOptions(new UfsFileSystemOptions(ufsAddress)) - .build(); - return UpdateChecker.create( - FuseOptions.Builder.fromConfig(Configuration.global()) - .setFileSystemOptions(fileSystemOptions) - .setUpdateCheckEnabled(false) - .build()); - } - - private UpdateChecker getUpdateCheckerWithMountOptions(String mountOptions) { - InstancedConfiguration conf = Configuration.copyGlobal(); - conf.set(PropertyKey.FUSE_MOUNT_OPTIONS, - PropertyKey.FUSE_MOUNT_OPTIONS.formatValue(mountOptions)); - return UpdateChecker.create(FuseOptions.Builder.fromConfig(conf).build()); - } - - private boolean containsTargetInfo(List list, String targetInfo) { - for (String info : list) { - if (info.contains(targetInfo)) { - return true; - } - } - return false; - } -} diff --git a/dora/integration/fuse/src/test/java/alluxio/fuse/options/FuseCliOptionsTest.java b/dora/integration/fuse/src/test/java/alluxio/fuse/options/FuseCliOptionsTest.java index 64c6be2bbf8a..dc1dc9927dda 100644 --- a/dora/integration/fuse/src/test/java/alluxio/fuse/options/FuseCliOptionsTest.java +++ b/dora/integration/fuse/src/test/java/alluxio/fuse/options/FuseCliOptionsTest.java @@ -55,12 +55,6 @@ public void testGetRootUfsUri() { assertEquals(Optional.of(new AlluxioURI("scheme://host/path")), mOptions.getRootUfsUri()); } - @Test - public void testGetUpdateCheck() { - mJCommander.parse("-m", "/tmp/fuse-mp", "--update-check"); - assertEquals(Optional.of(true), mOptions.getUpdateCheck()); - } - @Test public void testGetHelp() { mJCommander.parse("--help"); diff --git a/dora/integration/fuse/src/test/java/alluxio/fuse/ufs/AbstractFuseFileSystemTest.java b/dora/integration/fuse/src/test/java/alluxio/fuse/ufs/AbstractFuseFileSystemTest.java index b92347beef4e..b0e6556682c3 100644 --- a/dora/integration/fuse/src/test/java/alluxio/fuse/ufs/AbstractFuseFileSystemTest.java +++ b/dora/integration/fuse/src/test/java/alluxio/fuse/ufs/AbstractFuseFileSystemTest.java @@ -47,7 +47,6 @@ public void beforeActions() { .setFileSystemOptions(FileSystemOptions.Builder.fromConf(mConf) .setUfsFileSystemOptions(mUfsOptions) .build()) - .setUpdateCheckEnabled(false) .build(); mFuseFs = new AlluxioJniFuseFileSystem(mContext, mFileSystem, fuseOptions); mFileStat = FileStat.of(ByteBuffer.allocateDirect(256)); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/AbstractFuseFileSystemTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/AbstractFuseFileSystemTest.java index a16d9bd0e10c..1ff49f1ba9f4 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/AbstractFuseFileSystemTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/AbstractFuseFileSystemTest.java @@ -41,7 +41,6 @@ public void beforeActions() { mFuseFs = new AlluxioJniFuseFileSystem(mContext, mFileSystem, FuseOptions.Builder.fromConfig(Configuration.global()) .setFileSystemOptions(fileSystemOptions) - .setUpdateCheckEnabled(false) .build()); mFileStat = FileStat.of(ByteBuffer.allocateDirect(256)); mFileInfo = new AlluxioFuseUtils.CloseableFuseFileInfo(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/FuseEndToEndTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/FuseEndToEndTest.java index 2fd4644ebe26..a52cb573d155 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/FuseEndToEndTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/FuseEndToEndTest.java @@ -85,7 +85,6 @@ public static void beforeClass() throws Exception { AlluxioJniFuseFileSystem fuseFileSystem = new AlluxioJniFuseFileSystem(context, fileSystem, FuseOptions.Builder.fromConfig(Configuration.global()) .setFileSystemOptions(fileSystemOptions) - .setUpdateCheckEnabled(false) .build()); fuseFileSystem.mount(false, false, new HashSet<>()); if (!FuseUtils.waitForFuseMounted(MOUNT_POINT)) { diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/hdfs3/AbstractFuseHdfsIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/hdfs3/AbstractFuseHdfsIntegrationTest.java index 49bf0c6db820..d3e9e625667f 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/hdfs3/AbstractFuseHdfsIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/hdfs3/AbstractFuseHdfsIntegrationTest.java @@ -119,7 +119,6 @@ private void mountFuse() throws IOException { mAlluxioClusterResource.get().getClient(), FuseOptions.Builder.fromConfig(Configuration.global()) .setFileSystemOptions(fileSystemOptions) - .setUpdateCheckEnabled(false) .build()); mFuseFileSystem.mount(false, false, new HashSet<>()); if (!FuseUtils.waitForFuseMounted(MOUNT_POINT)) { diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/readonly/AbstractFuseFileSystemTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/readonly/AbstractFuseFileSystemTest.java index 17da81f8c67d..5dc72a6be43a 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/readonly/AbstractFuseFileSystemTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/dora/readonly/AbstractFuseFileSystemTest.java @@ -38,7 +38,6 @@ public void beforeActions() { mFuseFs = new AlluxioJniFuseFileSystem(mContext, mFileSystem, FuseOptions.Builder.fromConfig(Configuration.global()) .setFileSystemOptions(fileSystemOptions) - .setUpdateCheckEnabled(false) .build()); mFileStat = FileStat.of(ByteBuffer.allocateDirect(256)); mFileInfo = new AlluxioFuseUtils.CloseableFuseFileInfo();