From c04f5c02d96c7fc4ebf60f8fcd482bbf0f22b90f Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 15 Jan 2025 22:33:46 +0100 Subject: [PATCH 1/3] Extract reusable service. --- .../core/profiles/ChecksumProfileMatcher.java | 23 ++--- .../core/profiles/ProfileMatcher.java | 27 ++---- .../profiles/ProfilesSynchronizeWorker.java | 57 ++---------- .../core/profiles/ProfilesSynchronizer.java | 24 +++++ .../ProtocolFactoryProfilesSynchronizer.java | 88 +++++++++++++++++++ .../profiles/ChecksumProfileMatcherTest.java | 9 +- 6 files changed, 134 insertions(+), 94 deletions(-) create mode 100644 core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizer.java create mode 100644 core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ChecksumProfileMatcher.java b/core/src/main/java/ch/cyberduck/core/profiles/ChecksumProfileMatcher.java index 696497d0b5e..49c042c140f 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ChecksumProfileMatcher.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ChecksumProfileMatcher.java @@ -28,30 +28,19 @@ public class ChecksumProfileMatcher implements ProfileMatcher { private static final Logger log = LogManager.getLogger(ChecksumProfileMatcher.class.getName()); - /** - * Remote list of profiles - */ - private final Set repository; - - /** - * @param repository Profiles from remote repository - */ - public ChecksumProfileMatcher(final Set repository) { - this.repository = repository; - } - /** * Filter locally installed profiles by matching checksum * - * @param next Description of profile installed in application support directory + * @param repository Remote list of profiles + * @param installed Description of profile installed in application support directory * @return Non-null if matching profile is found in remote list and not latest version */ @Override - public Optional compare(final ProfileDescription next) { + public Optional compare(final Set repository, final ProfileDescription installed) { // Filter out profiles with matching checksum final Optional found = repository.stream() - .filter(description -> Objects.equals(description.getChecksum(), next.getChecksum())) - .findFirst(); + .filter(description -> Objects.equals(description.getChecksum(), installed.getChecksum())) + .findFirst(); if(found.isPresent()) { // Found matching checksum. Determine if latest version if(found.get().isLatest()) { @@ -63,7 +52,7 @@ public Optional compare(final ProfileDescription next) { return found; } } - log.warn("Local only profile {}", next); + log.warn("Local only profile {}", installed); return Optional.empty(); } } diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProfileMatcher.java b/core/src/main/java/ch/cyberduck/core/profiles/ProfileMatcher.java index f1878b601fa..06211d10906 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ProfileMatcher.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProfileMatcher.java @@ -15,31 +15,14 @@ * GNU General Public License for more details. */ -import ch.cyberduck.core.Protocol; - -import org.apache.commons.lang3.StringUtils; - import java.util.Optional; -import java.util.function.Predicate; +import java.util.Set; public interface ProfileMatcher { /** - * @param next Description of profile installed - * @return Non null if profile from server has been updated. Matching profile with later version in repository. + * @param repository Description of available profiles in repository + * @param installed Description of profile installed + * @return Non-null if profile from server has been updated. Matching profile with later version from remote repository. */ - Optional compare(ProfileDescription next); - - class IdentifierProtocolPredicate implements Predicate { - private final Protocol installed; - - public IdentifierProtocolPredicate(final Protocol installed) { - this.installed = installed; - } - - @Override - public boolean test(final Protocol protocol) { - return StringUtils.equals(installed.getIdentifier(), protocol.getIdentifier()) - && StringUtils.equals(installed.getProvider(), protocol.getProvider()); - } - } + Optional compare(Set repository, ProfileDescription installed); } diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java index e3044308b45..667627b40ad 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java @@ -26,19 +26,13 @@ import ch.cyberduck.core.transfer.symlink.DisabledDownloadSymlinkResolver; import ch.cyberduck.core.worker.Worker; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; import java.util.Set; /** * Merge local set with latest versions from server */ public class ProfilesSynchronizeWorker extends Worker> { - private static final Logger log = LogManager.getLogger(ProfilesSynchronizeWorker.class.getName()); private final ProtocolFactory registry; private final Local directory; @@ -75,54 +69,13 @@ public Set initialize() { @Override public Set run(final Session session) throws BackgroundException { - final Set returned = new HashSet<>(); // Find all locally installed profiles - final LocalProfilesFinder localProfilesFinder = new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE); - final Set installed = localProfilesFinder.find(); + final LocalProfilesFinder local = new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE); // Find all profiles from repository - final RemoteProfilesFinder remoteProfilesFinder = new RemoteProfilesFinder(registry, session, this.filter(session)); - final Set remote = remoteProfilesFinder.find(); - final ProfileMatcher matcher = new ChecksumProfileMatcher(remote); - // Iterate over every installed profile and find match in repository - installed.forEach(local -> { - // Check for matching remote checksum and download profile if this version is not equal to latest - final Optional match = matcher.compare(local); - if(match.isPresent()) { - // Found matching checksum for profile in remote list which is not marked as latest version - log.warn("Override {} with latest profile verison {}", local, match); - // Remove previous version - local.getProfile().ifPresent(registry::unregister); - // Register updated profile by copying temporary file to application support - match.get().getFile().ifPresent(value -> { - final Local copy = registry.register(value); - if(null != copy) { - final LocalProfileDescription d = new LocalProfileDescription(registry, copy); - log.debug("Add synched profile {}", d); - returned.add(d); - visitor.visit(d); - } - }); - } - else { - log.debug("Add local only profile {}", local); - returned.add(local); - visitor.visit(local); - } - }); - // Iterate over all fetched profiles and when not installed - remote.forEach(description -> { - if(description.isLatest()) { - // Check if not already added previously when syncing with local list - if(!returned.contains(description)) { - log.debug("Add remote profile {}", description); - returned.add(description); - visitor.visit(description); - } - } - }); - localProfilesFinder.cleanup(); - remoteProfilesFinder.cleanup(); - return returned; + final RemoteProfilesFinder remote = new RemoteProfilesFinder(registry, session, this.filter(session)); + return new ProtocolFactoryProfilesSynchronizer(registry, local, remote).sync( + // Match profiles by ETag and MD5 checksum of profile on disk + new ChecksumProfileMatcher(), visitor); } protected CompareFilter filter(final Session session) { diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizer.java b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizer.java new file mode 100644 index 00000000000..087017296f1 --- /dev/null +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizer.java @@ -0,0 +1,24 @@ +package ch.cyberduck.core.profiles; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.exception.BackgroundException; + +import java.util.Set; + +public interface ProfilesSynchronizer { + Set sync(ProfileMatcher matcher, ProfilesFinder.Visitor visitor) throws BackgroundException; +} diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java b/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java new file mode 100644 index 00000000000..1bbf18cdee2 --- /dev/null +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java @@ -0,0 +1,88 @@ +package ch.cyberduck.core.profiles; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Local; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.exception.BackgroundException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class ProtocolFactoryProfilesSynchronizer implements ProfilesSynchronizer { + private static final Logger log = LogManager.getLogger(ProtocolFactoryProfilesSynchronizer.class.getName()); + + private final ProtocolFactory registry; + private final LocalProfilesFinder local; + private final RemoteProfilesFinder remote; + + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final LocalProfilesFinder local, final RemoteProfilesFinder remote) { + this.registry = registry; + this.local = local; + this.remote = remote; + } + + @Override + public Set sync(final ProfileMatcher matcher, final ProfilesFinder.Visitor visitor) throws BackgroundException { + final Set result = new HashSet<>(); + final Set installed = local.find(); + final Set available = remote.find(); + // Iterate over every installed profile and find match in repository + installed.forEach(l -> { + // Check for matching remote checksum and download profile if this version is not equal to latest + final Optional match = matcher.compare(available, l); + if(match.isPresent()) { + // Found matching checksum for profile in remote list which is not marked as latest version + log.warn("Override {} with latest profile verison {}", l, match); + // Remove previous version + l.getProfile().ifPresent(registry::unregister); + // Register updated profile by copying temporary file to application support + match.get().getFile().ifPresent(value -> { + final Local copy = registry.register(value); + if(null != copy) { + final LocalProfileDescription d = new LocalProfileDescription(registry, copy); + log.debug("Add synched profile {}", d); + result.add(d); + visitor.visit(d); + } + }); + } + else { + log.debug("Add local only profile {}", l); + result.add(l); + visitor.visit(l); + } + }); + // Iterate over all fetched profiles and when not installed + available.forEach(description -> { + if(description.isLatest()) { + // Check if not already added previously when syncing with local list + if(!result.contains(description)) { + log.debug("Add remote profile {}", description); + result.add(description); + visitor.visit(description); + } + } + }); + local.cleanup(); + remote.cleanup(); + return result; + } +} diff --git a/core/src/test/java/ch/cyberduck/core/profiles/ChecksumProfileMatcherTest.java b/core/src/test/java/ch/cyberduck/core/profiles/ChecksumProfileMatcherTest.java index 25506c32c33..051bf6d6eb6 100644 --- a/core/src/test/java/ch/cyberduck/core/profiles/ChecksumProfileMatcherTest.java +++ b/core/src/test/java/ch/cyberduck/core/profiles/ChecksumProfileMatcherTest.java @@ -37,7 +37,8 @@ public void testLocalOnly() throws Exception { // Local only profile final ProfileDescription local = new ProfileDescription( ProtocolFactory.get(), new Checksum(HashAlgorithm.md5, "d41d8cd98f00b204e9800998ecf8427e"), null); - assertFalse(new ChecksumProfileMatcher(Stream.empty().collect(Collectors.toSet())).compare(local).isPresent()); + assertFalse(new ChecksumProfileMatcher().compare( + Stream.empty().collect(Collectors.toSet()), local).isPresent()); } @Test @@ -52,7 +53,8 @@ public boolean isLatest() { }; final ProfileDescription local = new ProfileDescription( ProtocolFactory.get(), new Checksum(HashAlgorithm.md5, "d41d8cd98f00b204e9800998ecf8427e"), null); - assertFalse(new ChecksumProfileMatcher(Stream.of(remote).collect(Collectors.toSet())).compare(local).isPresent()); + assertFalse(new ChecksumProfileMatcher().compare( + Stream.of(remote).collect(Collectors.toSet()), local).isPresent()); } @Test @@ -70,6 +72,7 @@ public Optional getFile() { return Optional.of(new NullLocal("Profile.cyberduckprofile")); } }; - assertTrue(new ChecksumProfileMatcher(Stream.of(remote).collect(Collectors.toSet())).compare(local).isPresent()); + assertTrue(new ChecksumProfileMatcher().compare( + Stream.of(remote).collect(Collectors.toSet()), local).isPresent()); } } From 94921b165f08676419410954b5aa1c6a4f68d17b Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 15 Jan 2025 22:43:19 +0100 Subject: [PATCH 2/3] Allow custom filename filter. --- .../core/profiles/ProfilesSynchronizeWorker.java | 8 +------- .../ch/cyberduck/core/profiles/RemoteProfilesFinder.java | 8 +++++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java index 667627b40ad..6cfcaeb8a81 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java @@ -22,8 +22,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.preferences.PreferencesFactory; import ch.cyberduck.core.preferences.SupportDirectoryFinderFactory; -import ch.cyberduck.core.transfer.download.CompareFilter; -import ch.cyberduck.core.transfer.symlink.DisabledDownloadSymlinkResolver; import ch.cyberduck.core.worker.Worker; import java.util.Collections; @@ -72,13 +70,9 @@ public Set run(final Session session) throws BackgroundEx // Find all locally installed profiles final LocalProfilesFinder local = new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE); // Find all profiles from repository - final RemoteProfilesFinder remote = new RemoteProfilesFinder(registry, session, this.filter(session)); + final RemoteProfilesFinder remote = new RemoteProfilesFinder(registry, session); return new ProtocolFactoryProfilesSynchronizer(registry, local, remote).sync( // Match profiles by ETag and MD5 checksum of profile on disk new ChecksumProfileMatcher(), visitor); } - - protected CompareFilter filter(final Session session) { - return new CompareFilter(new DisabledDownloadSymlinkResolver(), session); - } } diff --git a/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java b/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java index 728fe5dcd61..eded4530345 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java @@ -57,25 +57,27 @@ public class RemoteProfilesFinder implements ProfilesFinder { private final ProtocolFactory protocols; private final Session session; private final TransferPathFilter comparison; + private final Filter filter; public RemoteProfilesFinder(final Session session) { this(ProtocolFactory.get(), session); } public RemoteProfilesFinder(final ProtocolFactory protocols, final Session session) { - this(protocols, session, new CompareFilter(new DisabledDownloadSymlinkResolver(), session)); + this(protocols, session, new CompareFilter(new DisabledDownloadSymlinkResolver(), session), new ProfileFilter()); } - public RemoteProfilesFinder(final ProtocolFactory protocols, final Session session, final TransferPathFilter comparison) { + public RemoteProfilesFinder(final ProtocolFactory protocols, final Session session, + final TransferPathFilter comparison, final Filter filter) { this.protocols = protocols; this.session = session; this.comparison = comparison; + this.filter = filter; } @Override public Set find(final Visitor visitor) throws BackgroundException { log.info("Fetch profiles from {}", session.getHost()); - final ProfileFilter filter = new ProfileFilter(); final AttributedList list = session.getFeature(ListService.class).list(new DelegatingHomeFeature( new DefaultPathHomeFeature(session.getHost())).find(), new DisabledListProgressListener()); return list.filter(filter).toStream().map(file -> visitor.visit(new RemoteProfileDescription(protocols, file, From d3c526c360780c8151eac3a517a965708b645829 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 15 Jan 2025 23:02:27 +0100 Subject: [PATCH 3/3] Overload constructors. --- .../profiles/ProfilesSynchronizeWorker.java | 6 +--- .../ProtocolFactoryProfilesSynchronizer.java | 32 +++++++++++++++++++ .../core/profiles/RemoteProfilesFinder.java | 5 +++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java index 6cfcaeb8a81..0408d2ed1fc 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProfilesSynchronizeWorker.java @@ -67,11 +67,7 @@ public Set initialize() { @Override public Set run(final Session session) throws BackgroundException { - // Find all locally installed profiles - final LocalProfilesFinder local = new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE); - // Find all profiles from repository - final RemoteProfilesFinder remote = new RemoteProfilesFinder(registry, session); - return new ProtocolFactoryProfilesSynchronizer(registry, local, remote).sync( + return new ProtocolFactoryProfilesSynchronizer(session).sync( // Match profiles by ETag and MD5 checksum of profile on disk new ChecksumProfileMatcher(), visitor); } diff --git a/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java b/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java index 1bbf18cdee2..73518c5cd9f 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/ProtocolFactoryProfilesSynchronizer.java @@ -16,8 +16,12 @@ */ import ch.cyberduck.core.Local; +import ch.cyberduck.core.LocalFactory; import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.Session; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.preferences.PreferencesFactory; +import ch.cyberduck.core.preferences.SupportDirectoryFinderFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,6 +37,34 @@ public class ProtocolFactoryProfilesSynchronizer implements ProfilesSynchronizer private final LocalProfilesFinder local; private final RemoteProfilesFinder remote; + public ProtocolFactoryProfilesSynchronizer(final Session session) { + this(ProtocolFactory.get(), session, LocalFactory.get(SupportDirectoryFinderFactory.get().find(), + PreferencesFactory.get().getProperty("profiles.folder.name"))); + } + + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final Session session, final Local directory) { + this(ProtocolFactory.get(), + // Find all locally installed profiles + new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE), session); + } + + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final LocalProfilesFinder local, final Session session) { + this(registry, local, + // Find all profiles from repository + new RemoteProfilesFinder(registry, session)); + } + + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final RemoteProfilesFinder remote) { + this(registry, LocalFactory.get(SupportDirectoryFinderFactory.get().find(), + PreferencesFactory.get().getProperty("profiles.folder.name")), remote); + } + + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final Local directory, final RemoteProfilesFinder remote) { + this(registry, + // Find all locally installed profiles + new LocalProfilesFinder(registry, directory, ProtocolFactory.BUNDLED_PROFILE_PREDICATE), remote); + } + public ProtocolFactoryProfilesSynchronizer(final ProtocolFactory registry, final LocalProfilesFinder local, final RemoteProfilesFinder remote) { this.registry = registry; this.local = local; diff --git a/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java b/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java index eded4530345..17b721cd34c 100644 --- a/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java +++ b/core/src/main/java/ch/cyberduck/core/profiles/RemoteProfilesFinder.java @@ -67,6 +67,11 @@ public RemoteProfilesFinder(final ProtocolFactory protocols, final Session se this(protocols, session, new CompareFilter(new DisabledDownloadSymlinkResolver(), session), new ProfileFilter()); } + public RemoteProfilesFinder(final Session session, + final TransferPathFilter comparison, final Filter filter) { + this(ProtocolFactory.get(), session, comparison, filter); + } + public RemoteProfilesFinder(final ProtocolFactory protocols, final Session session, final TransferPathFilter comparison, final Filter filter) { this.protocols = protocols;