Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache template directory in init command #1896

Merged
merged 34 commits into from
Aug 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
afdcf36
Cache template directory in init command
hpmellema Aug 1, 2023
320244e
Switch to using rev-list instead of git status
hpmellema Aug 2, 2023
7de3c28
Add cache interface and update clean method to clear caches
hpmellema Aug 2, 2023
2bdb9d5
Merge branch 'main' into cache-template-repo
hpmellema Aug 3, 2023
a82c616
Merge branch 'main' into cache-template-repo
hpmellema Aug 3, 2023
4704d7b
Fix error introduced by merge commit
hpmellema Aug 3, 2023
a5ee24d
Add two integration tests for template caching
hpmellema Aug 3, 2023
b79ce28
Correct checkstyle error
hpmellema Aug 3, 2023
73700f3
Try to correct issue with rmdir on windows
hpmellema Aug 3, 2023
5e39597
Revert "Try to correct issue with rmdir on windows"
hpmellema Aug 3, 2023
a3333c5
Force cache tests to run in sequence
hpmellema Aug 3, 2023
1aeb0c4
Revert "Force cache tests to run in sequence"
hpmellema Aug 3, 2023
37c2546
Disable fs monitor on windows runners
hpmellema Aug 3, 2023
dc9959c
Update workflow step
hpmellema Aug 3, 2023
f13723f
Revert "Update workflow step"
hpmellema Aug 3, 2023
d7131b9
Revert "Disable fs monitor on windows runners"
hpmellema Aug 3, 2023
da20fae
Set temp dir in windows runner
hpmellema Aug 3, 2023
a9818cb
Update workflow step
hpmellema Aug 3, 2023
f5cf9ec
Revert "Update workflow step"
hpmellema Aug 3, 2023
c9cca57
Revert "Set temp dir in windows runner"
hpmellema Aug 3, 2023
8e56d37
Disable fs monitor daemon
hpmellema Aug 3, 2023
0b8e2dd
Revert "Disable fs monitor daemon"
hpmellema Aug 4, 2023
5c67f5b
Attempt to force packfiles to be writeable as workaround for windows …
hpmellema Aug 4, 2023
92b3adb
Attempt to force permissions in workflow step
hpmellema Aug 4, 2023
1417ad4
Update workflow step
hpmellema Aug 4, 2023
b108b60
remove unused import
hpmellema Aug 4, 2023
76163a8
Update workaround
hpmellema Aug 4, 2023
0e021c4
Remove unused imports
hpmellema Aug 4, 2023
75e23e0
Fix typo
hpmellema Aug 4, 2023
7b0a45d
Add temporary debug print statement
hpmellema Aug 4, 2023
df67398
Update temporary debug statement
hpmellema Aug 4, 2023
12125b3
Allow long git filenames on ms runners
hpmellema Aug 4, 2023
1989e69
Correct workflow step ordering
hpmellema Aug 4, 2023
20d65e6
Remove temporary debug statement
hpmellema Aug 4, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ jobs:
- name: Clean, build and javadoc
run: ./gradlew clean build javadoc -Plog-tests --stacktrace

- name: Allow long file names in git for windows
if: matrix.os == 'windows-latest'
run: git config --system core.longpaths true

- name: CLI integration tests
if: matrix.java >= 17
run: ./gradlew :smithy-cli:integ -Plog-tests --stacktrace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,33 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasLength;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.junit.jupiter.api.Test;
import software.amazon.smithy.utils.IoUtils;
import software.amazon.smithy.utils.ListUtils;


public class CleanCommandTest {
private static final String PROJECT_NAME = "simple-config-sources";
@Test
public void exitNormallyIfBuildDirMissing() {
IntegUtils.run("simple-config-sources", ListUtils.of("clean"), result -> {
IntegUtils.run(PROJECT_NAME, ListUtils.of("clean"), result -> {
assertThat(result.getExitCode(), equalTo(0));
assertThat(result.getOutput(), hasLength(0));
});
}

@Test
public void deletesContentsOfBuildDir() {
IntegUtils.withProject("simple-config-sources", root -> {
IntegUtils.withProject(PROJECT_NAME, root -> {
try {
Path created = Files.createDirectories(root.resolve("build").resolve("smithy").resolve("foo"));
assertThat(Files.exists(created), is(true));
Expand All @@ -35,4 +42,52 @@ public void deletesContentsOfBuildDir() {
}
});
}

@Test
public void cleanRemovesAllCacheDirectories() throws IOException {
IntegUtils.clearCacheDirIfExists();
try {
Files.createDirectories(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH);
assertTrue(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH)
&& Files.isDirectory(IntegUtils.SMITHY_ROOT_CACHE_PATH));
IntegUtils.run(PROJECT_NAME, ListUtils.of("clean"), result -> {
assertThat(result.getExitCode(), equalTo(0));
assertThat(result.getOutput(), hasLength(0));
assertFalse(Files.exists(IntegUtils.SMITHY_ROOT_CACHE_PATH));
});
} finally {
IntegUtils.clearCacheDirIfExists();
}
}

@Test
public void cleanWithTemplateOptionRemovesOnlyTemplateDir() throws IOException {
IntegUtils.clearCacheDirIfExists();
try {
Files.createDirectories(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH);
assertTrue(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH)
&& Files.isDirectory(IntegUtils.SMITHY_ROOT_CACHE_PATH));

IntegUtils.withProject(PROJECT_NAME, root -> {
Path created = null;
try {
created = Files.createDirectories(root.resolve("build").resolve("smithy").resolve("foo"));
assertThat(Files.exists(created), is(true));
RunResult result = IntegUtils.run(root, ListUtils.of("clean", "--templates"));
assertThat(Files.exists(created), is(true));
assertThat(result.getExitCode(), is(0));
assertTrue(Files.exists(IntegUtils.SMITHY_ROOT_CACHE_PATH));
assertFalse(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH));
} catch (IOException e) {
throw new UncheckedIOException(e);
} finally {
if (created != null) {
IoUtils.rmdir(created);
}
}
});
} finally {
IntegUtils.clearCacheDirIfExists();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.emptyString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import org.junit.jupiter.api.Test;
import software.amazon.smithy.utils.IoUtils;
import software.amazon.smithy.utils.ListUtils;
Expand Down Expand Up @@ -295,6 +300,54 @@ public void badIncludePathFailureExpected() {
});
}

@Test
public void cacheCreatedOnFirstCreationOfTemplate() {
IntegUtils.clearCacheDirIfExists();
IntegUtils.withProject(PROJECT_NAME, root -> {
try {
RunResult resultFirst = IntegUtils.run(root, ListUtils.of("init", "-o", "hello-world"));
assertTrue(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH)
&& Files.isDirectory(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH));
assertThat(resultFirst.getExitCode(), equalTo(0));
assertThat(resultFirst.getOutput(),
containsString("template repo cloned"));
assertThat(resultFirst.getOutput(),
containsString("Smithy project created in directory: hello-world"));
IoUtils.rmdir(root.resolve("hello-world"));

RunResult resultSecond = IntegUtils.run(root, ListUtils.of("init", "-o", "hello-world"));
assertTrue(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH)
&& Files.isDirectory(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH));
assertThat(resultSecond.getExitCode(), equalTo(0));
assertThat(resultSecond.getOutput(), not(containsString("template repo cloned")));
assertThat(resultSecond.getOutput(),
containsString("Smithy project created in directory: hello-world"));
} finally {
IntegUtils.clearCacheDirIfExists();
}
});
}

@Test
public void noCacheCreatedWhenLocalRepo() {
IntegUtils.clearCacheDirIfExists();
IntegUtils.withProject(PROJECT_NAME, templatesDir -> {
setupTemplatesDirectory(templatesDir);

IntegUtils.withTempDir("exitZero", dir -> {
RunResult result = IntegUtils.run(
dir, ListUtils.of("init", "-t", "quickstart-cli", "-u", templatesDir.toString()));
assertThat(result.getOutput(),
containsString("Smithy project created in directory: quickstart-cli"));
assertThat(result.getExitCode(), is(0));
assertThat(Files.exists(Paths.get(dir.toString(), "quickstart-cli")), is(true));

assertFalse(Files.exists(IntegUtils.SMITHY_TEMPLATE_CACHE_PATH));
assertFalse(Files.exists(IntegUtils.SMITHY_ROOT_CACHE_PATH));
});
});
}

private static void run(List<String> args, Path root) {
StringBuilder output = new StringBuilder();
int result = IoUtils.runCommand(args, root, output, Collections.emptyMap());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@
import software.amazon.smithy.utils.MapUtils;

public final class IntegUtils {

public static final Path SMITHY_ROOT_CACHE_PATH = Paths.get(System.getProperty("java.io.tmpdir"))
.resolve("smithy-cache");
public static final Path SMITHY_TEMPLATE_CACHE_PATH = SMITHY_ROOT_CACHE_PATH.resolve("templates");
private static final Logger LOGGER = Logger.getLogger(IntegUtils.class.getName());

private IntegUtils() {}
Expand Down Expand Up @@ -146,4 +148,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
}
});
}

public static void clearCacheDirIfExists() {
IoUtils.rmdir(SMITHY_ROOT_CACHE_PATH);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
import java.util.logging.Logger;
import software.amazon.smithy.build.SmithyBuild;
import software.amazon.smithy.build.model.SmithyBuildConfig;
import software.amazon.smithy.cli.ArgumentReceiver;
import software.amazon.smithy.cli.Arguments;
import software.amazon.smithy.cli.Command;
import software.amazon.smithy.cli.HelpPrinter;
import software.amazon.smithy.utils.IoUtils;

final class CleanCommand implements Command {
Expand All @@ -40,20 +42,32 @@ public String getName() {

@Override
public String getSummary() {
return "Removes Smithy build artifacts.";
return "Removes Smithy build artifacts and caches.";
}

@Override
public int execute(Arguments arguments, Env env) {
arguments.addReceiver(new ConfigOptions());
arguments.addReceiver(new Options());

CommandAction action = HelpActionWrapper.fromCommand(this, parentCommandName, this::run);
return action.apply(arguments, env);
}

private int run(Arguments arguments, Env env) {
ConfigOptions options = arguments.getReceiver(ConfigOptions.class);
SmithyBuildConfig config = options.createSmithyBuildConfig();
ConfigOptions configOptions = arguments.getReceiver(ConfigOptions.class);
Options options = arguments.getReceiver(Options.class);

if (options.cleanTemplateCache) {
LOGGER.fine(() -> "Clearing template cache.");
if (CliCache.getTemplateCache().clear()) {
LOGGER.fine(() -> "No template cache found.");
}
return 0;
}


SmithyBuildConfig config = configOptions.createSmithyBuildConfig();
Path dir = config.getOutputDirectory()
.map(Paths::get)
.orElseGet(SmithyBuild::getDefaultOutputDirectory);
Expand All @@ -62,6 +76,35 @@ private int run(Arguments arguments, Env env) {
LOGGER.fine(() -> "Directory does not exist: " + dir);
}
LOGGER.fine(() -> "Deleted directory " + dir);

LOGGER.fine(() -> "Clearing all caches.");
if (!CliCache.clearAll()) {
LOGGER.fine(() -> "No caches found.");
}

return 0;
}


private static final class Options implements ArgumentReceiver {
private Boolean cleanTemplateCache = false;

@Override
public boolean testOption(String name) {
switch (name) {
case "--templates":
case "-t":
cleanTemplateCache = true;
return true;
default:
return false;
}
}

@Override
public void registerHelp(HelpPrinter printer) {
printer.param("--templates", "-t", null,
"Clean only the templates cache.");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package software.amazon.smithy.cli.commands;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import software.amazon.smithy.cli.CliError;
import software.amazon.smithy.utils.IoUtils;

interface CliCache {
Path DEFAULT_TEMP_DIR = Paths.get(System.getProperty("java.io.tmpdir"));
Path ROOT_CACHE_DIR = DEFAULT_TEMP_DIR.resolve("smithy-cache");

static CliCache getTemplateCache() {
return () -> ROOT_CACHE_DIR.resolve("templates");
}

Path getPath();

default boolean clear() {
return IoUtils.rmdir(getPath());
}

default Path get() {
Path cachePath = getPath();
if (Files.exists(cachePath)) {
return cachePath;
}
// If cache dir does not exist, create it and all required parent directories
try {
return Files.createDirectories(cachePath);
} catch (IOException e) {
throw new CliError("Could not create cache at path: " + cachePath);
}
}

static boolean clearAll() {
return IoUtils.rmdir(ROOT_CACHE_DIR);
}
}
Loading