Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: migrate to dynamic properties #108

Merged
merged 5 commits into from
Jan 8, 2025
Merged

refactor: migrate to dynamic properties #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c2e7913
refactor: migrate Git to dynamic properties
mgabelle Jan 7, 2025
ac37127
refactor: migrate AbstractCloningTask
mgabelle Jan 7, 2025
e9ac62c
refactor: migrate AbstractPushTask
mgabelle Jan 7, 2025
13b2472
refactor: migrate Clone and Push task
mgabelle Jan 7, 2025
62d2d35
refactor: migrate SyncFlows, PushFlow, SyncNamespaceFiles, PushNamesp…
mgabelle Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/io/kestra/plugin/git/AbstractCloningTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
Expand All @@ -13,6 +14,5 @@ public abstract class AbstractCloningTask extends AbstractGitTask {
@Schema(
title = "Whether to clone submodules."
)
@PluginProperty
protected Boolean cloneSubmodules;
protected Property<Boolean> cloneSubmodules;
}
32 changes: 11 additions & 21 deletions src/main/java/io/kestra/plugin/git/AbstractGitTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.git.services.SshTransportConfigCallback;
Expand All @@ -23,63 +24,52 @@ public abstract class AbstractGitTask extends Task {
@Schema(
title = "The URI to clone from."
)
@PluginProperty(dynamic = true)
protected String url;
protected Property<String> url;

@Schema(
title = "The username or organization."
)
@PluginProperty(dynamic = true)
protected String username;
protected Property<String> username;

@Schema(
title = "The password or Personal Access Token (PAT). When you authenticate the task with a PAT, any flows or files pushed to Git from Kestra will be pushed from the user associated with that PAT. This way, you don't need to configure the commit author (the `authorName` and `authorEmail` properties)."
)
@PluginProperty(dynamic = true)
protected String password;
protected Property<String> password;

@Schema(
title = "PEM-format private key content that is paired with a public key registered on Git.",
description = "To generate an ECDSA PEM format key from OpenSSH, use the following command: `ssh-keygen -t ecdsa -b 256 -m PEM`. " +
"You can then set this property with your private key content and put your public key on Git."
)
@PluginProperty(dynamic = true)
protected String privateKey;
protected Property<String> privateKey;

@Schema(
title = "The passphrase for the `privateKey`."
)
@PluginProperty(dynamic = true)
protected String passphrase;
protected Property<String> passphrase;


@Schema(
title = "The initial Git branch."
)
@PluginProperty(dynamic = true)
public abstract String getBranch();
public abstract Property<String> getBranch();

public <T extends TransportCommand<T, ?>> T authentified(T command, RunContext runContext) throws Exception {
if (this.username != null && this.password != null) {
command.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
runContext.render(this.username),
runContext.render(this.password)
runContext.render(this.username).as(String.class).orElseThrow(),
runContext.render(this.password).as(String.class).orElseThrow()
));
}

if (this.privateKey != null) {
command.setTransportConfigCallback(new SshTransportConfigCallback(
runContext.render(this.privateKey).getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase)
runContext.render(this.privateKey).as(String.class).orElseThrow().getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase).as(String.class).orElse(null)
));
}

return command;
}

protected void detectPasswordLeaks() {
if (this.password != null && !PEBBLE_TEMPLATE_PATTERN.matcher(this.password).find()) {
throw new IllegalArgumentException("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).");
}
}
}
47 changes: 22 additions & 25 deletions src/main/java/io/kestra/plugin/git/AbstractPushTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -46,41 +46,37 @@
@NoArgsConstructor
@Getter
public abstract class AbstractPushTask<O extends AbstractPushTask.Output> extends AbstractCloningTask implements RunnableTask<O> {
@PluginProperty(dynamic = true)
protected String commitMessage;
protected Property<String> commitMessage;

@Schema(
title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

@Schema(
title = "The commit author email.",
description = "If null, no author will be set on this commit."
)
@PluginProperty(dynamic = true)
private String authorEmail;
private Property<String> authorEmail;

@Schema(
title = "The commit author name.",
description = "If null, the username will be used instead.",
defaultValue = "`username`"
)
@PluginProperty(dynamic = true)
private String authorName;
private Property<String> authorName;

public abstract String getCommitMessage();
public abstract Property<String> getCommitMessage();

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract Object globs();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElse(null)));
flowDirectory.toFile().mkdirs();
return flowDirectory;
}
Expand Down Expand Up @@ -130,14 +126,16 @@ protected void writeResourceFile(Path path, InputStream inputStream) throws IOEx
Files.copy(inputStream, path, REPLACE_EXISTING);
}

private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException {
private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException, IllegalVariableEvaluationException {
File diffFile = runContext.workingDir().createTempFile(".ion").toFile();
boolean dryRunValue = runContext.render(this.dryRun).as(Boolean.class).orElseThrow();

try (DiffFormatter diffFormatter = new DiffFormatter(null);
BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
diffFormatter.setRepository(git.getRepository());

DiffCommand diff = git.diff();
if (this.dryRun) {
if (dryRunValue) {
diff = diff.setCached(true);
} else {
diff = diff.setOldTree(treeIterator(git, "HEAD~1"))
Expand Down Expand Up @@ -204,21 +202,21 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
String commitId = null;
ObjectId commit;
try {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url));
if (this.isDryRun()) {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url).as(String.class).orElse(null));
if (runContext.render(this.getDryRun()).as(Boolean.class).orElseThrow()) {
logger.info(
"Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false",
httpUrl
);
} else {
String renderedBranch = runContext.render(this.getBranch());
String renderedBranch = runContext.render(this.getBranch()).as(String.class).orElse(null);
logger.info(
"Pushing to {} on branch {}",
httpUrl,
renderedBranch
);

String message = runContext.render(this.getCommitMessage());
String message = runContext.render(this.getCommitMessage()).as(String.class).orElse(null);
ObjectId head = git.getRepository().resolve(Constants.HEAD);
commit = git.commit()
.setAllowEmpty(false)
Expand Down Expand Up @@ -247,13 +245,13 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
}

private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException {
String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username));
String authorEmail = this.authorEmail;
String name = runContext.render(this.authorName).as(String.class).orElse(runContext.render(this.username).as(String.class).orElse(null));
String authorEmail = runContext.render(this.authorEmail).as(String.class).orElse(null);
if (authorEmail == null || name == null) {
return null;
}

return new PersonIdent(runContext.render(name), runContext.render(authorEmail));
return new PersonIdent(name, authorEmail);
}

private String buildCommitUrl(String httpUrl, String branch, String commitId) {
Expand All @@ -274,9 +272,8 @@ public O run(RunContext runContext) throws Exception {
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());
this.detectPasswordLeaks();

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()).as(String.class).orElse(null), this.cloneSubmodules);

Path localGitDirectory = this.createGitDirectory(runContext);

Expand All @@ -291,7 +288,7 @@ public O run(RunContext runContext) throws Exception {
this.writeResourceFiles(contentByPath);

AddCommand add = git.add();
add.addFilepattern(runContext.render(this.getGitDirectory()));
add.addFilepattern(runContext.render(this.getGitDirectory()).as(String.class).orElse(null));
add.call();

Output pushOutput = this.push(git, runContext, gitService);
Expand Down
40 changes: 19 additions & 21 deletions src/main/java/io/kestra/plugin/git/AbstractSyncTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -43,24 +43,23 @@ public abstract class AbstractSyncTask<T, O extends AbstractSyncTask.Output> ext
@Schema(
title = "If `true`, the task will only output modifications without performing any modification to Kestra. If `false` (default), all listed modifications will be applied."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

public abstract boolean isDelete();
public abstract Property<Boolean> getDelete();

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElse(null)));
syncDirectory.toFile().mkdirs();
return syncDirectory;
}

protected Map<URI, Supplier<InputStream>> gitResourcesContentByUri(Path baseDirectory) throws IOException {
try (Stream<Path> paths = Files.walk(baseDirectory, this.traverseDirectories() ? MAX_VALUE : 1)) {
protected Map<URI, Supplier<InputStream>> gitResourcesContentByUri(Path baseDirectory, RunContext runContext) throws IOException, IllegalVariableEvaluationException {
try (Stream<Path> paths = Files.walk(baseDirectory, runContext.render(this.traverseDirectories()).as(Boolean.class).orElseThrow() ? MAX_VALUE : 1)) {
Stream<Path> filtered = paths.skip(1);
KestraIgnore kestraIgnore = new KestraIgnore(baseDirectory);
filtered = filtered.filter(path -> !kestraIgnore.isIgnoredFile(path.toString(), true));
Expand All @@ -78,10 +77,10 @@ protected Map<URI, Supplier<InputStream>> gitResourcesContentByUri(Path baseDire
}
}

protected boolean traverseDirectories() {
return true;
protected Property<Boolean> traverseDirectories() {
return Property.of(true);
}

protected boolean mustKeep(RunContext runContext, T instanceResource) {
return false;
}
Expand All @@ -100,7 +99,7 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
try (BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
List<SyncResult> syncResults = new ArrayList<>();

String renderedGitDirectory = runContext.render(this.getGitDirectory());
String renderedGitDirectory = runContext.render(this.getGitDirectory()).as(String.class).orElse(null);
if (deletedResources != null) {
deletedResources.stream()
.map(throwFunction(deletedResource -> wrapper(
Expand Down Expand Up @@ -146,17 +145,16 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
}

public O run(RunContext runContext) throws Exception {
this.detectPasswordLeaks();
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()).as(String.class).orElse(null), this.cloneSubmodules);

Path localGitDirectory = this.createGitDirectory(runContext);
Map<URI, Supplier<InputStream>> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory);
Map<URI, Supplier<InputStream>> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory, runContext);

String renderedNamespace = runContext.render(this.fetchedNamespace());
String renderedNamespace = runContext.render(this.fetchedNamespace()).as(String.class).orElse(null);

Map<URI, T> beforeUpdateResourcesByUri = this.fetchResources(runContext, renderedNamespace)
.stream()
Expand All @@ -170,7 +168,7 @@ public O run(RunContext runContext) throws Exception {
.map(throwFunction(e -> {
InputStream inputStream = e.getValue().get();
T resource;
if (this.dryRun) {
if (runContext.render(this.dryRun).as(Boolean.class).orElseThrow()) {
resource = this.simulateResourceWrite(runContext, renderedNamespace, e.getKey(), inputStream);
} else {
resource = this.writeResource(runContext, renderedNamespace, e.getKey(), inputStream);
Expand All @@ -191,14 +189,14 @@ public O run(RunContext runContext) throws Exception {
);

List<T> deleted;
if (this.isDelete()) {
if (runContext.render(this.getDelete()).as(Boolean.class).orElseThrow()) {
deleted = new ArrayList<>();
beforeUpdateResourcesByUri.entrySet().stream().filter(e -> !updatedResourcesByUri.containsKey(e.getKey())).forEach(throwConsumer(e -> {
if (this.mustKeep(runContext, e.getValue())) {
return;
}

if (!this.dryRun) {
if (!runContext.render(this.dryRun).as(Boolean.class).orElseThrow()) {
this.deleteResource(runContext, renderedNamespace, e.getValue());
}

Expand All @@ -215,7 +213,7 @@ public O run(RunContext runContext) throws Exception {
return output(diffFileStorageUri);
}

protected abstract List<T> fetchResources(RunContext runContext, String renderedNamespace) throws IOException;
protected abstract List<T> fetchResources(RunContext runContext, String renderedNamespace) throws IOException, IllegalVariableEvaluationException;

protected abstract URI toUri(String renderedNamespace, T resource);

Expand Down
Loading
Loading