Preserve file permissions and support include / exclude filters for "…

…Use Jenkins source" (awslabs#127)

* [JENKINS-67853] Add a test to reproduce JENKINS-67853

* [JENKINS-67853] Use Archiver to creaete zip file

* Preserves file modes.
* Also preparation for includes/excludes feature.

* [JENKINS-67853] Use the new archiving method instead of `ZipSourceCallable.zipSource` in tests

* [JENKINS-67853] Archive directories explicitly to allow empty directories

* [JENKINS-67854] Add workspaceInclude / workspaceExclude feature

* [JENKINS-67854] Add test for workspaceIncludes/workspaceExcludes

Co-authored-by: Leo Herran <leobaran@amazon.com>

src/main/java/CodeBuildStep.java

@@ -55,6 +55,8 @@ public class CodeBuildStep extends AbstractStepImpl {
     @Getter private String sourceControlType;
     @Getter private String localSourcePath;
     @Getter private String workspaceSubdir;
+    @DataBoundSetter public String workspaceIncludes;
+    @DataBoundSetter public String workspaceExcludes;
     @Getter private String sourceVersion;
     @Getter private String sseAlgorithm;
     @Getter private String gitCloneDepthOverride;
@@ -645,6 +647,8 @@ protected CodeBuildResult run() throws Exception {
                     step.getInsecureSslOverride(), step.getPrivilegedModeOverride(), step.getCwlStreamingDisabled(), step.getExceptionFailureMode(),
                     step.getDownloadArtifacts(), step.getDownloadArtifactsRelativePath()
             ).readResolve();
+            builder.workspaceIncludes = step.workspaceIncludes;
+            builder.workspaceExcludes = step.workspaceExcludes;
 
             try {
                 builder.perform(run, ws, launcher, listener, getContext());

src/main/java/CodeBuilder.java

@@ -39,6 +39,7 @@
 import net.sf.json.JSONObject;
 import org.jenkinsci.plugins.workflow.steps.StepContext;
 import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.DataBoundSetter;
 import org.kohsuke.stapler.QueryParameter;
 import org.kohsuke.stapler.StaplerRequest;
 
@@ -68,6 +69,8 @@ public class CodeBuilder extends Builder implements SimpleBuildStep {
     @Getter private String sourceControlType;
     @Getter private String localSourcePath;
     @Getter private String workspaceSubdir;
+    @DataBoundSetter public String workspaceIncludes;
+    @DataBoundSetter public String workspaceExcludes;
     @Getter private String sourceVersion;
     @Getter private String sseAlgorithm;
     @Getter private String gitCloneDepthOverride;
@@ -419,7 +422,7 @@ public void perform(@Nonnull Run<?, ?> build, @Nonnull FilePath ws, @Nonnull Lau
                 return;
             }
 
-            S3DataManager s3DataManager = new S3DataManager(awsClientFactory.getS3Client(), sourceS3Bucket, sourceS3Key, getParameterized(sseAlgorithm), getParameterized(localSourcePath), getParameterized(workspaceSubdir));
+            S3DataManager s3DataManager = new S3DataManager(awsClientFactory.getS3Client(), sourceS3Bucket, sourceS3Key, getParameterized(sseAlgorithm), getParameterized(localSourcePath), getParameterized(workspaceSubdir), getParameterized(workspaceIncludes), getParameterized(workspaceExcludes));
             String uploadedSourceVersion = "";
 
             try {

src/main/java/S3DataManager.java

@@ -42,6 +42,12 @@ public class S3DataManager {
     private final String sseAlgorithm;
     private final String localSourcePath;
     private final String workspaceSubdir;
+    private final String workspaceIncludes;
+    private final String workspaceExcludes;
+
+    public S3DataManager(AmazonS3Client s3Client, String s3InputBucket, String s3InputKey, String sseAlgorithm, String localSourcePath, String workspaceSubdir) {
+        this(s3Client, s3InputBucket, s3InputKey, sseAlgorithm, localSourcePath, workspaceSubdir, null, null);
+    }
 
     // if localSourcePath is empty, clones, zips, and uploads the given workspace. Otherwise, uploads the file referred to by localSourcePath.
     // The upload bucket used is this.s3InputBucket and the name of the zip file is this.s3InputKey.
@@ -66,7 +72,7 @@ public UploadToS3Output uploadSourceToS3(TaskListener listener, FilePath workspa
             LoggingHelper.log(listener, "Zipping directory to upload to S3: " + sourcePath);
 
             localFile = new FilePath(workspace, getTempFilePath(sourcePath));
-            zipFileMD5 = localFile.act(new ZipSourceCallable(workspace));
+            zipFileMD5 = localFile.act(new ZipSourceCallable(workspace, workspaceIncludes, workspaceExcludes));
         }
 
         // Add MD5 checksum as S3 Object metadata

src/main/java/ZipSourceCallable.java

@@ -16,9 +16,18 @@
 
 import com.amazonaws.services.codebuild.model.InvalidInputException;
 import hudson.FilePath;
+import hudson.Util;
 import hudson.remoting.VirtualChannel;
+import hudson.util.io.Archiver;
+import hudson.util.io.ArchiverFactory;
 import jenkins.MasterToSlaveFileCallable;
 import org.apache.commons.io.FileUtils;
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.DirectoryScanner;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.types.FileSet;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -34,23 +43,27 @@
 public class ZipSourceCallable extends MasterToSlaveFileCallable<String> {
 
     final FilePath workspace;
+    final String includes;  // never null
+    final String excludes;  // never null
 
     public static final String zipSourceError = "zipSource usage: prefixToTrim must be contained in the given directory.";
 
     public ZipSourceCallable(FilePath workspace) {
+        this(workspace, null, null);
+    }
+
+    public ZipSourceCallable(FilePath workspace, String includes, String excludes) {
         this.workspace = workspace;
+        this.includes = Util.fixNull(includes);
+        this.excludes = Util.fixNull(excludes);
     }
 
     @Override
     public String invoke(File f, VirtualChannel channel) throws IOException {
-        String sourceFilePath = workspace.getRemote();
-
         // Create a temp file to zip into so we do not zip ourselves
         File tempFile = File.createTempFile(f.getName(), null, null);
         try(OutputStream zipFileOutputStream = new FileOutputStream(tempFile)) {
-            try(ZipOutputStream out = new ZipOutputStream(zipFileOutputStream)) {
-                zipSource(workspace, sourceFilePath, out, sourceFilePath);
-            }
+            zipSourceWithArchiver(zipFileOutputStream);
         } catch (IOException e) {
             throw e;
         } catch (Exception e) {
@@ -71,6 +84,46 @@ public String invoke(File f, VirtualChannel channel) throws IOException {
         return S3DataManager.getZipMD5(f);
     }
 
+    @Restricted(NoExternalUse.class)    // For testing purpose
+    protected void zipSourceWithArchiver(final OutputStream out) throws InvalidInputException, IOException, InterruptedException {
+        if (!workspace.exists() || !workspace.isDirectory()) {
+            throw new InvalidInputException("Empty or invalid source directory: " + workspace.getRemote());
+        }
+        Archiver archiver = ArchiverFactory.ZIP.create(out);
+        try {
+            this.zipSourceWithArchiverImpl(archiver);
+        } finally {
+            archiver.close();
+        }
+    }
+
+    private void zipSourceWithArchiverImpl(final Archiver archiver) throws InvalidInputException, IOException, InterruptedException {
+        String sourceFilePath = workspace.getRemote();
+
+        // NOTE: This code is running on the remote.
+        // FilePath.list() is really powerful, but cannot be used as it doesn't pick empty directories.
+        FileSet fs = Util.createFileSet(new File(sourceFilePath), includes, excludes);
+        fs.setDefaultexcludes(false);   // for backward compatibility
+        DirectoryScanner ds;
+        try {
+            ds = fs.getDirectoryScanner(new Project());
+        } catch (BuildException e) {
+            throw new IOException(e.getMessage());
+        }
+        // To include directories with no files
+        for (String dir: ds.getIncludedDirectories()) {
+            if ("".equals(dir)) {
+                // skip the top directory to make an invalid archive if no files.
+                // (backward compatibility)
+                continue;
+            }
+            archiver.visit(new File(sourceFilePath, dir), dir);
+        }
+        for (String file: ds.getIncludedFiles()) {
+            archiver.visit(new File(sourceFilePath, file), file);
+        }
+    }
+
     // Recursively zips everything in the given directory into a zip file using the given ZipOutputStream.
     // @param directory: whose contents will be zipped.
     // @param out: ZipOutputStream that will write data to its zip file.
@@ -83,6 +136,8 @@ public String invoke(File f, VirtualChannel channel) throws IOException {
     //     The given directory is /tmp/dir/folder/ which contains one file /tmp/dir/folder/file.txt
     //     The given prefixToTrim is /tmp/dir/folder
     //     Then the zip file created will expand into file.txt
+    // @deprecated no longer used. left for binary compatibility.
+    @Deprecated
     public static void zipSource(FilePath workspace, final String directory, final ZipOutputStream out, final String prefixToTrim) throws InvalidInputException, IOException, InterruptedException {
         if (!Paths.get(directory).startsWith(Paths.get(prefixToTrim))) {
             throw new InvalidInputException(zipSourceError + "prefixToTrim: " + prefixToTrim + ", directory: "+ directory);
@@ -141,6 +196,8 @@ public static void zipSource(FilePath workspace, final String directory, final Z
     //     The given path is /tmp/dir/folder/file.txt
     //     The given prefixToTrim can be /tmp/dir/ or /tmp/dir
     //     Then the returned path string will be folder/file.txt.
+    // @deprecated no longer used. left for binary compatibility.
+    @Deprecated
     public static String trimPrefix(final String path, final String prefixToTrim) {
         return Paths.get(prefixToTrim).relativize(Paths.get(path)).toString();
     }

src/main/resources/CodeBuilder/config.jelly

@@ -97,6 +97,14 @@
           <f:textbox />
         </f:entry>
 
+        <f:entry title="Inclusion pattern (optional)" field="workspaceIncludes" help="/plugin/aws-codebuild/help-workspaceIncludes.html">
+          <f:textbox />
+        </f:entry>
+
+        <f:entry title="Exclusion pattern (optional)" field="workspaceExcludes" help="/plugin/aws-codebuild/help-workspaceExcludes.html">
+          <f:textbox />
+        </f:entry>
+
     </f:radioBlock>
 
   </f:section>

src/main/webapp/help-workspaceExcludes.html

@@ -0,0 +1,16 @@
+<!--
+  ~     Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+  ~
+  ~     Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License.
+  ~     A copy of the License is located at
+  ~
+  ~         http://aws.amazon.com/apache2.0/
+  ~
+  ~     or in the "license" file accompanying this file.
+  ~     This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~     See the License for the specific language governing permissions and limitations under the License.
+  -->
+<div>
+    <a href="https://ant.apache.org/manual/Types/fileset.html">Ant fileset patterns</a> to specify files not to be zipped even if files match "Inclusion pattern"(workspaceIncludes).
+    Multiple patterns can be specified separating with commas(,).
+</div>

src/main/webapp/help-workspaceIncludes.html

@@ -0,0 +1,27 @@
+<!--
+  ~     Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+  ~
+  ~     Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License.
+  ~     A copy of the License is located at
+  ~
+  ~         http://aws.amazon.com/apache2.0/
+  ~
+  ~     or in the "license" file accompanying this file.
+  ~     This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~     See the License for the specific language governing permissions and limitations under the License.
+  -->
+<div>
+    <p>
+    <a href="https://ant.apache.org/manual/Types/fileset.html">Ant fileset patterns</a> to specify files to be zipped.
+    Multiple patterns can be specified separating with commas(,).
+    Leaving blank results all files will be zipped.
+    </p>
+    <p>
+    Some examples:
+    <ul>
+        <li><code>**/*.java</code>: All files with '.java' extension. This includes files in subdirectories.</li>
+        <li><code>*.sh</code>: Files with '.sh' extension in the root directory. This doesn't include files in subdirectories.</li>
+        <li><code>subdir/</code>: All files in the 'subdir' directory.</li>
+    </ul>
+    </p>
+</div>

src/test/java/CodeBuilderPerformTest.java

@@ -52,6 +52,8 @@ public void testConfigAllNull() throws Exception {
                                            null, null, null, null, null, null, null, null, null, null, null, null,
                                            null, null, null, null, null, null, "", null, null, null,
                                            null, null, null, null, null, null, null);
+        test.workspaceIncludes = null;
+        test.workspaceExcludes = null;
 
         ArgumentCaptor<Result> savedResult = ArgumentCaptor.forClass(Result.class);
         test.perform(build, ws, launcher, listener, mockStepContext);
@@ -69,6 +71,8 @@ public void testConfigAllBlank() throws Exception {
         CodeBuilder test = new CodeBuilder("", "", "", "", "", null, "", "", "", "", "", "", "", "", "", "", "", "",
                                            "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
                                            "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
+        test.workspaceIncludes = "";
+        test.workspaceExcludes = "";
 
         ArgumentCaptor<Result> savedResult = ArgumentCaptor.forClass(Result.class);
         test.perform(build, ws, launcher, listener, mockStepContext);

src/test/java/CodeBuilderTest.java

@@ -47,6 +47,9 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+/**
+ * This is a base class for other test classes. Provides no tests.
+ */
 public class CodeBuilderTest {
 
     AWSClientFactory mockFactory = mock(AWSClientFactory.class);