"buck project" to copy generated Go code to vendor

build.xml

@@ -614,7 +614,7 @@
     </build-buck-module-jar>
   </target>
 
-  <target name="build-module-intellij" depends="compile">
+  <target name="build-module-intellij" depends="compile, build-module-go">
     <build-buck-module-jar-with-resources module-name="intellij">
       <module-javac-params>
         <include name="com/facebook/buck/features/project/intellij/**/*.java" />

src/com/facebook/buck/features/project/intellij/IjProjectCommandHelper.java

@@ -21,6 +21,7 @@
 import com.facebook.buck.config.ProjectTestsMode;
 import com.facebook.buck.config.resources.ResourcesConfig;
 import com.facebook.buck.core.cell.Cell;
+import com.facebook.buck.core.description.arg.HasSrcs;
 import com.facebook.buck.core.exceptions.HumanReadableException;
 import com.facebook.buck.core.model.BuildTarget;
 import com.facebook.buck.core.model.actiongraph.ActionGraphAndBuilder;
@@ -30,10 +31,14 @@
 import com.facebook.buck.core.model.targetgraph.TargetNode;
 import com.facebook.buck.core.model.targetgraph.impl.TargetGraphAndTargets;
 import com.facebook.buck.core.rules.ActionGraphBuilder;
+import com.facebook.buck.core.rules.SourcePathRuleFinder;
+import com.facebook.buck.core.sourcepath.BuildTargetSourcePath;
+import com.facebook.buck.core.sourcepath.resolver.impl.DefaultSourcePathResolver;
 import com.facebook.buck.event.BuckEventBus;
 import com.facebook.buck.event.ConsoleEvent;
 import com.facebook.buck.features.project.intellij.aggregation.AggregationMode;
 import com.facebook.buck.features.project.intellij.model.IjProjectConfig;
+import com.facebook.buck.io.filesystem.ProjectFilesystem;
 import com.facebook.buck.jvm.core.JavaPackageFinder;
 import com.facebook.buck.jvm.java.JavaBuckConfig;
 import com.facebook.buck.jvm.java.JavaFileParser;
@@ -64,10 +69,18 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ForkJoinPool;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.pf4j.PluginManager;
 
 public class IjProjectCommandHelper {
 
@@ -87,6 +100,7 @@ public class IjProjectCommandHelper {
   private final boolean updateOnly;
   private final BuckBuildRunner buckBuildRunner;
   private final Function<Iterable<String>, ImmutableList<TargetNodeSpec>> argsParser;
+  private final PluginManager pluginManager;
 
   private final ProjectGeneratorParameters projectGeneratorParameters;
 
@@ -105,7 +119,8 @@ public IjProjectCommandHelper(
       boolean updateOnly,
       BuckBuildRunner buckBuildRunner,
       Function<Iterable<String>, ImmutableList<TargetNodeSpec>> argsParser,
-      ProjectGeneratorParameters projectGeneratorParameters) {
+      ProjectGeneratorParameters projectGeneratorParameters,
+      PluginManager pluginManager) {
     this.buckEventBus = buckEventBus;
     this.console = projectGeneratorParameters.getConsole();
     this.executor = executor;
@@ -124,6 +139,7 @@ public IjProjectCommandHelper(
     this.argsParser = argsParser;
 
     this.projectGeneratorParameters = projectGeneratorParameters;
+    this.pluginManager = pluginManager;
   }
 
   public ExitCode parseTargetsAndRunProjectGenerator(List<String> arguments)
@@ -195,7 +211,10 @@ public ExitCode parseTargetsAndRunProjectGenerator(List<String> arguments)
 
       return ExitCode.SUCCESS;
     }
-
+    ExitCode result = initGoWorkspace(targetGraphAndTargets);
+    if (result != ExitCode.SUCCESS) {
+      return result;
+    }
     return runIntellijProjectGenerator(targetGraphAndTargets);
   }
 
@@ -238,6 +257,131 @@ private TargetGraph getProjectGraphForIde(
         buckEventBus, cell, enableParserProfiling, executor, passedInTargets);
   }
 
+  /**
+   * Instead of specifying the location of libraries in project files, Go requires libraries to be
+   * in locations consistent with their package name, either relative to GOPATH environment variable
+   * or to the "vendor" folder of a project. This method identifies code generation targets, builds
+   * them, and copy the generated code from buck-out to vendor, so that they are accessible by IDEs.
+   */
+  private ExitCode initGoWorkspace(TargetGraphAndTargets targetGraphAndTargets)
+      throws IOException, InterruptedException {
+    Map<BuildTargetSourcePath, Path> generatedPackages =
+        findCodeGenerationTargets(targetGraphAndTargets);
+    if (generatedPackages.size() == 0) {
+      return ExitCode.SUCCESS;
+    }
+    // Run code generation targets
+    ExitCode exitCode =
+        runBuild(
+            generatedPackages
+                .keySet()
+                .stream()
+                .map(BuildTargetSourcePath::getTarget)
+                .collect(ImmutableSet.toImmutableSet()));
+    if (exitCode != ExitCode.SUCCESS) {
+      return exitCode;
+    }
+
+    copyGeneratedGoCode(targetGraphAndTargets, generatedPackages);
+    return ExitCode.SUCCESS;
+  }
+
+  /**
+   * Assuming GOPATH is set to a directory higher or equal to buck root, copy generated code to the
+   * package path relative to the highest level vendor directory. Not handling the case of GOPATH
+   * lower than buck root for now, as it requires walking the directory structure, which can be
+   * expensive and unreliable (e.g., what if there are multiple src directory?).
+   */
+  private void copyGeneratedGoCode(
+      TargetGraphAndTargets targetGraphAndTargets,
+      Map<BuildTargetSourcePath, Path> generatedPackages)
+      throws IOException {
+    Path vendorPath;
+    ProjectFilesystem fs = cell.getFilesystem();
+
+    Optional<String> vendorConfig = buckConfig.getValue("go", "vendor_path");
+    if (vendorConfig.isPresent()) {
+      vendorPath = Paths.get(vendorConfig.get());
+    }else if (fs.exists(Paths.get("src"))) {
+      vendorPath = Paths.get("src", "vendor");
+    } else {
+      vendorPath = Paths.get("vendor");
+    }
+    ActionGraphAndBuilder result =
+        Preconditions.checkNotNull(getActionGraph(targetGraphAndTargets.getTargetGraph()));
+    DefaultSourcePathResolver sourcePathResolver =
+        DefaultSourcePathResolver.from(new SourcePathRuleFinder(result.getActionGraphBuilder()));
+    for (BuildTargetSourcePath sourcePath : generatedPackages.keySet()) {
+      Path desiredPath = vendorPath.resolve(generatedPackages.get(sourcePath));
+      fs.mkdirs(desiredPath);
+      for (Path f : fs.getDirectoryContents(desiredPath)) {
+        if (fs.isFile(f)) {
+          fs.deleteFileAtPath(f);
+        }
+      }
+      Path generatedSrc = sourcePathResolver.getAbsolutePath(sourcePath);
+      if (fs.isDirectory(generatedSrc)) {
+        fs.copyFolder(generatedSrc, desiredPath);
+      } else {
+        fs.copyFile(generatedSrc, desiredPath.resolve(generatedSrc.getFileName()));
+      }
+    }
+  }
+
+  /**
+   * Find code generation targets by inspecting go_library targets in the target graph with "srcs"
+   * pointing to other Buck targets rather than Go files. Those Buck targets in "srcs" are assumed
+   * to be code generation targets. Their output is intended to be used as some package name, either
+   * specified by package_name argument of go_library, or guessed from the base path of the the
+   * go_library
+   */
+  @SuppressWarnings("unchecked")
+  private Map<BuildTargetSourcePath, Path> findCodeGenerationTargets(
+      TargetGraphAndTargets targetGraphAndTargets) {
+    ClassLoader classLoader = pluginManager.getPluginClassLoader("com.facebook.buck.features.go");
+    Map<BuildTargetSourcePath, Path> generatedPackages = new HashMap<>();
+    try {
+      Class<?> goLibraryDescriptionArgClass;
+      if (classLoader != null) {
+        goLibraryDescriptionArgClass =
+            Class.forName(
+                "com.facebook.buck.features.go.GoLibraryDescriptionArg", true, classLoader);
+      } else {
+        // Try to see if it is already in the default class loader
+        goLibraryDescriptionArgClass =
+            Class.forName("com.facebook.buck.features.go.GoLibraryDescriptionArg");
+      }
+      for (TargetNode<?, ?> targetNode : targetGraphAndTargets.getTargetGraph().getNodes()) {
+        Object constructorArg = targetNode.getConstructorArg();
+        if (constructorArg.getClass() == goLibraryDescriptionArgClass) {
+          Method getPackageNameMethod = goLibraryDescriptionArgClass.getMethod("getPackageName");
+          Optional<String> packageNameArg =
+              (Optional<String>) getPackageNameMethod.invoke(constructorArg);
+          Path pkgName =
+              packageNameArg
+                  .map(Paths::get)
+                  .orElse(
+                      Paths.get(buckConfig.getValue("go", "prefix").orElse(""))
+                          .resolve(targetNode.getBuildTarget().getBasePath()));
+          generatedPackages.putAll(
+              ((HasSrcs) constructorArg)
+                  .getSrcs()
+                  .stream()
+                  .filter(srcPath -> srcPath instanceof BuildTargetSourcePath)
+                  .collect(Collectors.toMap(src -> (BuildTargetSourcePath) src, src -> pkgName)));
+        }
+      }
+    } catch (ClassNotFoundException
+        | NoSuchMethodException
+        | IllegalAccessException
+        | InvocationTargetException e) {
+      // If any of these exception occur, the Go module is not available, and thus none of the Go
+      // rules is available either
+      buckEventBus.post(ConsoleEvent.warning(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
+    }
+    return generatedPackages;
+  }
+
   /** Run intellij specific project generation actions. */
   private ExitCode runIntellijProjectGenerator(TargetGraphAndTargets targetGraphAndTargets)
       throws IOException, InterruptedException {

src/com/facebook/buck/features/project/intellij/IjProjectSubCommand.java

@@ -171,7 +171,8 @@ public ExitCode run(
             updateOnly,
             (buildTargets, disableCaching) -> runBuild(params, buildTargets, disableCaching),
             projectGeneratorParameters.getArgsParser(),
-            projectGeneratorParameters);
+            projectGeneratorParameters,
+            params.getPluginManager());
     return projectCommandHelper.parseTargetsAndRunProjectGenerator(projectCommandArguments);
   }
 

test/com/facebook/buck/features/project/intellij/BUCK

@@ -137,6 +137,7 @@ java_test(
     srcs = TEST_SRCS,
     module_deps = [
         "//src/com/facebook/buck/features/zip/rules:rules",
+        "//src/com/facebook/buck/features/go:go",
     ],
     resources = glob(["testdata/**"]),
     deps = [

test/com/facebook/buck/features/project/intellij/ProjectIntegrationTest.java

@@ -496,6 +496,37 @@ public void testCrossCellIntelliJProject() throws Exception {
     assertThat(doc2, Matchers.hasXPath(urlXpath, Matchers.startsWith("jar://$PROJECT_DIR$/..")));
   }
 
+  @Test
+  public void testBuckProjectCopyGeneratedGoCode() throws IOException {
+    ProjectWorkspace workspace =
+        TestDataHelper.createProjectWorkspaceForScenario(
+            this, "go_project_with_generated_code", temporaryFolder.newFolder());
+    workspace.setUp();
+    ProcessResult result = workspace.runBuckCommand("project");
+    result.assertSuccess("buck project should exit cleanly");
+
+    assertTrue(workspace.resolve("vendor/a/b1.go").toFile().exists());
+    assertTrue(workspace.resolve("vendor/a/b2.go").toFile().exists());
+  }
+
+  @Test
+  public void testBuckProjectPurgeExistingVendor() throws IOException {
+    ProjectWorkspace workspace =
+        TestDataHelper.createProjectWorkspaceForScenario(
+            this, "go_project_with_existing_vendor", temporaryFolder.newFolder());
+    workspace.setUp();
+    assertTrue(workspace.resolve("vendor/a/b0.go").toFile().exists());
+    assertTrue(workspace.resolve("vendor/a/b/b.go").toFile().exists());
+    ProcessResult result = workspace.runBuckCommand("project");
+    result.assertSuccess("buck project should exit cleanly");
+
+    assertFalse(workspace.resolve("vendor/a/b0.go").toFile().exists());
+    assertTrue(workspace.resolve("vendor/a/b1.go").toFile().exists());
+    assertTrue(workspace.resolve("vendor/a/b2.go").toFile().exists());
+    assertTrue(workspace.resolve("vendor/a/b/b.go").toFile().exists());
+  }
+
+
   private ProcessResult runBuckProjectAndVerify(String folderWithTestData, String... commandArgs)
       throws InterruptedException, IOException {
     AssumeAndroidPlatform.assumeSdkIsAvailable();

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_existing_vendor/.buckconfig

@@ -0,0 +1,2 @@
+[project]
+  ide = intellij

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_existing_vendor/BUCK.fixture

@@ -0,0 +1,29 @@
+go_binary(
+    name = "main",
+    srcs = [
+        ":main.go",
+    ],
+    deps = [
+        ":a",
+    ],
+)
+
+genrule(
+    name = "main.go",
+    out = "main.go",
+    cmd = "echo 'package main\nimport \"a\"\nfunc main() { a.A() }\n' > $OUT",
+)
+
+go_library(
+    name = "a",
+    package_name = "a",
+    srcs = [
+        ":b",
+    ],
+)
+
+genrule(
+    name = "b",
+    out = ".",
+    cmd = "echo 'package a\nfunc A() {}\n' > $OUT/b1.go; echo 'package a\nfunc B() {}\n' > $OUT/b2.go;",
+)

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_existing_vendor/vendor/a/b/b.go

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018-present Facebook, Inc. and Uber Technologies, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *  not use this file except in compliance with the License. You may obtain
+ *  a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  License for the specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package b
+
+func B() {
+
+}

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_existing_vendor/vendor/a/b0.go

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018-present Facebook, Inc. and Uber Technologies, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *  not use this file except in compliance with the License. You may obtain
+ *  a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  License for the specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package a
+
+func A() {
+
+}

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_generated_code/.buckconfig

@@ -0,0 +1,2 @@
+[project]
+  ide = intellij

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_generated_code/BUCK.fixture

@@ -0,0 +1,29 @@
+go_binary(
+    name = "main",
+    srcs = [
+        ":main.go",
+    ],
+    deps = [
+        ":a",
+    ],
+)
+
+genrule(
+    name = "main.go",
+    out = "main.go",
+    cmd = "echo 'package main\nimport \"a\"\nfunc main() { a.A() }\n' > $OUT",
+)
+
+go_library(
+    name = "a",
+    package_name = "a",
+    srcs = [
+        ":b",
+    ],
+)
+
+genrule(
+    name = "b",
+    out = ".",
+    cmd = "echo 'package a\nfunc A() {}\n' > $OUT/b1.go; echo 'package a\nfunc B() {}\n' > $OUT/b2.go;",
+)

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_generated_code/a/b1.go

@@ -0,0 +1,3 @@
+package a
+func A() {}
+

test/com/facebook/buck/features/project/intellij/testdata/go_project_with_generated_code/a/b2.go

@@ -0,0 +1,3 @@
+package a
+func B() {}
+