feat(release): Publish Halyard JARS independentlt (spinnaker#1853)

dev/annotate_source.py

@@ -296,7 +296,8 @@ def __determine_current_version(self):
     sorted_filtered_tags = sorted(self.__filtered_tags,
                                   key=lambda ht: ht.version, reverse=True)
     if len(sorted_filtered_tags) == 0:
-      raise GitTagMissingException("No version tags of the form 'version-X.Y.Z'.")
+      raise GitTagMissingException("No version tags of the form 'version-X.Y.Z' in {path}."
+                                   .format(path=self.path))
 
     self.__current_version = sorted_filtered_tags[0]
 

dev/build_prevalidation.py

@@ -92,6 +92,8 @@ def main():
 
   bom_generator = BomGenerator(options)
   bom_generator.determine_and_tag_versions()
+  # bom_generator.determine_and_tag_halyard()
+  bom_generator.write_component_version_files()
   if options.container_builder == 'gcb':
     bom_generator.write_container_builder_gcr_config()
   elif options.container_builder == 'gcb-trigger':

dev/build_release.py

@@ -351,6 +351,18 @@ def start_container_build(self, name):
       raise NotImplemented(
           'container_builder="{0}"'.format(self.__options.container_builder))
 
+  def start_jar_build(self, name):
+    """Start a subprocess to build a JAR for the given subcomponent
+
+    Relies on gradle's installDist task to produce build artifacts to be
+    packaged into an installable zip file.
+
+    Args:
+      name [string]: Name of the subsystem repository.
+    """
+    gradle_root = self.determine_gradle_root(name)
+    self.__jar_build(name, gradle_root)
+
   @classmethod
   def __gcb_build(cls, name, gradle_root, gcb_service_account, gcb_project):
     # Local .gradle dir stomps on GCB's .gradle directory when the gradle
@@ -472,6 +484,18 @@ def __docker_build(cls, name, gradle_root):
       os.remove(logfile)
     run_shell_and_log(cmds, logfile, cwd=gradle_root)
 
+  @classmethod
+  def __jar_build(cls, name, gradle_root):
+    version = run_quick('cat {name}-component-version.yml'.format(name=name), 
+                        echo=False).stdout.strip()
+    cmds = [
+      './release/all.sh {version} nightly'.format(version=version),
+    ]
+    logfile = '{name}-jar-build.log'.format(name=name)
+    if os.path.exists(logfile):
+      os.remove(logfile)
+    run_shell_and_log(cmds, logfile, cwd=gradle_root)
+
   def publish_to_bintray(self, source, package, version, path, debian_tags=''):
     bintray_key = os.environ['BINTRAY_KEY']
     bintray_user = os.environ['BINTRAY_USER']
@@ -694,6 +718,13 @@ def start_copy_redhat_target(self, name):
 
       return pids
 
+  def __do_jar_build(self, subsys):
+    if self.__options.do_jar_build:
+      try:
+        self.start_jar_build(subsys)
+      except Exception as ex:
+        self.__build_failures.append(BuildFailure(subsys, ex))
+
   def __do_build(self, subsys):
     if self.__options.platform == 'debian':
       try:
@@ -744,6 +775,19 @@ def build_container_images(self):
 
     self.__check_build_failures(subsystems)
 
+  def build_jars(self):
+    """Build the Spinnaker packages as jars
+    """
+    subsystems = ['halyard']
+
+    if self.__options.do_jar_build:
+      weighted_processes = self.__options.cpu_ratio * multiprocessing.cpu_count()
+      pool = multiprocessing.pool.ThreadPool(
+        processes=int(max(1, weighted_processes)))
+      pool.map(self.__do_jar_build, subsystems)
+
+    self.__check_build_failures(subsystems)
+
   def build_packages(self):
       """Build all the Spinnaker packages."""
       all_subsystems = []
@@ -857,7 +901,9 @@ def init_argument_parser(cls, parser):
       parser.add_argument(
           '--run_unit_tests', type=bool, default=False,
           help='Run unit tests during build for all components other than Deck.')
-
+      parser.add_argument(
+          '--do_jar_build', type=bool, default=True,
+          help='Build & publish jars independently to GCS.')
 
   def __verify_bintray(self):
     if not os.environ.get('BINTRAY_KEY', None):
@@ -882,8 +928,14 @@ def do_build(cls, options, build_number=None, container_builder=None, sync_branc
     if options.pull_origin:
         builder.refresher.pull_all_from_origin()
 
+    print "Starting JAR build..."
+    builder.build_jars()
+
+    print "Starting package build..."
     builder.build_packages()
+
     if container_builder:
+      print "Starting container build..."
       builder.build_container_images()
 
     if options.build and options.bintray_repo:

dev/generate_bom.py

@@ -192,9 +192,20 @@ def __generate_gcb_config(self, comp):
       }
     return config
 
+  def write_component_version_files(self):
+    """Write a file containing the full version for each microservice.
+    """
+    for comp in dict(self.__component_versions.items() + self.__halyard_version.items()):
+      if comp == 'spinnaker':
+        pass
+      gradle_version = self.__version_from_tag(comp)
+      config_file = '{0}-component-version.yml'.format(comp)
+      with open(config_file, 'w') as cfg:
+        cfg.write(gradle_version)
 
   def write_gcb_trigger_version_files(self):
-    """Write a file containing the full tag for each microservice for Docker.
+    """Write a file containing the full tag for each microservice for GCB 
+    triggers.
     """
     for comp in self.__component_versions:
       if comp == 'spinnaker':
@@ -206,7 +217,6 @@ def write_gcb_trigger_version_files(self):
       with open(config_file, 'w') as cfg:
         cfg.write(git_tag)
 
-
   def write_docker_version_files(self):
     """Write a file containing the full tag for each microservice for Docker.
     """
@@ -427,6 +437,7 @@ def main(cls):
     bom_generator = cls(options)
     bom_generator.determine_and_tag_versions()
     bom_generator.write_rpm_version_files()
+    bom_generator.write_component_version_files()
     if options.container_builder == 'gcb':
       bom_generator.write_container_builder_gcr_config()
     elif options.container_builder == 'gcb-trigger':