Merge

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy

@@ -0,0 +1,80 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.gradle.plugins.ci
+
+import groovy.transform.builder.Builder
+
+import java.util.regex.Pattern
+import org.gradle.api.Project
+
+/** Determines a set of subprojects that own the 'changedPaths'. */
+class AffectedProjectFinder {
+    Project project;
+    Set<String> changedPaths;
+
+    @Builder
+    AffectedProjectFinder(Project project,
+                          Set<String> changedPaths,
+                          List<Pattern> ignorePaths) {
+        this.project = project
+        this.changedPaths = changedPaths.findAll {
+            for(def path : ignorePaths) {
+                if(it ==~ path) {
+                    return false
+                }
+            }
+            return true
+        }
+    }
+
+    Set<Project> find() {
+        Set<String> paths = changedPaths.collect()
+        def projects = changedSubProjects(project, paths)
+
+        if(!containsRootProject(projects)) {
+            return projects
+        }
+        return project.subprojects
+    }
+
+    /**
+     * Performs a post-order project tree traversal and returns a set of projects that own the
+     * 'changedPaths'.
+     */
+    private static Set<Project> changedSubProjects(Project project, Set<String> changedPaths) {
+        // project.subprojects include all descendents of a given project, we only want immediate
+        // children.
+        Set<Project> immediateChildProjects = project.subprojects.findAll { it.parent == project }
+
+        Set<Project> projects = immediateChildProjects.collectMany {
+            changedSubProjects(it, changedPaths)
+        }
+        def relativePath = project.rootDir.toURI().relativize(project.projectDir.toURI()).toString()
+
+        Iterator itr = changedPaths.iterator()
+        while (itr.hasNext()) {
+            def file = itr.next()
+            if (file.startsWith(relativePath)) {
+                itr.remove()
+                projects.add(project)
+            }
+        }
+        return projects
+    }
+
+    private static boolean containsRootProject(Set<Project> projects) {
+        return projects.any { it.rootProject == it };
+    }
+}

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationExtension.groovy

@@ -0,0 +1,23 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.gradle.plugins.ci
+
+import java.util.regex.Pattern
+
+/** Contains plugin configuration properties. */
+class ContinuousIntegrationExtension {
+    /** List of paths that the plugin should ignore when querying the Git commit. */
+    List<Pattern> ignorePaths = []
+}

buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy

@@ -0,0 +1,141 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.gradle.plugins.ci
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+
+
+/**
+ * Provides 'checkChanged' and 'connectedCheckChanged' tasks to the root project.
+ *
+ * <p>The task definition is dynamic and depends on the latest git changes in the project. Namely
+ * it gets a list of changed files from the latest Git pull/merge and determines which subprojects
+ * the files belong to. Then, for each affected project, it declares a dependency on the
+ * 'checkDependents' or 'connectedCheckChanged' task respectively in that project.
+ *
+ * <p>Note: If the commits contain a file that does not belong to any subproject, *all* subprojects
+ * will be built.
+ */
+class ContinuousIntegrationPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+
+        def extension = project.extensions.create(
+                "firebaseContinuousIntegration",
+                ContinuousIntegrationExtension)
+
+        project.configure(project.subprojects) {
+            def checkDependents = it.task('checkDependents') {}
+            def connectedCheckDependents = it.task('connectedCheckDependents')
+
+            configurations.all {
+                if (it.name == 'debugUnitTestRuntimeClasspath') {
+                    checkDependents.dependsOn(configurations
+                            .debugUnitTestRuntimeClasspath.getTaskDependencyFromProjectDependency(
+                            false, "checkDependents"))
+                    checkDependents.dependsOn 'check'
+                }
+
+                if (it.name == 'debugAndroidTestRuntimeClasspath') {
+                    connectedCheckDependents.dependsOn(configurations
+                            .debugAndroidTestRuntimeClasspath.getTaskDependencyFromProjectDependency(
+                            false, "connectedCheckDependents"))
+                    connectedCheckDependents.dependsOn 'connectedCheck'
+                }
+
+                if (it.name == 'annotationProcessor') {
+                    connectedCheckDependents.dependsOn(configurations
+                            .annotationProcessor.getTaskDependencyFromProjectDependency(
+                            false, "connectedCheckDependents"))
+                    checkDependents.dependsOn(configurations
+                            .annotationProcessor.getTaskDependencyFromProjectDependency(
+                            false, "checkDependents"))
+                }
+            }
+
+            afterEvaluate {
+                // non-android projects need to define the custom configurations due to the way
+                // getTaskDependencyFromProjectDependency works.
+                if (!isAndroidProject(it)) {
+                    configurations {
+                        debugUnitTestRuntimeClasspath
+                        debugAndroidTestRuntimeClasspath
+                        annotationProcessor
+                    }
+                    // noop task to avoid having to handle the edge-case of tasks not being
+                    // defined.
+                    tasks.maybeCreate('connectedCheck')
+                    tasks.maybeCreate('check')
+                }
+            }
+        }
+
+        def affectedProjects = AffectedProjectFinder.builder()
+                .project(project)
+                .changedPaths(changedPaths(project.rootDir))
+                .ignorePaths(extension.ignorePaths)
+                .build()
+                .find()
+
+        project.task('checkChanged') { task ->
+            task.group = 'verification'
+            task.description = 'Runs the check task in all changed projects.'
+            affectedProjects.each {
+                task.dependsOn("$it.path:checkDependents")
+            }
+        }
+        project.task('connectedCheckChanged') { task ->
+            task.group = 'verification'
+            task.description = 'Runs the connectedCheck task in all changed projects.'
+            affectedProjects.each {
+                task.dependsOn("$it.path:connectedCheckDependents")
+            }
+        }
+
+        project.task('ciTasksSanityCheck') {
+            doLast {
+                [':firebase-common', ':tools:errorprone'].each { projectName ->
+                    def task = project.project(projectName).tasks.findByName('checkDependents')
+                    def dependents = task.taskDependencies.getDependencies(task).collect { it.path}
+
+                    def expectedDependents = [
+                            'database',
+                            'firestore',
+                            'functions',
+                            'storage'].collect { ":firebase-$it:checkDependents"}
+                    assert expectedDependents.intersect(dependents) == expectedDependents :
+                            "$projectName:checkDependents does not depend on expected projects"
+                }
+            }
+        }
+    }
+
+    private static Set<String> changedPaths(File workDir) {
+        return 'git diff --name-only --submodule=diff HEAD@{0} HEAD@{1}'
+                .execute([], workDir)
+                .text
+                .readLines()
+    }
+
+    private static final ANDROID_PLUGINS = ["com.android.application", "com.android.library",
+                                           "com.android.test"]
+
+    private static boolean isAndroidProject(Project project) {
+        ANDROID_PLUGINS.find { plugin -> project.plugins.hasPlugin(plugin) }
+    }
+}

ci/fireci/fireci/commands.py

@@ -37,16 +37,26 @@ def gradle_command(task, gradle_opts):
     help=
     'App build variant to use while running the smoke Tests. One of release|debug'
 )
+@click.option(
+    '--test-apps-dir',
+    '-d',
+    multiple=True,
+    type=click.Path(exists=True, file_okay=False, resolve_path=True),
+    default=['test-apps'],
+    help=
+    'Directory that contains gradle build with apps to test against. Multiple values are allowed.'
+)
 @ci_command()
-def smoke_tests(app_build_variant):
+def smoke_tests(app_build_variant, test_apps_dir):
   """Builds all SDKs in release mode and then tests test-apps against them."""
   gradle.run('publishAllToBuildDir')
 
   cwd = os.getcwd()
-  gradle.run(
-      'connectedCheck',
-      '-PtestBuildType=%s' % (app_build_variant),
-      gradle_opts='-Dmaven.repo.local={}'.format(
-          os.path.join(cwd, 'build', 'm2repository')),
-      workdir=os.path.join(cwd, 'test-apps'),
-  )
+  for location in test_apps_dir:
+    gradle.run(
+        'connectedCheck',
+        '-PtestBuildType=%s' % (app_build_variant),
+        gradle_opts='-Dmaven.repo.local={}'.format(
+            os.path.join(cwd, 'build', 'm2repository')),
+        workdir=location,
+    )

ci/fireci/fireci/emulator.py

@@ -74,7 +74,7 @@ def __enter__(self):
         stdout=self._stdout,
         stderr=self._stderr)
     try:
-      self._wait_for_boot(datetime.timedelta(minutes=5))
+      self._wait_for_boot(datetime.timedelta(minutes=10))
     except:
       self._kill(self._process)
       self._close_files()

ci/fireci/fireciplugins/copyright.py

@@ -0,0 +1,107 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+import click
+import contextlib
+import os
+import re
+
+from fireci import ci_command
+
+
+@click.option(
+    '--ignore-path',
+    '-i',
+    default=(),
+    multiple=True,
+    type=str,
+    help='Unix path pattern to ignore when searching for matching files. '
+    'Multiple values allowed.',
+)
+@click.option(
+    '--include-extension',
+    '-e',
+    default=(),
+    multiple=True,
+    type=str,
+    help='File extensions to scan for copyright violation. '
+    'Multiple values allowed.',
+    required=True,
+)
+@click.option(
+    '--expected-regex',
+    '-r',
+    default='.*Copyright [0-9]{4} Google LLC',
+    type=str,
+    help='Regex expected to be present in the file.',
+)
+@click.argument(
+    'dir_to_scan',
+    type=click.Path(exists=True, file_okay=False),
+    default='.',
+    nargs=1,
+)
+@ci_command()
+def copyright_check(dir_to_scan, ignore_path, include_extension,
+                    expected_regex):
+  """Checks matching files' content for copyright information."""
+  expression = re.compile(expected_regex)
+  failed_files = []
+  with chdir(dir_to_scan):
+    for x in walk('.', ignore_path, include_extension):
+      with open(x) as f:
+        if not match_any(f, lambda line: expression.match(line)):
+          failed_files.append(x)
+
+  if failed_files:
+    raise click.ClickException(
+        "The following files do not have valid copyright information:\n{}"
+        .format('\n'.join(failed_files)))
+
+
+@contextlib.contextmanager
+def chdir(directory):
+  original_dir = os.getcwd()
+  os.chdir(directory)
+  try:
+    yield
+  finally:
+    os.chdir(original_dir)
+
+
+def match_any(iterable, predicate):
+  """Returns True if at least one item in the iterable matches the predicate."""
+  for x in iterable:
+    if predicate(x):
+      return True
+  return False
+
+
+def walk(dir_to_scan, ignore_paths, extensions_to_include):
+  """Recursively walk the provided directory and yield matching paths."""
+  for root, dirs, filenames in os.walk(dir_to_scan):
+    dirs[:] = (
+        x for x in dirs if not matches(os.path.join(root, x), ignore_paths))
+
+    for f in filenames:
+      filename = os.path.join(root, f)
+      if os.path.splitext(f)[1][1:] in extensions_to_include and not matches(
+          filename, ignore_paths):
+        yield os.path.normpath(filename)
+
+
+def matches(path, paths):
+  path = os.path.normpath(path)
+  return match_any(paths, lambda p: fnmatch.fnmatch(path, p))