Add metadata reporting for Git sources. (#132)

.coveragerc

@@ -0,0 +1,6 @@
+[run]
+branch = True
+
+[report]
+exclude_lines =
+    pragma: no cover

noxfile.py

@@ -32,9 +32,9 @@ def lint(session):
 
 @nox.session(python='3.6')
 def test(session):
-    session.install('pytest')
+    session.install('pytest', 'pytest-cov')
     session.run('pip', 'install', '-e', '.')
-    session.run('pytest', 'tests', *session.posargs)
+    session.run('pytest', '--cov-report', 'term-missing', '--cov', 'synthtool', 'tests', *session.posargs)
 
 
 @nox.session(python='3.6')

synthtool/__main__.py

@@ -22,8 +22,9 @@
 
 @click.command()
 @click.version_option(message="%(version)s")
-def main():
-    synth_file = os.path.join(os.getcwd(), "synth.py")
+@click.argument("synthfile", default="synth.py")
+def main(synthfile):
+    synth_file = os.path.abspath(synthfile)
 
     if os.path.lexists(synth_file):
         log.debug(f"Executing {synth_file}.")

synthtool/metadata.py

@@ -0,0 +1,59 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import atexit
+import functools
+import sys
+
+import google.protobuf.json_format
+
+from synthtool import log
+from synthtool.protos import metadata_pb2
+
+
+_metadata = metadata_pb2.Metadata()
+
+
+def reset() -> None:
+    """Clear all metadata so far."""
+    global _metadata
+    _metadata = metadata_pb2.Metadata()
+
+
+def get():
+    return _metadata
+
+
+def add_git_source(**kwargs) -> None:
+    """Adds a git source to the current metadata."""
+    _metadata.sources.add(git=metadata_pb2.GitSource(**kwargs))
+
+
+def write(outfile: str = "synth.metadata") -> None:
+    """Writes out the metadata to a file."""
+    jsonified = google.protobuf.json_format.MessageToJson(_metadata)
+
+    with open(outfile, "w") as fh:
+        fh.write(jsonified)
+
+    log.debug(f"Wrote metadata to {outfile}.")
+
+
+def register_exit_hook(**kwargs) -> None:
+    atexit.register(functools.partial(write, **kwargs))
+
+
+# Only register this hook if pytest is not active.
+if "pytest" not in sys.modules:  # pragma: no cover
+    register_exit_hook()

synthtool/sources/git.py

@@ -16,10 +16,12 @@
 import pathlib
 import re
 import shutil
-from typing import Dict
+import subprocess
+from typing import Dict, Tuple
 
 from synthtool import _tracked_paths
 from synthtool import cache
+from synthtool import metadata
 from synthtool import shell
 
 REPO_REGEX = (
@@ -70,6 +72,17 @@ def clone(
     # track all git repositories
     _tracked_paths.add(dest)
 
+    # add repo to metadata
+    sha, message = get_latest_commit(dest)
+    commit_metadata = extract_commit_message_metadata(message)
+
+    metadata.add_git_source(
+        name=dest.name,
+        remote=url,
+        sha=sha,
+        internal_ref=commit_metadata.get("PiperOrigin-RevId"),
+    )
+
     return dest
 
 
@@ -96,3 +109,37 @@ def parse_repo_url(url: str) -> Dict[str, str]:
         name = name[:-4]
 
     return {"owner": owner, "name": name}
+
+
+def get_latest_commit(repo: pathlib.Path = None) -> Tuple[str, str]:
+    """Return the sha and commit message of the latest commit."""
+    output = subprocess.check_output(
+        ["git", "log", "-1", "--pretty=%H%n%B"], cwd=repo
+    ).decode("utf-8")
+    commit, message = output.split("\n", 1)
+    return commit, message
+
+
+def extract_commit_message_metadata(message: str) -> Dict[str, str]:
+    """Extract extended metadata stored in the Git commit message.
+
+    For example, a commit that looks like this::
+
+        Do the thing!
+
+        Piper-Changelog: 1234567
+
+    Will return::
+
+        {"Piper-Changelog": "1234567"}
+
+    """
+    metadata = {}
+    for line in message.splitlines():
+        if ":" not in line:
+            continue
+
+        key, value = line.split(":", 1)
+        metadata[key] = value.strip()
+
+    return metadata

tests/test_git.py

@@ -0,0 +1,69 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest import mock
+
+import pytest
+
+from synthtool.sources import git
+
+
+def test_make_repo_clone_url(monkeypatch):
+    monkeypatch.setattr(git, "USE_SSH", True)
+    assert (
+        git.make_repo_clone_url("theacodes/nox") == "git@github.com:theacodes/nox.git"
+    )
+
+
+def test_make_repo_clone_url_https(monkeypatch):
+    monkeypatch.setattr(git, "USE_SSH", False)
+    assert (
+        git.make_repo_clone_url("theacodes/nox")
+        == "https://github.com/theacodes/nox.git"
+    )
+
+
+@pytest.mark.parametrize(
+    ("input, expected"),
+    [
+        ("git@github.com:theacodes/nox.git", {"owner": "theacodes", "name": "nox"}),
+        ("https://github.com/theacodes/nox.git", {"owner": "theacodes", "name": "nox"}),
+        ("theacodes/nox", {"owner": "theacodes", "name": "nox"}),
+        ("theacodes/nox.git", {"owner": "theacodes", "name": "nox"}),
+    ],
+)
+def test_parse_repo_url(input, expected):
+    assert git.parse_repo_url(input) == expected
+
+
+@mock.patch("subprocess.check_output", autospec=True)
+def test_get_latest_commit(check_call):
+    check_call.return_value = b"abc123\ncommit\nmessage."
+
+    sha, message = git.get_latest_commit()
+
+    assert sha == "abc123"
+    assert message == "commit\nmessage."
+
+
+def test_extract_commit_message_metadata():
+    message = """\
+Hello, world!
+
+One: Hello!
+Two: 1234
+"""
+    metadata = git.extract_commit_message_metadata(message)
+
+    assert metadata == {"One": "Hello!", "Two": "1234"}

tests/test_metadata.py

@@ -0,0 +1,47 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from synthtool import metadata
+
+
+def test_add_git_source():
+    metadata.reset()
+
+    metadata.add_git_source(sha="sha", name="name", remote="remote")
+
+    current = metadata.get()
+
+    assert current.sources[0].git.sha == "sha"
+    assert current.sources[0].git.name == "name"
+    assert current.sources[0].git.remote == "remote"
+
+
+def test_write(tmpdir):
+    metadata.reset()
+
+    metadata.add_git_source(sha="sha", name="name", remote="remote")
+
+    output_file = tmpdir / "synth.metadata"
+
+    metadata.write(str(output_file))
+
+    data = output_file.read()
+
+    # Ensure the file was written, that *some* metadata is in it, and that it
+    # is valid JSON.
+    assert data
+    assert "sha" in data
+    assert json.loads(data)