build

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#
# Copyright (c) 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import hashlib
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.request

# The import path of the project:
IMPORT_PATH = "github.com/jhernand/openshift-monitoring"

# The name and version of the project:
PROJECT_NAME = "openshift-monitoring"
PROJECT_VERSION = "0.0.0"

# The regular expression that will be used to replace variable names
# enclused with {{...}} with their values:
VARIABLE_RE = re.compile(
    r"""
    \{\{\s*
    (?P<name>\w+)
    s*\}\}
    """,
    re.VERBOSE
)

# The regular expression used to check if a file is a template, i.e, if its
# name ends with '.in':
TEMPLATE_RE = re.compile(
    r"""
    \.in$
    """,
    re.VERBOSE
)

# The regular expression to extract the digests, URLs and output file names
# from from the 'Downloads' files:
DOWNLOAD_RE = re.compile(
    r"""
    ^
    \s*
    (?P<digest>[^\s]+)
    \s+
    (?P<url>[^\s]+)
    \s+
    (?P<output>[^\s]+)
    \s*
    $
    """,
    re.VERBOSE
)

# The values extracted from the command line:
argv = None


def say(what):
    """
    Writes a message to the standard output, and then flushes it, so that
    the output doesn't appear out of order.
    """
    sys.stdout.write(what)
    sys.stdout.write("\n")
    sys.stdout.flush()


def cache(function):
    """
    A decorator that creates a cache for the results of a function, so that
    when the function is called the second time the result will be returned
    from the cache without actually executing it.
    """
    cache = dict()

    def helper(*key):
        try:
            value = cache[key]
        except KeyError:
            value = function(*key)
            cache[key] = value
        return value
    return helper


def find_paths(base, include=None, exclude=None):
    """
    Recursively finds the paths inside the 'base' directory whose complete
    names match the 'include' regular expression and don't match the 'exclude'
    regular expression. By default all paths are included and no path is
    excluded.
    """
    include_re = re.compile(include) if include else None
    exclude_re = re.compile(exclude) if exclude else None
    paths = []
    for root, _, names in os.walk(base):
        for name in names:
            path = os.path.join(root, name)
            path = os.path.abspath(path)
            should_include = include_re and include_re.search(path)
            should_exclude = exclude_re and exclude_re.search(path)
            if should_include and not should_exclude:
                paths.append(path)
    return paths


def go_tool(*args):
    """
    Executes a command with the 'GOPATH' environment variable pointing to the
    project specific path, and with the symbolic link as the working directory.
    The argument should be a list containing the complete command line to
    execute.
    """
    # Make sure that the required directories exist:
    go_path = ensure_go_path()
    project_dir = ensure_project_dir()
    project_link = ensure_project_link()

    # Modify the environment so that the Go tool will find the project files
    # using the `GOPATH` environment variable. Note that setting the `PWD`
    # environment is necessary, because the `cwd` is always resolved to a
    # real path by the operating system.
    env = dict(os.environ)
    env["GOPATH"] = go_path
    env["PWD"] = project_link

    # Run the Go tool and wait till it finishes:
    say("Running command '%s'" % " ".join(args))
    process = subprocess.Popen(
        args=args,
        env=env,
        cwd=project_dir
    )
    result = process.wait()
    if result != 0:
        raise Exception("Command '%s' failed with exit code %d" % (
            " ".join(args), result
        ))


@cache
def ensure_project_dir():
    say("Calculating project directory")
    return os.path.dirname(os.path.realpath(__file__))


@cache
def ensure_go_path():
    """
    Creates and returns the `.gopath` directory that will be used as the
    'GOPATH' for the project.
    """
    project_dir = ensure_project_dir()
    go_path = os.path.join(project_dir, '.gopath')
    if not os.path.exists(go_path):
        say('Creating Go path `{path}`'.format(path=go_path))
        os.mkdir(go_path)
    return go_path


@cache
def ensure_go_bin():
    """
    Creates and returns the Go 'bin' directory that will be used for the
    project.
    """
    go_path = ensure_go_path()
    go_bin = os.path.join(go_path, "bin")
    if not os.path.exists(go_bin):
        os.mkdir(go_bin)
    return go_bin


@cache
def ensure_go_pkg():
    """
    Creates and returns the Go 'pkg' directory that will be used for the
    project.
    """
    go_path = ensure_go_path()
    go_pkg = os.path.join(go_path, "pkg")
    if not os.path.exists(go_pkg):
        os.mkdir(go_pkg)
    return go_pkg


@cache
def ensure_go_src():
    """
    Creates and returns the Go 'src' directory that will be used for the
    project.
    """
    go_path = ensure_go_path()
    go_src = os.path.join(go_path, "src")
    if not os.path.exists(go_src):
        os.mkdir(go_src)
    return go_src


@cache
def ensure_project_link():
    """
    Creates the symbolik link that will be used to make the project appear
    in the 'GOPATH' expected by go tools. Returns the full path of the link.
    """
    project_dir = ensure_project_dir()
    go_src = ensure_go_src()
    project_link = os.path.join(go_src, IMPORT_PATH)
    link_dir = os.path.dirname(project_link)
    if not os.path.exists(link_dir):
        os.makedirs(link_dir)
    if not os.path.exists(project_link):
        os.symlink(project_dir, project_link)
    return project_link


@cache
def ensure_vendor_dir():
    """
    Creates and populates the 'vendor' directory if it doesn't exist yet.
    Returns the full path of the directory.
    """
    project_link = ensure_project_link()
    vendor_dir = os.path.join(project_link, "vendor")
    if not os.path.exists(vendor_dir):
        go_tool("dep", "ensure", "--vendor-only", "-v")
    return vendor_dir


@cache
def ensure_gen(name):
    """
    Downloads and install the Kubernetes code generator with the given name.
    For example, if the name is 'client-gen', it will check if the 'client-gen'
    binary is already available in the Go 'bin' directory, and if it isn't it
    will download and install it. Returns the full path of the binary.
    """
    go_bin = ensure_go_bin()
    gen_bin = os.path.join(go_bin, name)
    gen_pkg = "k8s.io/code-generator/cmd/" + name

    if not os.path.exists(gen_bin):
        go_tool("go", "get", gen_pkg)
    return gen_bin


def run_gen(name, *args):
    """
    Runs the Kubernetes code generator with the given name.
    """
    # The code generators need to load some Kubernetes source files, so we
    # need to make sure that the dependencies have already been downloaded:
    ensure_vendor_dir()

    # Add the option to specify the license header file:
    args += (
        "--go-header-file=boilerplate.go.txt",
        "--logtostderr",
    )

    # Add debug options:
    if argv.debug:
        args += ("--v=8",)

    # Run the code generator:
    gen_bin = ensure_gen(name)
    go_tool(gen_bin, *args)


@cache
def ensure_generated_code():
    """
    Generates the clients, listers and informers from the API types used by the
    project.
    """
    # Calculate a hash of the contents of the non generated files in the
    # 'pkg/apis' directory:
    project_dir = ensure_project_dir()
    apis_dir = os.path.join(project_dir, "pkg", "apis")
    apis_paths = find_paths(
        base=apis_dir,
        include=r'\.go$',
        exclude=r'/zz_generated\..*\.go$',
    )
    apis_paths.sort()
    calculated_hash = hashlib.sha256()
    for path in apis_paths:
        with open(path, "rb") as fd:
            content = fd.read()
            calculated_hash.update(content)
    calculated_hash = calculated_hash.hexdigest()

    # Read the stored hash, if it exists:
    stored_hash = None
    hash_path = os.path.join(project_dir, ".hash")
    if os.path.exists(hash_path):
        with open(hash_path) as fd:
            stored_hash = fd.read()

    # Do nothing if the calculated and stored hash are identical, as that means
    # that there were no changes in the sources since the last time that we run
    # the code generation tools.
    if calculated_hash == stored_hash:
        return

    # Store the calculated hash:
    with open(hash_path, 'w') as fd:
        fd.write(calculated_hash)

    # Calculate the names of the input and output packages:
    input_pkg = "%s/pkg/apis/monitoring/v1alpha1" % IMPORT_PATH
    output_pkg = "%s/pkg/client" % IMPORT_PATH

    # Generate the deep copy functions:
    run_gen(
        "deepcopy-gen",
        "--input-dirs=%s" % input_pkg,
        "--bounding-dirs=%s" % input_pkg,
        "--output-file-base=zz_generated.deepcopy",
    )

    # Generate the client set:
    run_gen(
        "client-gen",
        "--clientset-name=openshift",
        "--input-base=",
        "--input=%s" % input_pkg,
        "--output-package=%s" % output_pkg,
    )

    # Generate the listers:
    run_gen(
        "lister-gen",
        "--input-dirs=%s" % input_pkg,
        "--output-package=%s/listers" % output_pkg,
    )

    # Generate the informers:
    run_gen(
        "informer-gen",
        "--input-dirs=%s" % input_pkg,
        "--versioned-clientset-package=%s/openshift" % output_pkg,
        "--listers-package=%s/listers" % output_pkg,
        "--output-package=%s/informers" % output_pkg,
        "--single-directory",
    )


@cache
def ensure_binaries():
    """
    Builds the binaries corresponding to each subdirectory of the 'cmd'
    directory. Returns a list containing the absolute path names of the
    generated binaries.
    """
    # Make sure that the vendor directory is populated and that the API client
    # code has been generated:
    ensure_vendor_dir()
    ensure_generated_code()

    # Get the names of the subdirectories of the 'cmd' directory:
    project_dir = ensure_project_dir()
    cmd_dir = os.path.join(project_dir, 'cmd')
    cmd_names = []
    for cmd_name in os.listdir(cmd_dir):
        cmd_path = os.path.join(cmd_dir, cmd_name)
        if os.path.isdir(cmd_path):
            cmd_names.append(cmd_name)
    cmd_names.sort()

    # Build the binaries:
    for cmd_name in cmd_names:
        say("Building binary '%s'" % cmd_name)
        cmd_path = "%s/cmd/%s" % (IMPORT_PATH, cmd_name)
        go_tool("go", "install", cmd_path)

    # Build the result:
    go_bin = ensure_go_bin()
    result = []
    for cmd_name in cmd_names:
        cmd_path = os.path.join(go_bin, cmd_name)
        result.append(cmd_path)
    return result


@cache
def ensure_images():
    """
    Builds the container images corresponding to each subdirectory of the
    'images' directory. Returns a list containing the tags assigned to the
    images.
    """
    # Get the names of the subdirectories of the 'images' directory:
    project_dir = ensure_project_dir()
    images_dir = os.path.join(project_dir, 'images')
    image_names = []
    for image_name in os.listdir(images_dir):
        image_path = os.path.join(images_dir, image_name)
        if os.path.isdir(image_path):
            image_names.append(image_name)
    image_names.sort()

    # Build the images:
    image_tags = []
    for image_name in image_names:
        say("Building image '%s'" % image_name)
        image_tag = ensure_image(image_name)
        image_tags.append(image_tag)

    # Return the list of tags:
    return image_tags


@cache
def ensure_image(image_name):
    """
    Builds the container image corresponding to the given name. Returns the
    tag assigned to the image.
    """
    # Locate the image directory:
    project_dir = ensure_project_dir()
    image_dir = os.path.join(project_dir, "images", image_name)

    # Calculate the image variables:
    image_vars = dict(
        image_name=image_name,
        image_dir=image_dir,
    )

    # Calculate the image tag:
    image_tag = "{project}/{image}:{version}".format(
        project=PROJECT_NAME,
        image=image_name,
        version=PROJECT_VERSION,
    )

    # The binaries will most probably be included in the images, so we need
    # to make sure that they are built:
    cmd_paths = ensure_binaries()

    # The image will be built in a temporary directory, and we need to
    # make sure that it is removed once done:
    try:
        tmp_dir = tempfile.mkdtemp(prefix=image_name + ".")
        say("Created temporary directory '%s' to build image '%s'" % (
            tmp_dir, image_name
        ))

        # Copy files from the original image directory to the temporary
        # directory, replacing templages (files ending with .in) with
        # the result of processing them.
        for src_dir, _, src_names in os.walk(image_dir):
            rel_path = os.path.relpath(src_dir, image_dir)
            dst_dir = os.path.join(tmp_dir, rel_path)
            if not os.path.exists(dst_dir):
                os.makedirs(dst_dir)
            for src_name in src_names:
                src_path = os.path.join(src_dir, src_name)
                if TEMPLATE_RE.search(src_name) is not None:
                    dst_name = TEMPLATE_RE.sub("", src_name)
                    dst_path = os.path.join(dst_dir, dst_name)
                    with open(src_path) as fd:
                        src_content = fd.read()
                    dst_content = process_template(src_content, image_vars)
                    with open(dst_path, "w") as fd:
                        fd.write(dst_content)
                else:
                    dst_name = src_name
                    dst_path = os.path.join(dst_dir, dst_name)
                    shutil.copyfile(src_path, dst_path)
                shutil.copymode(src_path, dst_path)

        # Download other source files that are be needed by the image, as
        # specified in the 'Downloads' file:
        downloads_path = os.path.join(tmp_dir, "Downloads")
        if os.path.exists(downloads_path):
            with open(downloads_path) as fd:
                for line in fd:
                    match = DOWNLOAD_RE.search(line)
                    if match is not None:
                        url = match.group("url")
                        digest = match.group("digest")
                        output = match.group("output")
                        src_path = download_file(url, digest)
                        dst_path = os.path.join(tmp_dir, output)
                        say("Copy file downloaded from URL '%s' to '%s'" % (
                            url, dst_path
                        ))
                        shutil.copyfile(src_path, dst_path)

        # Copy the binaries to the temporary directory, as most probably the
        # image will want to use them:
        for cmd_path in cmd_paths:
            src_name = os.path.basename(cmd_path)
            dst_path = os.path.join(tmp_dir, src_name)
            say("Copy binary '%s' to '%s'" % (cmd_path, dst_path))
            shutil.copyfile(cmd_path, dst_path)
            shutil.copymode(cmd_path, dst_path)

        # Run the build:
        args = [
            "docker",
            "build",
            "--tag=%s" % image_tag,
            ".",
        ]
        say("Running command '%s'" % " ".join(args))
        process = subprocess.Popen(args=args, cwd=tmp_dir)
        result = process.wait()
        if result != 0:
            raise Exception("Command '%s' failed with exit code %d" % (
                " ".join(args), result
            ))
    finally:
        shutil.rmtree(tmp_dir)

    # Return the tag:
    return image_tag


@cache
def ensure_image_tar(image_tag, compress=False):
    """
    Saves the image with the given tag to a .tar file. Returns the path
    of the .tar file.
    """
    # Calculate the name of the tar file:
    say("Saving image with tag '%s'" % image_tag)
    project_dir = ensure_project_dir()
    tar_name = image_tag
    tar_name = tar_name.replace("/", "_") # File system friendly.
    tar_name = tar_name.replace(":", "_") # SSH friendly.
    tar_name += ".tar"
    tar_path = os.path.join(project_dir, tar_name)

    # Run the command:
    args = [
        "docker",
        "save",
        "--output=%s" % tar_path,
        image_tag,
    ]
    say("Running command '%s'" % " ".join(args))
    process = subprocess.Popen(args)
    result = process.wait()
    if result != 0:
        raise Exception("Command '%s' failed with exit code %d" % (
            " ".join(args), result
        ))

    # Compress the tar files:
    if argv.compress:
        args = [
            "gzip",
            "--force",
            tar_path,
        ]
        say("Running command '%s'" % " ".join(args))
        process = subprocess.Popen(args)
        result = process.wait()
        if result != 0:
            raise Exception("Command '%s' failed with exit code %d" % (
                " ".join(args), result
            ))
        tar_path += ".gz"

    # Return the name of the tar file:
    return tar_path


@cache
def ensure_global_variables():
    """
    Returns a dictionary containing a set of global variables that are useful
    for processing templates.
    """
    return dict(
        go_bin=ensure_go_bin(),
        go_path=ensure_go_path(),
        go_pkg=ensure_go_pkg(),
        go_src=ensure_go_src(),
        import_path=IMPORT_PATH,
        project_dir=ensure_project_dir(),
        project_link=ensure_project_link(),
        project_name=PROJECT_NAME,
        project_version=PROJECT_VERSION,
    )


def process_template(template, variables={}):
    """
    Process the given template text and returns the result. The processing
    consists on replacing occurences of {{name}} with the value corresponding
    to the 'name' key in the variables dictionary.

    The given local variables dictionary will be merged with global variables
    (see the 'ensure_global_variables' function) so that local variables
    override global variables.
    """
    # Merge local and global variables, making sure that if there are local
    # variables override global variables with the same name:
    local_vars = dict(ensure_global_variables())
    for name, value in variables.items():
        local_vars[name] = value

    # Replace all the occurences of {{...}} with the value of the
    # corresponding variables:
    return VARIABLE_RE.sub(
        lambda match: local_vars.get(match.group("name")),
        template
    )


def download_file(url, digest):
    """
    Downloads the content of the given URL, verifies that the SHA265 digest
    is correct, and returns the absolute path of where the file is stored.
    """
    # To save time and IO downloads are saved to the '.downloads' directory
    # of the project, so first we need to make sure that it exists:
    project_dir = ensure_project_dir()
    cache_dir = os.path.join(project_dir, '.downloads')
    if not os.path.exists(cache_dir):
        os.mkdir(cache_dir)

    # If the file is already in the cache then we don't need to do anything
    # else, just return the location:
    cache_file = os.path.join(cache_dir, digest)
    if os.path.exists(cache_file):
        return cache_file

    # Download the URL to a temporary file, and calculate the digest while
    # at it:
    say("Downloading file from URL '%s'" % url)
    tmp_file = cache_file + ".tmp"
    tmp_digest = hashlib.sha256()
    with urllib.request.urlopen(url) as url_fd, open(tmp_file, "wb") as tmp_fd:
        while True:
            data = url_fd.read(8192)
            if len(data) == 0:
                break
            tmp_fd.write(data)
            tmp_digest.update(data)
    tmp_digest = tmp_digest.hexdigest()

    # Compare the digest provided by the caller with the calculated one:
    if tmp_digest != digest:
        os.remove(tmp_file)
        raise Exception(
            "The calculated digest '%s' for url '%s' is different to the "
            "expected '%s'" % (tmp_digest, url, digest)
        )

    # Rename the temporary file to the definitive one:
    os.rename(tmp_file, cache_file)

    # The download worked correctly, and the digest is good, just return
    # the path of the cache file:
    return cache_file


def build_binaries():
    """
    Implements the 'binaries' subcommand.
    """
    ensure_binaries()


def build_images():
    """
    Implements the 'images' subcommand.
    """
    # Build the images:
    image_tags = ensure_images()

    # Save the images to tar files:
    if argv.save:
        for image_tag in image_tags:
            ensure_image_tar(image_tag)


def main():
    # Create the top level command line parser:
    parser = argparse.ArgumentParser(
        prog=os.path.basename(sys.argv[0]),
        description="A simple build tool, just for this project.",
    )
    parser.add_argument(
        "--verbose",
        help="Genenerate verbose output.",
        default=False,
        action="store_true",
    )
    parser.add_argument(
        "--debug",
        help="Genenerate debug, very verbose, output.",
        default=False,
        action="store_true",
    )
    subparsers = parser.add_subparsers()

    # Create the parser for the 'binaries' command:
    binaries_parser = subparsers.add_parser("binaries")
    binaries_parser.set_defaults(func=build_binaries)

    # Create the parser for the 'images' command:
    images_parser = subparsers.add_parser('images')
    images_parser.add_argument(
        "--save",
        help="Save the images to tar files.",
        default=False,
        action="store_true",
    )
    images_parser.add_argument(
        "--compress",
        help="Compress the tar files.",
        default=False,
        action="store_true",
    )
    images_parser.set_defaults(func=build_images)

    # Parse the command line:
    global argv
    argv = parser.parse_args()
    if not hasattr(argv, "func"):
        parser.print_usage()
        sys.exit(1)

    # Run the selected subcommand:
    try:
        argv.func()
        sys.exit(0)
    except Exception as error:
        say(str(error))
        sys.exit(1)


if __name__ == "__main__":
    main()