From babe610c7a626b7a834f645981ff56bbe9b4b480 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Sat, 23 Jun 2018 11:24:09 -0400 Subject: [PATCH 1/7] Change default src dir to someplace in tmp for now --- ska_conda/ska_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index cd4a7c53..b65d45f7 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -11,7 +11,7 @@ class SkaBuilder(object): def __init__(self, ska_root=None): if ska_root is None: - ska_root = "/data/acis/ska3_pkg/" + ska_root = "/tmp/ska3_pkg/" self.ska_root = ska_root self.ska_build_dir = os.path.join(self.ska_root, "builds") self.ska_src_dir = os.path.join(self.ska_root, "src") From 80331470a104d56021953196533e4096fe6e829b Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Sat, 23 Jun 2018 11:28:15 -0400 Subject: [PATCH 2/7] Clone repos at latest tag by default This sets the git clone pieces to checkout at the latest (by commit date) tag for the repos by default. Should also tag a tag as an option for the individual clone/fetches. Also adds a few lines to try to get the repos via ssh from the sot org if possible. --- ska_conda/ska_builder.py | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index b65d45f7..b48789f0 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -17,26 +17,48 @@ def __init__(self, ska_root=None): self.ska_src_dir = os.path.join(self.ska_root, "src") os.environ["SKA_TOP_SRC_DIR"] = self.ska_src_dir - def _clone_repo(self, name): + def _clone_repo(self, name, tag=None): if name == "ska": return print("Cloning source %s." % name) clone_path = os.path.join(self.ska_src_dir, name) - if os.path.exists(clone_path): - print("Source %s exists, skipping." % name) - return - yml = os.path.join(pkg_defs_path, name, "meta.yaml") - with open(yml) as f: - requires = False - while not requires: - line = f.readline().strip() - if line.startswith("path:"): - requires = True - data = yaml.load(f) - url = data['about']['home'] - git.Repo.clone_from(url, clone_path) + if not os.path.exists(clone_path): + # Try ssh first to avoid needing passwords for the private repos + # We could add these ssh strings to the meta.yaml for convenience + try: + git_ssh_path = 'git@github.com:sot/' + name + '.git' + repo = git.Repo.clone_from(git_ssh_path, clone_path) + assert not repo.bare + except: + yml = os.path.join(pkg_defs_path, name, "meta.yaml") + with open(yml) as f: + requires = False + while not requires: + line = f.readline().strip() + if line.startswith("path:"): + requires = True + data = yaml.load(f) + url = data['about']['home'] + repo = git.Repo.clone_from(url, clone_path) + else: + repo = git.Repo(clone_path) + assert not repo.is_dirty() + # I think we want the commit/tag with the most recent date, though + # if we actually want the most recently created tag, that would probably be + # tags = sorted(repo.tags, key=lambda t: t.tag.tagged_date) + # I suppose we could also use github to get the most recent release (not tag) + if tag is None: + tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime) + repo.git.checkout(tags[-1].name) + if tags[-1].commit == repo.heads.master.commit: + print("Auto-checked out at {} which is also tip of master".format(tags[-1].name)) + else: + print("Auto-checked out at {} NOT AT tip of master".format(tags[-1].name)) + else: + repo.git.checkout(tag) + print("Checked out at {}".format(tag)) - def clone_one_package(self, name): + def clone_one_package(self, name, tag=None): self._clone_repo(name) def clone_all_packages(self): From eca8683a4b31436c7ce767c48f74a136af601254 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Mon, 25 Jun 2018 17:26:58 -0400 Subject: [PATCH 3/7] Fetch recent tags from remote into clone --- ska_conda/ska_builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index b48789f0..03a1e76e 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -42,6 +42,8 @@ def _clone_repo(self, name, tag=None): repo = git.Repo.clone_from(url, clone_path) else: repo = git.Repo(clone_path) + repo.remotes.origin.fetch() + repo.remotes.origin.fetch("--tags") assert not repo.is_dirty() # I think we want the commit/tag with the most recent date, though # if we actually want the most recently created tag, that would probably be From caafcc30d67f2f407eca6700a46d7a31cd8ea65b Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 26 Jun 2018 09:29:41 -0400 Subject: [PATCH 4/7] Use default root of . and change some dir names --- scripts/build_ska_packages | 6 +++--- scripts/clone_ska_sources | 6 +++--- ska_conda/ska_builder.py | 20 +++++++++----------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/build_ska_packages b/scripts/build_ska_packages index 49e4a638..d8183852 100644 --- a/scripts/build_ska_packages +++ b/scripts/build_ska_packages @@ -13,13 +13,13 @@ parser = argparse.ArgumentParser(description="Build Ska Conda packages.") parser.add_argument("mode", type=str, help="The build mode. Either 'all' for all " "packages, 'updated' for updated " "packages, or a specific package name.") -parser.add_argument("--ska_root", type=str, +parser.add_argument("--build_root", default=".", type=str, help="The path to the root directory for the Ska packages. " - "Default: /data/acis/ska3_pkg") + "Default: '.'") args = parser.parse_args() -ska_builder = SkaBuilder(ska_root=args.ska_root) +ska_builder = SkaBuilder(build_root=args.build_root) if args.mode == "all": ska_builder.build_all_packages() diff --git a/scripts/clone_ska_sources b/scripts/clone_ska_sources index e791b85a..0891b110 100644 --- a/scripts/clone_ska_sources +++ b/scripts/clone_ska_sources @@ -8,13 +8,13 @@ parser = argparse.ArgumentParser(description="Clone Ska Conda sources.") parser.add_argument("mode", type=str, help="The clone mode. Either 'all' for all " "packages, or a specific package name.") -parser.add_argument("--ska_root", type=str, +parser.add_argument("--build_root", default=".", type=str, help="The path to the root directory for the Ska packages. " - "Default: /data/acis/ska3_pkg") + "Default: '.'") args = parser.parse_args() -ska_builder = SkaBuilder(ska_root=args.ska_root) +ska_builder = SkaBuilder(build_root=args.build_root) if args.mode == "all": ska_builder.clone_all_packages() diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 03a1e76e..5b052c5c 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -9,25 +9,23 @@ class SkaBuilder(object): - def __init__(self, ska_root=None): - if ska_root is None: - ska_root = "/tmp/ska3_pkg/" - self.ska_root = ska_root - self.ska_build_dir = os.path.join(self.ska_root, "builds") - self.ska_src_dir = os.path.join(self.ska_root, "src") - os.environ["SKA_TOP_SRC_DIR"] = self.ska_src_dir + def __init__(self, build_root='.', user='sot', git_repo_path='git@github.com:{user}/{name}.git'): + self.user = user + self.git_repo_path = git_repo_path + self.build_dir = os.path.join(build_root, "builds") + self.src_dir = os.path.join(build_root, "src") + os.environ["SKA_TOP_SRC_DIR"] = self.src_dir def _clone_repo(self, name, tag=None): if name == "ska": return print("Cloning source %s." % name) - clone_path = os.path.join(self.ska_src_dir, name) + clone_path = os.path.join(self.src_dir, name) if not os.path.exists(clone_path): # Try ssh first to avoid needing passwords for the private repos # We could add these ssh strings to the meta.yaml for convenience try: - git_ssh_path = 'git@github.com:sot/' + name + '.git' - repo = git.Repo.clone_from(git_ssh_path, clone_path) + repo = git.Repo.clone_from(self.git_repo_path.format(user=self.user, name=name), clone_path) assert not repo.bare except: yml = os.path.join(pkg_defs_path, name, "meta.yaml") @@ -73,7 +71,7 @@ def clone_all_packages(self): def _get_repo(self, name): if name == "ska": return None - repo_path = os.path.join(self.ska_src_dir, name) + repo_path = os.path.join(self.src_dir, name) if not os.path.exists(repo_path): self._clone_repo(name) repo = git.Repo(repo_path) From 3e4bd69e1848fbe0b4438bb0d7263f57fc983422 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 26 Jun 2018 09:30:27 -0400 Subject: [PATCH 5/7] Use --skip-existing to build new/updated packages --- ska_conda/ska_builder.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 5b052c5c..e9b1f0ea 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -81,14 +81,12 @@ def _build_package(self, name): print("Building package %s." % name) pkg_path = os.path.join(pkg_defs_path, name) cmd_list = ["conda", "build", pkg_path, "--croot", - self.ska_build_dir, "--no-test", - "--no-anaconda-upload"] + self.build_dir, "--no-test", + "--no-anaconda-upload", "--skip-existing"] subprocess.run(cmd_list) def build_one_package(self, name): repo = self._get_repo(name) - if repo is not None: - repo.remote().pull() self._build_package(name) def build_updated_packages(self, new_only=True): @@ -96,15 +94,7 @@ def build_updated_packages(self, new_only=True): for line in f.readlines(): pkg_name = line.strip() if not pkg_name.startswith("#"): - repo = self._get_repo(pkg_name) - if repo is None: - build = True - else: - current_tag = repo.tags[-1].name - repo.remote().pull() - build = repo.tags[-1].name != current_tag - if build or not new_only: - self._build_package(pkg_name) + self._build_package(pkg_name) def build_all_packages(self): self.build_updated_packages(new_only=False) From e7986c1b47f7c46c6b313b84a895e33616e3a21b Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 26 Jun 2018 09:35:23 -0400 Subject: [PATCH 6/7] Use absolute dir paths (conda build seems to require) --- ska_conda/ska_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index e9b1f0ea..44ad3f77 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -12,8 +12,8 @@ class SkaBuilder(object): def __init__(self, build_root='.', user='sot', git_repo_path='git@github.com:{user}/{name}.git'): self.user = user self.git_repo_path = git_repo_path - self.build_dir = os.path.join(build_root, "builds") - self.src_dir = os.path.join(build_root, "src") + self.build_dir = os.path.abspath(os.path.join(build_root, "builds")) + self.src_dir = os.path.abspath(os.path.join(build_root, "src")) os.environ["SKA_TOP_SRC_DIR"] = self.src_dir def _clone_repo(self, name, tag=None): From f88ac925f97f98d61c33c254ef21495f22e9c674 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 26 Jun 2018 09:45:48 -0400 Subject: [PATCH 7/7] Remove extra fetch of tags --- ska_conda/ska_builder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 44ad3f77..f688149b 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -41,7 +41,6 @@ def _clone_repo(self, name, tag=None): else: repo = git.Repo(clone_path) repo.remotes.origin.fetch() - repo.remotes.origin.fetch("--tags") assert not repo.is_dirty() # I think we want the commit/tag with the most recent date, though # if we actually want the most recently created tag, that would probably be