Skip to content

Commit

Permalink
Add new multi-arch build layout mode
Browse files Browse the repository at this point in the history
Add a new mode which allows cosa to manipulate multi-arch build layouts:

```
$ find builds
builds
builds/builds.json
builds/30.1
builds/30.1/x86_64
builds/30.1/x86_64/coreos-assembler-config.tar.gz
builds/30.1/x86_64/coreos-assembler-config-git.json
builds/30.1/x86_64/fedora-coreos-30.1-qemu.qcow2
...
```

A pipeline could e.g. dispatch builds for each architecture on different
nodes, then group them back into a single workdir and have it
manipulated by e.g. `buildupload` seamlessly.

This new layout also matches the bucket layout for FCOS (see
coreos/fedora-coreos-tracker#189).

The basic idea is to add a `schema-version` to `builds.json` and denote
the legacy behaviour as "pre-1.0.0", while `1.0.0` contains a different
schema: each element in the `builds` array is now an object, which has
an `id`, and a list of `archs` for which that build has been built:

```
$ cat builds/builds.json
{
    "schema-version": "1.0.0",
    "builds": [
        {
            "id": "30.1",
            "archs": [
                "x86_64"
            ]
        }
    ],
    "timestamp": "2019-06-28T20:50:54Z"
}
```

We retain backwards-compatibility by simply checking the schema version.
Right now, only new workdirs will have this layout. Pipelines which use
`buildprep` will fetch `builds.json` as is and key off of its contents
to determine the bucket layout as well. We can write new code in the
future to convert previously single-arch buckets into the new layout to
then enable multi-arch.
  • Loading branch information
jlebon committed Jul 10, 2019
1 parent da6622b commit e3ae633
Show file tree
Hide file tree
Showing 6 changed files with 108 additions and 18 deletions.
2 changes: 2 additions & 0 deletions src/cmd-buildprep
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ def main():
print("Remote has no builds!")
return

# NB: We only buildprep for the arch we're on for now. Could probably make
# this a switch in the future.

buildid = builds.get_latest()
builddir = builds.get_build_dir(buildid)
Expand Down
7 changes: 6 additions & 1 deletion src/cmd-buildupload
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,12 @@ def cmd_upload_s3(args):
if args.build == 'latest':
args.build = builds.get_latest()
print(f"Targeting build: {args.build}")
s3_upload_build(args, builds.get_build_dir(args.build), args.build)
if builds.is_legacy():
s3_upload_build(args, builds.get_build_dir(args.build), args.build)
else:
for arch in builds.get_build_archs(args.build):
s3_upload_build(args, builds.get_build_dir(args.build, arch),
f'{args.build}/{arch}')
s3_cp(args, 'builds/builds.json', 'builds.json',
'--cache-control=max-age=60')

Expand Down
10 changes: 8 additions & 2 deletions src/cmd-compress
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,13 @@ def compress_one_builddir(builddir):
return at_least_one


changed = compress_one_builddir(builds.get_build_dir(build))
changed = []
if builds.is_legacy():
changed.append(compress_one_builddir(builds.get_build_dir(build)))
else:
for arch in builds.get_build_archs(build):
builddir = builds.get_build_dir(build, arch)
changed.append(compress_one_builddir(builddir))

if not changed:
if not any(changed):
print(f"All builds already compressed")
66 changes: 57 additions & 9 deletions src/cmdlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import sys
import tempfile
import gi
import semver

gi.require_version("RpmOstree", "1.0")
from gi.repository import RpmOstree
Expand Down Expand Up @@ -181,39 +182,86 @@ def __init__(self, workdir=None):
elif os.path.isfile(self._fn):
self._data = load_json(self._fn)
else:
# must be a new workdir
# must be a new workdir; use new schema
self._data = {
'schema-version': "1.0.0",
'builds': []
}
self.flush()
self._version = semver.parse_version_info(
self._data.get('schema-version', "0.0.1"))
# we understand < 2.0.0 only
if self._version._major >= 2:
raise Exception("Builds schema too new; please update cosa")
# for now, since we essentially just support "1.0.0" and "0.0.1",
# just dillute to a bool
self._legacy = (self._version._major < 1)

def _path(self, path):
if not self._workdir:
return path
return os.path.join(self._workdir, path)

def has(self, build_id):
return build_id in self._data['builds']
if self._legacy:
return build_id in self._data['builds']
return any([b['id'] == build_id for b in self._data['builds']])

def is_empty(self):
return len(self._data['builds']) == 0

def get_latest(self):
# just let throw if there are none
return self._data['builds'][0]

def get_build_dir(self, build_id):
if self._legacy:
return self._data['builds'][0]
return self._data['builds'][0]['id']

def get_build_archs(self, build_id):
assert not self._legacy
for build in self._data['builds']:
if build['id'] == build_id:
return build['archs']
assert False, "Build not found!"

def get_build_dir(self, build_id, basearch=None):
if build_id == 'latest':
build_id = self.get_latest()
return self._path(f"builds/{build_id}")

def insert_build(self, build_id):
self._data['builds'].insert(0, build_id)
if self._legacy:
return self._path(f"builds/{build_id}")
if not basearch:
# just assume caller wants build dir for current arch
basearch = get_basearch()
return self._path(f"builds/{build_id}/{basearch}")

def insert_build(self, build_id, basearch=None):
if self._legacy:
self._data['builds'].insert(0, build_id)
else:
if not basearch:
basearch = get_basearch()
# for future tooling: allow inserting in an existing build for a
# separate arch
for build in self._data['builds']:
if build['id'] == build_id:
if basearch in build['archs']:
raise "Build {build_id} for {basearch} already exists"
build['archs'] += [basearch]
break
else:
self._data['builds'].insert(0, {
'id': build_id,
'archs': [
basearch
]
})

def bump_timestamp(self):
self._data['timestamp'] = rfc3339_time()
self.flush()

def is_legacy(self):
return self._legacy

def raw(self):
return self._data

Expand Down
3 changes: 3 additions & 0 deletions src/deps.txt
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ jq
# For interacting with AWS/HTTP
awscli python3-boto3 python3-requests

# For metadata versioning
python3-semver

# For ignition file validation in cmd-run
/usr/bin/ignition-validate

Expand Down
38 changes: 32 additions & 6 deletions src/prune_builds
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import collections
from datetime import timedelta, datetime, timezone

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from cmdlib import Builds
from cmdlib import get_basearch, Builds


def parse_date_string(date_string):
Expand All @@ -34,7 +34,7 @@ def parse_date_string(date_string):
return dt.replace(tzinfo=timezone.utc)


Build = collections.namedtuple('Build', ['id', 'timestamp'])
Build = collections.namedtuple('Build', ['id', 'timestamp', 'basearchs'])

# Let's just hardcode this here for now
DEFAULT_KEEP_LAST_N = 3
Expand Down Expand Up @@ -109,9 +109,34 @@ with os.scandir(builds_dir) as it:
print(f"Ignoring non-directory {entry.path}")
continue

ts = get_timestamp(entry)
if ts:
scanned_builds.append(Build(id=entry.name, timestamp=ts))
if builds.is_legacy():
ts = get_timestamp(entry)
if ts:
scanned_builds.append(Build(id=entry.name, timestamp=ts,
basearchs=[get_basearch()]))
continue

# scan all per-arch builds, pick up the most recent build of those as
# the overall "build" timestamp for pruning purposes
with os.scandir(entry.path) as basearch_it:
multiarch_build = None
for basearch_entry in basearch_it:
# ignore non-dirs
if not basearch_entry.is_dir(follow_symlinks=False):
print(f"Ignoring non-directory {basearch_entry.path}")
continue
ts = get_timestamp(basearch_entry)
if not ts:
continue
if not multiarch_build:
multiarch_build = Build(id=entry.name, timestamp=ts,
basearchs=[basearch_entry.name])
else:
multiarch_build.basearchs += [basearch_entry.name]
multiarch_build.timestamp = max(
multiarch_build.timestamp, ts)
if multiarch_build:
scanned_builds.append(multiarch_build)


# just get the trivial case out of the way
Expand Down Expand Up @@ -159,7 +184,8 @@ else:

builds.raw()['builds'] = []
for build in reversed(new_builds):
builds.insert_build(build.id)
for basearch in build.basearchs:
builds.insert_build(build.id, basearch)
builds.bump_timestamp()

# if we're not pruning, then we're done!
Expand Down

0 comments on commit e3ae633

Please sign in to comment.