diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 8bdc490958b0b..d34b22bd9e63b 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -17,6 +17,7 @@ from pathlib import Path import time +import sys import click @@ -273,16 +274,30 @@ def highlight(code): help='Fetch references (branches and tags) from the remote') @click.option('--task-filter', '-f', 'task_filters', multiple=True, help='Glob pattern for filtering relevant tasks') +@click.option('--validate/--no-validate', default=False, + help='Return non-zero exit code ' + 'if there is any non-success task') @click.pass_obj -def status(obj, job_name, fetch, task_filters): +def status(obj, job_name, fetch, task_filters, validate): output = obj['output'] queue = obj['queue'] if fetch: queue.fetch() job = queue.get(job_name) + success = True + + def asset_callback(task_name, task, asset): + nonlocal success + if task.status().combined_state in {'error', 'failure'}: + success = False + if asset is None: + success = False + report = ConsoleReport(job, task_filters=task_filters) - report.show(output) + report.show(output, asset_callback=asset_callback) + if validate and not success: + sys.exit(1) @crossbow.command() diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 49f915ec82d93..75b489f048d50 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -675,6 +675,7 @@ def put(self, job, prefix='build', increment_job_id=True): params = { **job.params, "arrow": job.target, + "job": job, "queue_remote_url": self.remote_url } files = task.render_files(job.template_searchpath, params=params) @@ -790,7 +791,7 @@ class Task(Serializable): submitting the job to a queue. """ - def __init__(self, ci, template, artifacts=None, params=None): + def __init__(self, name, ci, template, artifacts=None, params=None): assert ci in { 'circle', 'travis', @@ -799,6 +800,7 @@ def __init__(self, ci, template, artifacts=None, params=None): 'github', 'drone', } + self.name = name self.ci = ci self.template = template self.artifacts = artifacts or [] @@ -1011,6 +1013,7 @@ def render_tasks(self, params=None): params = { **self.params, "arrow": self.target, + "job": self, **(params or {}) } for task_name, task in self.tasks.items(): @@ -1081,9 +1084,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version} for task_name, task in task_definitions.items(): + task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml artifacts = [fn.format(**versions) for fn in artifacts] - tasks[task_name] = Task(artifacts=artifacts, **task) + tasks[task_name] = Task(task_name, artifacts=artifacts, **task) return cls(target=target, tasks=tasks, params=params, template_searchpath=config.template_searchpath) @@ -1219,7 +1223,7 @@ def validate(self): # validate that the tasks are constructible for task_name, task in self['tasks'].items(): try: - Task(**task) + Task(task_name, **task) except Exception as e: raise CrossbowError( 'Unable to construct a task object from the ' @@ -1236,13 +1240,19 @@ def validate(self): version='1.0.0dev123', email='dummy@example.ltd' ) + job = Job.from_config(config=self, + target=target, + tasks=self['tasks'], + groups=self['groups'], + params={}) for task_name, task in self['tasks'].items(): - task = Task(**task) + task = Task(task_name, **task) files = task.render_files( self.template_searchpath, params=dict( arrow=target, + job=job, queue_remote_url='https://github.com/org/crossbow' ) ) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index c7f7c647b7b5d..57a5cde32f526 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -69,25 +69,43 @@ on: uses: actions/setup-python@v4 with: python-version: 3.8 + - name: Checkout Crossbow + uses: actions/checkout@v3 + with: + path: crossbow + ref: {{ job.branch }} - name: Setup Crossbow shell: bash - run: pip install -e arrow/dev/archery[crossbow-upload] + run: pip install -e arrow/dev/archery[crossbow] - name: Upload artifacts shell: bash run: | archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - {% if pattern is string %} - "{{ pattern }}" - {% elif pattern is iterable %} - {% for p in pattern %} - "{{ p }}" {{ "\\" if not loop.last else "" }} - {% endfor %} - {% endif %} + --queue-path $(pwd)/crossbow \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} + - name: Verify uploaded artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd)/crossbow \ + --queue-remote {{ queue_remote_url }} \ + status \ + --task-filter '{{ task.name }}' \ + --no-fetch \ + --validate \ + {{ job.branch }} env: CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} {% endmacro %} @@ -170,7 +188,7 @@ on: {%- macro travis_upload_releases(pattern) -%} - sudo -H pip3 install pygit2==1.0 cryptography==36 - - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload] + - sudo -H pip3 install -e arrow/dev/archery[crossbow] - | archery crossbow \ --queue-path $(pwd) \ @@ -185,6 +203,16 @@ on: "{{ p }}" {{ "\\" if not loop.last else "" }} {% endfor %} {% endif %} + - git fetch origin {{ job.branch }}:remotes/origin/{{ job.branch }} + - | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + status \ + --task-filter '{{ task.name }}' \ + --no-fetch \ + --validate \ + {{ job.branch }} {% endmacro %} {%- macro travis_upload_gemfury(pattern) -%} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 00d41ec008e71..7ecc61022466c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -625,12 +625,14 @@ tasks: - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-glib1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-glib1000_{no_rc_version}-1_[a-z0-9]+.deb + {% if target != "ubuntu-bionic" %} - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python-flight1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-python-flight1000_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-python1000_{no_rc_version}-1_[a-z0-9]+.deb + {% endif %} - libarrow1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow1000_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb @@ -717,7 +719,7 @@ tasks: - arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if not is_rhel7_based %} + {% if not is_rhel7_based and architecture == "amd64" %} - arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm @@ -743,14 +745,14 @@ tasks: - arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if target != "amazon-linux-2" %} + {% if not is_rhel7_based %} - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if not is_rhel7_based %} + {% if architecture == "amd64" %} - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} + - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} {% if architecture == "amd64" %}