From 697654da7a0d64c6567b32622e60c1fad7ea5abe Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 29 Jul 2022 14:52:15 +0900 Subject: [PATCH 01/12] ARROW-17233: [Packaging][Linux] Update artifact patterns --- dev/tasks/macros.jinja | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index c7f7c647b7b5d..ced9952bec31d 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -76,18 +76,28 @@ on: shell: bash run: | archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - {% if pattern is string %} - "{{ pattern }}" - {% elif pattern is iterable %} - {% for p in pattern %} - "{{ p }}" {{ "\\" if not loop.last else "" }} - {% endfor %} - {% endif %} + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} + - name: Verify uploaded artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + download-artifacts \ + {{ task.tag }} env: CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} {% endmacro %} @@ -185,6 +195,12 @@ on: "{{ p }}" {{ "\\" if not loop.last else "" }} {% endfor %} {% endif %} + - | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + download-artifacts \ + {{ task.tag }} {% endmacro %} {%- macro travis_upload_gemfury(pattern) -%} From 147f68887c24407ac48813f434f59059c2ca0aa1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 29 Jul 2022 17:15:49 +0900 Subject: [PATCH 02/12] Add --no-fetch --- dev/tasks/macros.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index ced9952bec31d..231f74a694e55 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -94,6 +94,7 @@ on: shell: bash run: | archery crossbow \ + --no-fetch \ --queue-path $(pwd) \ --queue-remote {{ queue_remote_url }} \ download-artifacts \ @@ -197,6 +198,7 @@ on: {% endif %} - | archery crossbow \ + --no-fetch \ --queue-path $(pwd) \ --queue-remote {{ queue_remote_url }} \ download-artifacts \ From b498960d72233eeaaff87b8affbd2f1a6a4147d6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 30 Jul 2022 06:44:14 +0900 Subject: [PATCH 03/12] Fix arguments order --- dev/tasks/macros.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 231f74a694e55..d1e39878acabd 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -94,10 +94,10 @@ on: shell: bash run: | archery crossbow \ - --no-fetch \ --queue-path $(pwd) \ --queue-remote {{ queue_remote_url }} \ download-artifacts \ + --no-fetch \ {{ task.tag }} env: CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} @@ -198,10 +198,10 @@ on: {% endif %} - | archery crossbow \ - --no-fetch \ --queue-path $(pwd) \ --queue-remote {{ queue_remote_url }} \ download-artifacts \ + --no-fetch \ {{ task.tag }} {% endmacro %} From 6be5739853974cf2afddcb903f0c4d675f726f7b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Aug 2022 12:21:03 +0900 Subject: [PATCH 04/12] Fix job branch --- dev/archery/archery/crossbow/core.py | 1 + dev/tasks/macros.jinja | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 49f915ec82d93..8c923503707f0 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -1011,6 +1011,7 @@ def render_tasks(self, params=None): params = { **self.params, "arrow": self.target, + "job": self, **(params or {}) } for task_name, task in self.tasks.items(): diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index d1e39878acabd..3d29094fcfd09 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -71,7 +71,7 @@ on: python-version: 3.8 - name: Setup Crossbow shell: bash - run: pip install -e arrow/dev/archery[crossbow-upload] + run: pip install -e arrow/dev/archery[crossbow] - name: Upload artifacts shell: bash run: | @@ -98,7 +98,7 @@ on: --queue-remote {{ queue_remote_url }} \ download-artifacts \ --no-fetch \ - {{ task.tag }} + {{ job.branch }} env: CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} {% endmacro %} @@ -181,7 +181,7 @@ on: {%- macro travis_upload_releases(pattern) -%} - sudo -H pip3 install pygit2==1.0 cryptography==36 - - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload] + - sudo -H pip3 install -e arrow/dev/archery[crossbow] - | archery crossbow \ --queue-path $(pwd) \ @@ -202,7 +202,7 @@ on: --queue-remote {{ queue_remote_url }} \ download-artifacts \ --no-fetch \ - {{ task.tag }} + {{ job.branch }} {% endmacro %} {%- macro travis_upload_gemfury(pattern) -%} From 1390cc5b8f8e08b59ee3bcdcef370313a4215942 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Aug 2022 12:26:45 +0900 Subject: [PATCH 05/12] Add more job --- dev/archery/archery/crossbow/core.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 8c923503707f0..91f3c83e78eaa 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -675,6 +675,7 @@ def put(self, job, prefix='build', increment_job_id=True): params = { **job.params, "arrow": job.target, + "job": job, "queue_remote_url": self.remote_url } files = task.render_files(job.template_searchpath, params=params) @@ -1237,6 +1238,10 @@ def validate(self): version='1.0.0dev123', email='dummy@example.ltd' ) + job = Job( + target=target, + tasks=self['tasks'], + ) for task_name, task in self['tasks'].items(): task = Task(**task) @@ -1244,6 +1249,7 @@ def validate(self): self.template_searchpath, params=dict( arrow=target, + job=job, queue_remote_url='https://github.com/org/crossbow' ) ) From e56fed56c950288bfca78fa1ee12eb6a651a94fd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Aug 2022 13:56:04 +0900 Subject: [PATCH 06/12] Use Job.from_config --- dev/archery/archery/crossbow/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 91f3c83e78eaa..ced594304c3e4 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -1238,10 +1238,11 @@ def validate(self): version='1.0.0dev123', email='dummy@example.ltd' ) - job = Job( - target=target, - tasks=self['tasks'], - ) + job = Job.from_config(config=self, + target=target, + tasks=self['tasks'], + groups=self['groups'], + params={}) for task_name, task in self['tasks'].items(): task = Task(**task) From 44798fb3480f24583141b976abdcee9b38e15a52 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 14 Aug 2022 00:53:01 +0900 Subject: [PATCH 07/12] Add support for "status --verify" --- dev/archery/archery/crossbow/cli.py | 10 +++++++++- dev/archery/archery/crossbow/core.py | 10 ++++++---- dev/tasks/macros.jinja | 18 ++++++++++++++---- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 8bdc490958b0b..61dcde529da4c 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -17,6 +17,7 @@ from pathlib import Path import time +import sys import click @@ -273,8 +274,10 @@ def highlight(code): help='Fetch references (branches and tags) from the remote') @click.option('--task-filter', '-f', 'task_filters', multiple=True, help='Glob pattern for filtering relevant tasks') +@click.option('--validate/--no-validate', default=False, + help='Return non-zero exit code if there is any non-success task') @click.pass_obj -def status(obj, job_name, fetch, task_filters): +def status(obj, job_name, fetch, task_filters, validate): output = obj['output'] queue = obj['queue'] if fetch: @@ -284,6 +287,11 @@ def status(obj, job_name, fetch, task_filters): report = ConsoleReport(job, task_filters=task_filters) report.show(output) + if validate: + states = [task.status().combined_state for task in report.tasks.values()] + if set(states) != {'success'}: + sys.exit(1) + @crossbow.command() @click.option('--arrow-remote', '-r', default=None, diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index ced594304c3e4..75b489f048d50 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -791,7 +791,7 @@ class Task(Serializable): submitting the job to a queue. """ - def __init__(self, ci, template, artifacts=None, params=None): + def __init__(self, name, ci, template, artifacts=None, params=None): assert ci in { 'circle', 'travis', @@ -800,6 +800,7 @@ def __init__(self, ci, template, artifacts=None, params=None): 'github', 'drone', } + self.name = name self.ci = ci self.template = template self.artifacts = artifacts or [] @@ -1083,9 +1084,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version} for task_name, task in task_definitions.items(): + task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml artifacts = [fn.format(**versions) for fn in artifacts] - tasks[task_name] = Task(artifacts=artifacts, **task) + tasks[task_name] = Task(task_name, artifacts=artifacts, **task) return cls(target=target, tasks=tasks, params=params, template_searchpath=config.template_searchpath) @@ -1221,7 +1223,7 @@ def validate(self): # validate that the tasks are constructible for task_name, task in self['tasks'].items(): try: - Task(**task) + Task(task_name, **task) except Exception as e: raise CrossbowError( 'Unable to construct a task object from the ' @@ -1245,7 +1247,7 @@ def validate(self): params={}) for task_name, task in self['tasks'].items(): - task = Task(**task) + task = Task(task_name, **task) files = task.render_files( self.template_searchpath, params=dict( diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3d29094fcfd09..e4003d789f7be 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -69,6 +69,11 @@ on: uses: actions/setup-python@v4 with: python-version: 3.8 + - name: Checkout Crossbow + uses: actions/checkout@v3 + with: + path: crossbow + ref: {{ job.branch }} - name: Setup Crossbow shell: bash run: pip install -e arrow/dev/archery[crossbow] @@ -76,7 +81,7 @@ on: shell: bash run: | archery crossbow \ - --queue-path $(pwd) \ + --queue-path $(pwd)/crossbow \ --queue-remote {{ queue_remote_url }} \ upload-artifacts \ --sha {{ task.branch }} \ @@ -94,10 +99,12 @@ on: shell: bash run: | archery crossbow \ - --queue-path $(pwd) \ + --queue-path $(pwd)/crossbow \ --queue-remote {{ queue_remote_url }} \ - download-artifacts \ + status \ + --task-filter '{{ task.name }}' \ --no-fetch \ + --validate \ {{ job.branch }} env: CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} @@ -196,12 +203,15 @@ on: "{{ p }}" {{ "\\" if not loop.last else "" }} {% endfor %} {% endif %} + - git fetch origin {{ job.branch }} - | archery crossbow \ --queue-path $(pwd) \ --queue-remote {{ queue_remote_url }} \ - download-artifacts \ + status \ + --task-filter '{{ task.name }}' \ --no-fetch \ + --validate \ {{ job.branch }} {% endmacro %} From 0d46eefc0a60cba430e7b8ae812334d3452442b4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 14 Aug 2022 14:35:45 +0900 Subject: [PATCH 08/12] Update artifacts --- dev/tasks/tasks.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 00d41ec008e71..7ecc61022466c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -625,12 +625,14 @@ tasks: - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-glib1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-glib1000_{no_rc_version}-1_[a-z0-9]+.deb + {% if target != "ubuntu-bionic" %} - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python-flight1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-python-flight1000_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-python1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow-python1000_{no_rc_version}-1_[a-z0-9]+.deb + {% endif %} - libarrow1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - libarrow1000_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb @@ -717,7 +719,7 @@ tasks: - arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if not is_rhel7_based %} + {% if not is_rhel7_based and architecture == "amd64" %} - arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm @@ -743,14 +745,14 @@ tasks: - arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if target != "amazon-linux-2" %} + {% if not is_rhel7_based %} - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if not is_rhel7_based %} + {% if architecture == "amd64" %} - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} + - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} {% if architecture == "amd64" %} From 234e6604b9d322a1d9da726ecc90e203eb39d4db Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 14 Aug 2022 21:23:51 +0900 Subject: [PATCH 09/12] Fix failure --- dev/archery/archery/crossbow/cli.py | 14 ++++++++------ dev/tasks/macros.jinja | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 61dcde529da4c..1907f704896c0 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -285,12 +285,14 @@ def status(obj, job_name, fetch, task_filters, validate): job = queue.get(job_name) report = ConsoleReport(job, task_filters=task_filters) - report.show(output) - - if validate: - states = [task.status().combined_state for task in report.tasks.values()] - if set(states) != {'success'}: - sys.exit(1) + success = True + def asset_callback(task_name, task, asset): + nonlocal success + if asset is None: + success = False + report.show(output, asset_callback=asset_callback) + if validate and not success: + sys.exit(1) @crossbow.command() diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index e4003d789f7be..fb3d2148035f3 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -203,7 +203,7 @@ on: "{{ p }}" {{ "\\" if not loop.last else "" }} {% endfor %} {% endif %} - - git fetch origin {{ job.branch }} + - git fetch origin {{ job.branch }}:{{ job.branch }} - | archery crossbow \ --queue-path $(pwd) \ From 8d6b27ec409cfb04a90360dbbbc7d81a6c606778 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 14 Aug 2022 21:32:16 +0900 Subject: [PATCH 10/12] Add more success check --- dev/archery/archery/crossbow/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 1907f704896c0..b46e726daef35 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -275,7 +275,8 @@ def highlight(code): @click.option('--task-filter', '-f', 'task_filters', multiple=True, help='Glob pattern for filtering relevant tasks') @click.option('--validate/--no-validate', default=False, - help='Return non-zero exit code if there is any non-success task') + help='Return non-zero exit code ' + 'if there is any non-success task') @click.pass_obj def status(obj, job_name, fetch, task_filters, validate): output = obj['output'] @@ -288,6 +289,8 @@ def status(obj, job_name, fetch, task_filters, validate): success = True def asset_callback(task_name, task, asset): nonlocal success + if task.status().combined_state in {'error', 'failure'}: + success = False if asset is None: success = False report.show(output, asset_callback=asset_callback) From b0465acfbbfca025d6982a7728b3e687fae18cb5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 14 Aug 2022 22:19:44 +0900 Subject: [PATCH 11/12] Fix style --- dev/archery/archery/crossbow/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index b46e726daef35..d34b22bd9e63b 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -285,14 +285,16 @@ def status(obj, job_name, fetch, task_filters, validate): queue.fetch() job = queue.get(job_name) - report = ConsoleReport(job, task_filters=task_filters) success = True + def asset_callback(task_name, task, asset): nonlocal success if task.status().combined_state in {'error', 'failure'}: success = False if asset is None: success = False + + report = ConsoleReport(job, task_filters=task_filters) report.show(output, asset_callback=asset_callback) if validate and not success: sys.exit(1) From f3cfeaee8c3010475d25e1d2c093abbdb8d6b6f0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 15 Aug 2022 00:11:31 +0900 Subject: [PATCH 12/12] Fix ref --- dev/tasks/macros.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index fb3d2148035f3..57a5cde32f526 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -203,7 +203,7 @@ on: "{{ p }}" {{ "\\" if not loop.last else "" }} {% endfor %} {% endif %} - - git fetch origin {{ job.branch }}:{{ job.branch }} + - git fetch origin {{ job.branch }}:remotes/origin/{{ job.branch }} - | archery crossbow \ --queue-path $(pwd) \