Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add --disable-job in 'eb' command used in jobs, to prevent infinite job cycle #3328

Merged
merged 1 commit into from
May 7, 2020
Merged

add --disable-job in 'eb' command used in jobs, to prevent infinite job cycle #3328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion easybuild/tools/parallelbuild.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,17 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
curdir = os.getcwd()

# regex pattern for options to ignore (help options can't reach here)
ignore_opts = re.compile('^--robot$|^--job$|^--try-.*$')
ignore_opts = re.compile('^--robot$|^--job|^--try-.*$')

# generate_cmd_line returns the options in form --longopt=value
opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])]

# add --disable-job to make sure the submitted job doesn't submit a job itself,
# resulting in an infinite cycle of jobs;
# this can happen if job submission is enabled via a configuration file or via $EASYBUILD_JOB,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/3307
opts.append('--disable-job')

# compose string with command line options, properly quoted and with '%' characters escaped
opts_str = ' '.join(opts).replace('%', '%%')

Expand Down
12 changes: 8 additions & 4 deletions test/framework/parallelbuild.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,10 @@ def test_build_easyconfigs_in_parallel_pbs_python(self):
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
regex = re.compile(r'toy-0.0-deps\.eb.* --hidden')
script_txt = jobs[3].deps[0].script
fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern, script_txt)
self.assertTrue(regex.search(script_txt), fail_msg)
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)

# also test use of --pre-create-installdir
Expand Down Expand Up @@ -290,6 +292,7 @@ def test_submit_jobs(self):
'--try-toolchain=intel,2016a', # should be excluded in job script
'--robot', self.test_prefix, # should be excluded in job script
'--job', # should be excluded in job script
'--job-cores=3',
]
eb_go = parse_options(args=args)
cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
Expand All @@ -306,16 +309,17 @@ def test_submit_jobs(self):
' eb %\(spec\)s ',
' %\(add_opts\)s ',
' --testoutput=%\(output_dir\)s',
' --disable-job ',
]
for regex in regexs:
regex = re.compile(regex)
self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd))

# these patterns should NOT be found, these options get filtered out
# (self.test_prefix was argument to --robot)
for regex in ['--job', '--try-toolchain', '--robot=[ =]', self.test_prefix + ' ']:
for regex in ['--job', '--job-cores', '--try-toolchain', '--robot=[ =]', self.test_prefix + ' ']:
regex = re.compile(regex)
self.assertFalse(regex.search(cmd), "Pattern '%s' *not* found in: %s" % (regex.pattern, cmd))
self.assertFalse(regex.search(cmd), "Pattern '%s' should *not* be found in: %s" % (regex.pattern, cmd))

def test_build_easyconfigs_in_parallel_slurm(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
Expand Down