Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild #3945

Merged
merged 10 commits into from
Feb 9, 2022
Merged

switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild #3945

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1d189b9
Python 2 does not work, version 3 does
scimerman Jan 27, 2022
946454f
Update containers.py
ocaisa Feb 4, 2022
30ac78d
Update default Ubuntu image
ocaisa Feb 4, 2022
0f3f29d
Also update docker container recipes
Feb 4, 2022
2090d5b
Add openssl devel headers
Feb 4, 2022
b9f0b2b
Merge pull request #1 from ocaisa/containers_update
scimerman Feb 4, 2022
affbf2c
Update Docker support
Feb 4, 2022
ae92038
Missed an Ubuntu 16.04 ref
Feb 4, 2022
7846210
Merge pull request #2 from ocaisa/containers_update
scimerman Feb 5, 2022
c20e0b5
Update containers.py
ocaisa Feb 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion easybuild/tools/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@
JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error'
JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run'

DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04'
DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04'
ocaisa marked this conversation as resolved.
Show resolved Hide resolved
DOCKER_BASE_IMAGE_CENTOS = 'centos:7'

LOCAL_VAR_NAMING_CHECK_ERROR = 'error'
Expand Down
20 changes: 13 additions & 7 deletions easybuild/tools/containers/docker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@
"""

DOCKER_INSTALL_EASYBUILD = """\
RUN pip install -U pip setuptools && \\
hash -r pip && \\
pip install -U easybuild
RUN pip3 install -U pip setuptools && \\
hash -r pip3&& \\
pip3 install -U easybuild

RUN mkdir /app && \\
mkdir /scratch && \\
Expand All @@ -72,9 +72,12 @@
CMD ["/bin/bash", "-l"]
"""

DOCKER_UBUNTU1604_INSTALL_DEPS = """\
DOCKER_UBUNTU2004_INSTALL_DEPS = """\
RUN apt-get update && \\
apt-get install -y python python-pip lmod curl wget
apt-get install -y python3 python3-pip lmod curl wget git \\
bzip2 gzip tar zip unzip xz-utils \\
patch automake git debianutils \\
g++ libdata-dump-perl libthread-queue-any-perl libssl-dev

RUN OS_DEPS='%(os_deps)s' && \\
test -n "${OS_DEPS}" && \\
Expand All @@ -83,15 +86,18 @@

DOCKER_CENTOS7_INSTALL_DEPS = """\
RUN yum install -y epel-release && \\
yum install -y python python-pip Lmod curl wget git
yum install -y python3 python3-pip Lmod curl wget git \\
bzip2 gzip tar zip unzip xz \\
patch makefile git which \\
gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev

RUN OS_DEPS='%(os_deps)s' && \\
test -n "${OS_DEPS}" && \\
yum --skip-broken install -y "${OS_DEPS}" || true
"""

DOCKER_OS_INSTALL_DEPS_TMPLS = {
DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS,
DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS,
DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS,
}

Expand Down
14 changes: 7 additions & 7 deletions easybuild/tools/containers/singularity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,9 +272,9 @@ def resolve_template_data(self):
# EPEL is required for installing Lmod & python-pip
'epel-release',
# EasyBuild requirements
'python setuptools Lmod',
# pip is used to install EasyBuild packages
'python-pip',
'python3 setuptools Lmod',
# pip3 is used to install EasyBuild packages
'python3-pip',
# useful utilities
'bzip2 gzip tar zip unzip xz', # extracting sources
'curl wget', # downloading
Expand Down Expand Up @@ -308,13 +308,13 @@ def resolve_template_data(self):
template_data['install_os_deps'] = '\n'.join(install_os_deps)

# install (latest) EasyBuild in container image
# use 'pip install', unless custom commands are specified via 'install_eb' keyword
# use 'pip3 install', unless custom commands are specified via 'install_eb' keyword
if 'install_eb' not in template_data:
template_data['install_eb'] = '\n'.join([
"# install EasyBuild using pip",
"# install EasyBuild using pip3",
# upgrade pip
"pip install -U pip",
"pip install easybuild",
"pip3 install -U pip",
"pip3 install easybuild",
])

# if no custom value is specified for 'post_commands' keyword,
Expand Down
10 changes: 5 additions & 5 deletions test/framework/containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@ def test_end2end_singularity_recipe_config(self):
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

pip_patterns = [
# EasyBuild is installed with pip by default
"pip install easybuild",
# EasyBuild is installed with pip3 by default
"pip3 install easybuild",
]
post_commands_patterns = [
# easybuild user is added if it doesn't exist yet
Expand Down Expand Up @@ -386,7 +386,7 @@ def test_end2end_dockerfile(self):
base_args + ['--container-config=not-supported'],
raise_error=True)

for cont_base in ['ubuntu:16.04', 'centos:7']:
for cont_base in ['ubuntu:20.04', 'centos:7']:
stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base])
self.assertFalse(stderr)
regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix]
Expand All @@ -406,10 +406,10 @@ def test_end2end_dockerfile(self):
remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))

base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb'))
self.run_main(base_args + ['--container-config=ubuntu:16.04'])
self.run_main(base_args + ['--container-config=ubuntu:20.04'])
def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
regexs = [
"FROM ubuntu:16.04",
"FROM ubuntu:20.04",
"eb toy-0.0.eb GCC-4.9.2.eb",
"module load toy/0.0 GCC/4.9.2",
]
Expand Down