-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
449 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# Some commands | ||
|
||
```bash | ||
cmr "get aria2" --version=1.37.0 | ||
cmr "get aria2" --install | ||
cmr "get aria2" --path={path to the directory with aria2} | ||
cmr "get aria2" --input={full path to aria2} | ||
cmr "get aria2" --shell | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
alias: get-aria2 | ||
uid: d83419a90a0c40d0 | ||
|
||
automation_alias: script | ||
automation_uid: 5b4e0237da074764 | ||
|
||
cache: true | ||
|
||
category: Detection or installation of tools and artifacts | ||
|
||
input_mapping: | ||
install: CM_FORCE_INSTALL | ||
src: CM_ARIA2_BUILD_FROM_SRC | ||
|
||
deps: | ||
- tags: detect,cpu | ||
- tags: detect,os | ||
|
||
#called after preprocess from customize.py | ||
#prehook_deps: | ||
# - tags: print,native,hello-world | ||
|
||
env: | ||
CM_REQUIRE_INSTALL: no | ||
CM_ARIA2_DEFAULT_INSTALL_VERSION: "1.37.0" | ||
|
||
new_env_keys: | ||
- CM_ARIA2_* | ||
- +PATH | ||
|
||
print_env_at_the_end: | ||
CM_ARIA2_INSTALLED_PATH: Path to the tool | ||
|
||
tags: | ||
- get | ||
- aria2 | ||
- get-aria2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,122 @@ | ||
from cmind import utils | ||
import os | ||
|
||
def preprocess(i): | ||
|
||
# Pre-set by CM | ||
os_info = i['os_info'] | ||
env = i['env'] | ||
recursion_spaces = i['recursion_spaces'] | ||
automation = i['automation'] | ||
run_script_input = i['run_script_input'] | ||
|
||
# Check if a given tool is already installed | ||
file_name_core = 'aria2c' | ||
file_name = file_name_core+'.exe' if os_info['platform'] == 'windows' else file_name_core | ||
|
||
force_install = env.get('CM_FORCE_INSTALL', False) == True | ||
|
||
if not force_install: | ||
r = i['automation'].find_artifact({'file_name': file_name, | ||
'env':env, | ||
'os_info':os_info, | ||
'default_path_env_key': 'PATH', | ||
'detect_version':True, | ||
'env_path_key':'CM_ARIA2_BIN_WITH_PATH', | ||
'run_script_input':i['run_script_input'], | ||
'recursion_spaces':recursion_spaces}) | ||
if r['return'] >0 : | ||
if r['return'] == 16: | ||
# Not found, try install | ||
force_install = True | ||
else: | ||
return r | ||
|
||
# Force install | ||
if force_install: | ||
# Attempt to run installer | ||
version = env.get('CM_VERSION','') | ||
if version == '': version = env['CM_ARIA2_DEFAULT_INSTALL_VERSION'] | ||
|
||
if os_info['platform'] == 'windows': | ||
archive = 'aria2-{}-win-64bit-build1' | ||
ext = '.zip' | ||
ext2 = '' | ||
else: | ||
archive = 'aria2-{}' | ||
ext = '.tar.bz2' | ||
ext2 = '.tar' | ||
|
||
archive = archive.format(version) | ||
archive_with_ext = archive+ext | ||
|
||
env['CM_ARIA2_DOWNLOAD_DIR'] = archive | ||
|
||
env['CM_ARIA2_DOWNLOAD_FILE'] = archive_with_ext | ||
if ext2!='': | ||
env['CM_ARIA2_DOWNLOAD_FILE2'] = archive+ext2 | ||
|
||
url = 'https://github.com/aria2/aria2/releases/download/release-{}/{}'.format(version, archive_with_ext) | ||
env['CM_ARIA2_DOWNLOAD_URL'] = url | ||
|
||
print ('URL to download ARIA2: {}'.format(url)) | ||
|
||
r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'}) | ||
if r['return']>0: return r | ||
|
||
if os_info['platform'] == 'windows' or env.get('CM_ARIA2_BUILD_FROM_SRC', '').lower() == 'true': | ||
install_path = os.path.join(os.getcwd(), archive) | ||
|
||
path_to_file = os.path.join(install_path, file_name) | ||
if not os.path.isfile(path_to_file): | ||
return {'return':1, 'error':'file not found: {}'.format(path_to_file)} | ||
|
||
env['CM_ARIA2_BIN_WITH_PATH'] = path_to_file | ||
env['CM_ARIA2_INSTALLED_TO_CACHE'] = 'yes' | ||
else: | ||
path_to_bin = r['env_tmp'].get('CM_ARIA2_BIN_WITH_PATH','') | ||
env['CM_ARIA2_BIN_WITH_PATH'] = path_to_bin | ||
|
||
r = i['automation'].find_artifact({'file_name': file_name, | ||
'env':env, | ||
'os_info':os_info, | ||
'default_path_env_key': 'PATH', | ||
'detect_version':True, | ||
'env_path_key':'CM_ARIA2_BIN_WITH_PATH', | ||
'run_script_input':i['run_script_input'], | ||
'recursion_spaces':recursion_spaces}) | ||
if r['return']>0: return r | ||
|
||
return {'return':0} | ||
|
||
def detect_version(i): | ||
env = i['env'] | ||
|
||
r = i['automation'].parse_version({'match_text': r'aria2 version\s*([\d.]+)', | ||
'group_number': 1, | ||
'env_key':'CM_ARIA2_VERSION', | ||
'which_env':i['env']}) | ||
if r['return'] >0: return r | ||
|
||
version = r['version'] | ||
print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) | ||
|
||
return {'return':0, 'version':version} | ||
|
||
def postprocess(i): | ||
|
||
env = i['env'] | ||
r = detect_version(i) | ||
if r['return'] >0: return r | ||
|
||
version = r['version'] | ||
found_file_path = env['CM_ARIA2_BIN_WITH_PATH'] | ||
|
||
found_path = os.path.dirname(found_file_path) | ||
|
||
env['CM_ARIA2_INSTALLED_PATH'] = found_path | ||
|
||
if env.get('CM_ARIA2_INSTALLED_TO_CACHE','')=='yes': | ||
env['+PATH'] = [env['CM_ARIA2_INSTALLED_PATH']] | ||
|
||
return {'return':0, 'version': version} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
echo. | ||
|
||
del /Q /S %CM_ARIA2_DOWNLOAD_FILE% | ||
|
||
wget --no-check-certificate %CM_ARIA2_DOWNLOAD_URL% | ||
IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% | ||
|
||
unzip -o -q %CM_ARIA2_DOWNLOAD_FILE% | ||
IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
#!/bin/bash | ||
|
||
echo "" | ||
|
||
if [[ "${CM_ARIA2_BUILD_FROM_SRC}" == "True" ]]; then | ||
|
||
echo "Building from sources ..." | ||
echo "" | ||
|
||
rm -rf ${CM_ARIA2_DOWNLOAD_FILE} | ||
rm -rf ${CM_ARIA2_DOWNLOAD_FILE2} | ||
|
||
wget --no-check-certificate ${CM_ARIA2_DOWNLOAD_URL} | ||
test $? -eq 0 || exit $? | ||
|
||
bzip2 -d ${CM_ARIA2_DOWNLOAD_FILE} | ||
test $? -eq 0 || exit $? | ||
|
||
tar xvf ${CM_ARIA2_DOWNLOAD_FILE2} | ||
test $? -eq 0 || exit $? | ||
|
||
cd ${CM_ARIA2_DOWNLOAD_DIR} | ||
test $? -eq 0 || exit $? | ||
|
||
./configure --prefix=$PWD/bin | ||
test $? -eq 0 || exit $? | ||
|
||
make | ||
test $? -eq 0 || exit $? | ||
|
||
make install | ||
test $? -eq 0 || exit $? | ||
|
||
else | ||
echo "Installing binary via sudo ..." | ||
echo "" | ||
|
||
cmd="sudo ${CM_HOST_OS_PACKAGE_MANAGER} install aria2" | ||
echo "$cmd" | ||
|
||
$cmd | ||
test $? -eq 0 || exit $? | ||
|
||
path_to_bin=`which aria2c` | ||
echo "CM_ARIA2_BIN_WITH_PATH=$path_to_bin" > tmp-run-env.out | ||
|
||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
rem Detect version | ||
|
||
%CM_ARIA2_BIN_WITH_PATH% --version > tmp-ver.out | ||
IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
#!/bin/bash | ||
|
||
# Detect version | ||
|
||
${CM_ARIA2_BIN_WITH_PATH} --version > tmp-ver.out | ||
test $? -eq 0 || exit 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,128 @@ | ||
Automatically generated README for this automation recipe: **get-croissant** | ||
|
||
Category: **AI/ML datasets** | ||
|
||
License: **Apache 2.0** | ||
|
||
Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) | ||
|
||
--- | ||
*[ [Online info and GUI to run this CM script](https://access.cknowledge.org/playground/?action=scripts&name=get-croissant,8fd653eac8da4c14) ]* | ||
|
||
--- | ||
#### Summary | ||
|
||
* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/dev/cm-mlops)* | ||
* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant)* | ||
* CM meta description for this script: *[_cm.json](_cm.json)* | ||
* All CM tags to find and reuse this script (see in above meta description): *get,mlcommons,croissant* | ||
* Output cached? *True* | ||
* See [pipeline of dependencies](#dependencies-on-other-cm-scripts) on other CM scripts | ||
|
||
|
||
--- | ||
### Reuse this script in your project | ||
|
||
#### Install MLCommons CM automation meta-framework | ||
|
||
* [Install CM](https://access.cknowledge.org/playground/?action=install) | ||
* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md) | ||
|
||
#### Pull CM repository with this automation recipe (CM script) | ||
|
||
```cm pull repo mlcommons@ck``` | ||
|
||
#### Print CM help from the command line | ||
|
||
````cmr "get mlcommons croissant" --help```` | ||
|
||
#### Customize and run this script from the command line with different variations and flags | ||
|
||
`cm run script --tags=get,mlcommons,croissant` | ||
|
||
`cm run script --tags=get,mlcommons,croissant ` | ||
|
||
*or* | ||
|
||
`cmr "get mlcommons croissant"` | ||
|
||
`cmr "get mlcommons croissant " ` | ||
|
||
|
||
#### Run this script from Python | ||
|
||
<details> | ||
<summary>Click here to expand this section.</summary> | ||
|
||
```python | ||
|
||
import cmind | ||
|
||
r = cmind.access({'action':'run' | ||
'automation':'script', | ||
'tags':'get,mlcommons,croissant' | ||
'out':'con', | ||
... | ||
(other input keys for this script) | ||
... | ||
}) | ||
|
||
if r['return']>0: | ||
print (r['error']) | ||
|
||
``` | ||
|
||
</details> | ||
|
||
|
||
#### Run this script via GUI | ||
|
||
```cmr "cm gui" --script="get,mlcommons,croissant"``` | ||
|
||
Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,mlcommons,croissant) to generate CM CMD. | ||
|
||
#### Run this script via Docker (beta) | ||
|
||
`cm docker script "get mlcommons croissant" ` | ||
|
||
___ | ||
### Customization | ||
|
||
#### Default environment | ||
|
||
<details> | ||
<summary>Click here to expand this section.</summary> | ||
|
||
These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. | ||
|
||
|
||
</details> | ||
|
||
___ | ||
### Dependencies on other CM scripts | ||
|
||
|
||
1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json)*** | ||
* detect,os | ||
- CM script: [detect-os](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os) | ||
* get,python3 | ||
* CM names: `--adr.['python3', 'python']...` | ||
- CM script: [get-python3](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-python3) | ||
* git,repo,_repo.https://github.com/mlcommons/croissant | ||
* CM names: `--adr.['git-mlcommons-croissant']...` | ||
- CM script: [get-git-repo](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-git-repo) | ||
1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/customize.py)*** | ||
1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json) | ||
1. ***Run native script if exists*** | ||
* [run.bat](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/run.bat) | ||
* [run.sh](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/run.sh) | ||
1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json) | ||
1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/customize.py)*** | ||
1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json) | ||
|
||
___ | ||
### Script output | ||
`cmr "get mlcommons croissant " -j` | ||
#### New environment keys (filter) | ||
|
||
#### New environment keys auto-detected from customize |
Oops, something went wrong.