From 9a0afef4d8226bfae4b6e4bc411a5e04b09dc133 Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 10:47:17 +0200 Subject: [PATCH 1/6] add gradio to install for template --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index ce71525e1..27a7ddc3b 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,8 @@ TEMPLATE_REQUIRE = [ # to populate metric template "cookiecutter" + # for the gradio widget + "gradio>=3.0.0" ] EVALUATOR_REQUIRE = [ From c80ec527dc8d726b5d652ce604cc30591d9282fe Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 11:12:28 +0200 Subject: [PATCH 2/6] add installation instructions --- docs/source/creating_and_sharing.mdx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/creating_and_sharing.mdx b/docs/source/creating_and_sharing.mdx index a15d0445b..1f7023308 100644 --- a/docs/source/creating_and_sharing.mdx +++ b/docs/source/creating_and_sharing.mdx @@ -1,5 +1,21 @@ # Creating and sharing a new evaluation +## Setup + +Before you can create a new metric make sure you have all the necessary dependencies installed: + +```bash +pip install evaluate[template] +``` + +Also make sure your Hugging Face token is registered so you can connect to the Hugging Face Hub: + +```bash +huggingface-cli login +``` + +## Create + All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). In principle, you could setup a new Space and add a new module following the same structure. However, we added a CLI that makes creating a new evaluation module much easier: ```bash From 9d33523c413c7b0f44774c4b2d0d5705a7178ab8 Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 11:12:40 +0200 Subject: [PATCH 3/6] fix readme formatting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8954b04dc..d8963ea4d 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,11 @@ pip install evaluate # Adding a new evaluation module First install the necessary dependencies to create a new metric with the following command: -``` +```bash pip install evaluate[template] ``` Then you can get started with the following command which will create a new folder for your metric and display the necessary steps: -```batch +```bash evaluate-cli create "Awesome Metric" ``` See this [step-by-step guide](https://huggingface.co/docs/evaluate/creating_and_sharing) in the documentation for detailed instructions. From 001c6a2f8809929c8f08743ac2d23487c99ef7a5 Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 11:13:02 +0200 Subject: [PATCH 4/6] add a helpful message when space already exist --- src/evaluate/commands/evaluate_cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/evaluate/commands/evaluate_cli.py b/src/evaluate/commands/evaluate_cli.py index fd1406b86..abbad84df 100644 --- a/src/evaluate/commands/evaluate_cli.py +++ b/src/evaluate/commands/evaluate_cli.py @@ -5,7 +5,9 @@ from cookiecutter.main import cookiecutter from huggingface_hub import HfApi, Repository, create_repo +from evaluate.utils.logging import get_logger +logger = get_logger(__name__) INSTRUCTIONS = """\ A new repository for your module "{module_name}" of type "{module_type}" has been created at {output_dir} and pushed to the Hugging Face Hub: {repo_url}. @@ -83,7 +85,11 @@ def main(): args["namespace"] = namespace repo_url = f"https://huggingface.co/spaces/{namespace}/{module_slug}" - create_repo(namespace + "/" + module_slug, repo_type="space", space_sdk="gradio", private=args["private"]) + try: + create_repo(namespace + "/" + module_slug, repo_type="space", space_sdk="gradio", private=args["private"]) + except Exception as exception: + logger.error(f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already.") + raise exception subprocess.run( f"git clone {repo_url}".split(), stderr=subprocess.PIPE, From 782dc8783d3e810147f15de2671720536f6e81d5 Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 11:13:12 +0200 Subject: [PATCH 5/6] fix cli command --- docs/source/creating_and_sharing.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/creating_and_sharing.mdx b/docs/source/creating_and_sharing.mdx index 1f7023308..c1b96d3fc 100644 --- a/docs/source/creating_and_sharing.mdx +++ b/docs/source/creating_and_sharing.mdx @@ -19,12 +19,13 @@ huggingface-cli login All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). In principle, you could setup a new Space and add a new module following the same structure. However, we added a CLI that makes creating a new evaluation module much easier: ```bash -evaluate-cli create "My Metric" module_type="metric" +evaluate-cli create "My Metric" --module_type "metric" ``` This will create a new Space on the 🤗 Hub, clone it locally, and populate it with a template. Instructions on how to fill the template will be displayed in the terminal, but are also explained here in more detail. For more information about Spaces, see the [Spaces documentation](https://huggingface.co/docs/hub/spaces). + ## Module script The evaluation module script (the file with suffix `*.py`) is the core of the new module and includes all the code for computing the evaluation. From 0169a711617b5ca6f664b086738dda77fa52708e Mon Sep 17 00:00:00 2001 From: leandro Date: Thu, 28 Jul 2022 11:18:51 +0200 Subject: [PATCH 6/6] fix style --- src/evaluate/commands/evaluate_cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/evaluate/commands/evaluate_cli.py b/src/evaluate/commands/evaluate_cli.py index abbad84df..80593c4df 100644 --- a/src/evaluate/commands/evaluate_cli.py +++ b/src/evaluate/commands/evaluate_cli.py @@ -5,8 +5,10 @@ from cookiecutter.main import cookiecutter from huggingface_hub import HfApi, Repository, create_repo + from evaluate.utils.logging import get_logger + logger = get_logger(__name__) INSTRUCTIONS = """\ @@ -88,7 +90,9 @@ def main(): try: create_repo(namespace + "/" + module_slug, repo_type="space", space_sdk="gradio", private=args["private"]) except Exception as exception: - logger.error(f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already.") + logger.error( + f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already." + ) raise exception subprocess.run( f"git clone {repo_url}".split(),