Updates for v1.2.0 release

Signed-off-by: Abolfazl Shahbazi <12436063+ashahba@users.noreply.github.com>

CODEOWNERS

@@ -5,7 +5,7 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-*       @ashahba @daniel-de-leon-user293 @mitalipo
+*       @ashahba @daniel-de-leon-user293 @mitalipo @qgao007
 
 # Order is important; the last matching pattern takes the most
 # precedence. When someone opens a pull request that only

DATASETS.md

@@ -4,12 +4,14 @@ This is a comprehensive list of public datasets used by this repository.
 
 | Name (Link/Source) | Framework | Use Case |
 |--------------------| --------- | -------- |
-| [Adult Income Dataset ](https://archive.ics.uci.edu/ml/machine-learning-databases/adult/) | PyTorch | Tabular Classification |
+| [Adult Income Dataset ](https://archive.ics.uci.edu/dataset/2/adult) | PyTorch | Tabular Classification |
 | [CDD-CESM](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=109379611) | PyTorch | Image & Text Classification |
 | [CIFAR-10 (TorchVision)](https://pytorch.org/vision/main/generated/torchvision.datasets.CIFAR10.html) | Tensorflow | Text Classification |
 | [Civil Comments (TFDS)](https://www.tensorflow.org/datasets/catalog/civil_comments) | Tensorflow | Text Classification |
 | [COMPAS Recidivism Risk Score Data and Analysis](https://github.com/propublica/compas-analysis/) | TensorFlow | Tabular Classification |
 | [ImageNet (TorchVision)](https://pytorch.org/vision/main/generated/torchvision.datasets.ImageNet.html) | PyTorch | Image Classification |
 | [IMDB Reviews](https://ai.stanford.edu/~amaas/data/sentiment/) | PyTorch | Text Classification |
 | [MNIST (TorchVision)](https://pytorch.org/vision/main/generated/torchvision.datasets.MNIST.html) | PyTorch | Image Classification |
-| [SMS Spam Collection](https://archive.ics.uci.edu/ml/machine-learning-databases/00228/) | PyTorch | Text Classification |
+| [SMS Spam Collection](https://archive.ics.uci.edu/dataset/228/sms+spam+collection) | PyTorch | Text Classification |
+| [ToxicChat](https://huggingface.co/datasets/lmsys/toxic-chat) | PyTorch | Toxicity Model Benchmarking |
+| [Jigsaw Unintended Bias](https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification) | PyTorch | Toxicity Model Benchmarking |

MODELS.md

@@ -0,0 +1,7 @@
+# Models
+
+This is a comprehensive list of public models used by this repository.
+
+| Model Name (Link/Source) | Framework | Model Hub |
+|--------------------| --------- | -------- |
+| [ toxic-prompt-roberta ](https://huggingface.co/Intel/toxic-prompt-roberta) | PyTorch | Hugging Face |

Makefile

@@ -26,6 +26,7 @@ ACTIVATE_NOTEBOOK = $(ACTIVATE_TEST)
 LISTEN_IP ?= 127.0.0.1
 LISTEN_PORT ?= 9090
 DOCS_DIR ?= docs
+BENCHMARK_DIR = "plugins/benchmark/"
 
 venv-test: poetry-lock
 	@echo "Creating a virtualenv $(VENV_DIR)..."
@@ -46,6 +47,11 @@ venv-lint:
 		flake8==7.0.0 \
 		black==24.4.2
 
+venv-test-benchmark:
+	@echo "Creating a virtualenv $(VENV_DIR) for benchmark testing..."
+	@test -d $(VENV_DIR) || python -m virtualenv $(VENV_DIR) || python3 -m virtualenv $(VENV_DIR)
+	@. $(ACTIVATE_TEST) && pip install --no-cache-dir --no-deps -r ${BENCHMARK_DIR}/classification_metrics/requirements.txt
+
 test-mcg: venv-test
 	@echo "Testing the Model Card Gen API..."
 	@. $(ACTIVATE_TEST) && pytest model_card_gen/tests
@@ -68,7 +74,11 @@ test-explainer: venv-test
 	@. $(ACTIVATE_TEST) && pytest plugins/explainers/cam-pytorch/tests
 	@. $(ACTIVATE_TEST) && pytest plugins/explainers/metrics/tests
 
-test: test-mcg test-explainer
+test-benchmark: venv-test-benchmark
+	@echo "Testing Benchmarking..."
+	@. $(ACTIVATE_TEST) && pytest plugins/benchmark/classification_metrics/tests
+
+test: test-mcg test-explainer test-benchmark
 
 venv-docs: venv-test ${DOCS_DIR}/requirements-docs.txt
 	@echo "Installing docs dependencies..."

README.md

@@ -139,6 +139,6 @@ Intel is committed to the respect of human rights and avoiding complicity in hum
 Intel® Explainable AI Tools is licensed under Apache License Version 2.0.
 
 #### Datasets and Models
-To the extent that any data, datasets, or models are referenced by Intel or accessed using tools or code on this site such data, datasets and models are provided by the third party indicated as the source of such content. Intel does not create the data, datasets, or models, provide a license to any third-party data, datasets, or models referenced, and does not warrant their accuracy or quality. By accessing such data, dataset(s) or model(s) you agree to the terms associated with that content and that your use complies with the applicable license. [DATASETS](DATASETS.md)
+To the extent that any data, datasets, or models are referenced by Intel or accessed using tools or code on this site such data, datasets and models are provided by the third party indicated as the source of such content. Intel does not create the data, datasets, or models, provide a license to any third-party data, datasets, or models referenced, and does not warrant their accuracy or quality. By accessing such data, dataset(s) or model(s) you agree to the terms associated with that content and that your use complies with the applicable license. [DATASETS](DATASETS.md), [MODELS](MODELS.md)
 
 Intel expressly disclaims the accuracy, adequacy, or completeness of any data, datasets or models, and is not liable for any errors, omissions, or defects in such content, or for any reliance thereon. Intel also expressly disclaims any warranty of non-infringement with respect to such data, dataset(s), or model(s). Intel is not liable for any liability or damages relating to your use of such data, datasets, or models.

docker/Dockerfile.mcg

@@ -113,4 +113,4 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 EXPOSE 8501
 
-CMD ["streamlit", "run", "home.py", "--server.port=8501", "--server.address=0.0.0.0"]
+CMD ["streamlit", "run", "home.py", "--server.port=8501", "--server.address=0.0.0.0"]

docker/README.md

@@ -21,7 +21,7 @@ docker compose build explainers
 ### Check existing image:
 ```bash
 docker images | grep -i explainers
-intel/ai-tools                                      intel-ai-safety-1.1.0-explainers           d8219a0cf128   About an hour ago    3.23GB
+intel/ai-tools                                      intel-ai-safety-1.2.0-explainers    9842919d01fd   1 minute ago    3.23GB
 ```
 
 ## Model Card Generator
@@ -34,7 +34,7 @@ docker compose build model_card_gen
 ### Check existing image
 ```bash
 docker images | grep -i mcg
-intel/ai-tools                                      intel-ai-safety-1.1.0-mcg                  82bdf7b239cc   About a minute ago   2.71GB
+intel/ai-tools                                      intel-ai-safety-1.2.0-mcg           5272f96fc69c   2 minutes ago   2.7GB
 ```
 ## Model Card Generator UI
 
@@ -46,13 +46,13 @@ docker compose build model_card_gen_ui
 ### Check existing image
 ```bash
 docker images | grep -i mcg-ui
-intel/ai-tools                                      intel-ai-safety-1.1.0-mcg-ui        e9bd59328e37   About a minute ago    2.71GB
+intel/ai-tools                                      intel-ai-safety-1.2.0-mcg-ui        9ef60e82aea1   15 minutes ago   2.7GB
 ```
 
 ### Running the UI
 To run the Model Card Generator UI, you can use the docker run command. 
 ```bash
-docker run --rm -p 8051:8051 --name mcg-ui intel/ai-tools:intel-ai-safety-1.1.0-mcg-ui
+docker run --rm -p 8051:8051 --name mcg-ui intel/ai-tools:intel-ai-safety-1.2.0-mcg-ui
 ```
 Once the container is running, you can access the Model Card Generator UI by navigating to `<HOST_NAME>:8051` in your web browser, where HOST_NAME is the name or IP address of the server that the container is running on.
 
@@ -62,31 +62,31 @@ Docker containers can run in either Interactive or Jupyter mode.
 This mode allows running the container in an interactive shell. This enables the ability to interact with the container's bash shell. Below is the command to start the container in interactive mode:
 
 ```bash
-docker run --rm -it intel/ai-tools:intel-ai-safety-1.1.0-explainers bash
+docker run --rm -it intel/ai-tools:intel-ai-safety-1.2.0-explainers bash
 ```
 or
 ```bash
-docker run --rm -it intel/ai-tools:intel-ai-safety-1.1.0-mcg bash
+docker run --rm -it intel/ai-tools:intel-ai-safety-1.2.0-mcg bash
 ```
 
 ### Jupyter
 This mode launches a jupyterlab notebook server. The command below will start the jupyterlab server which can be accessed from a web browser. Each container includes jupyter kernel to enable conda environment in jupyter notebook. The port for this server is `8888` and is exposed by default when you run the container, but you can change to any port that's open on the host. In this example we are using `8887` for `explainers` container and `8889` for `mcg` container on the host.
 
 ```bash
-docker run --rm -p 8887:8888 --name explainers intel/ai-tools:intel-ai-safety-1.1.0-explainers
+docker run --rm -p 8887:8888 --name explainers intel/ai-tools:intel-ai-safety-1.2.0-explainers
 ```
 or
 ```bash
-docker run --rm -p 8889:8888 --name model-card-gen intel/ai-tools:intel-ai-safety-1.1.0-mcg
+docker run --rm -p 8889:8888 --name model-card-gen intel/ai-tools:intel-ai-safety-1.2.0-mcg
 ```
 
 You can also run these containers in daemon mode:
 ```bash
-docker run --rm -d -p 8887:8888 --name explainers intel/ai-tools:intel-ai-safety-1.1.0-explainers
+docker run --rm -d -p 8887:8888 --name explainers intel/ai-tools:intel-ai-safety-1.2.0-explainers
 ```
 or
 ```bash
-docker run --rm -d -p 8889:8888 --name model-card-gen intel/ai-tools:intel-ai-safety-1.1.0-mcg
+docker run --rm -d -p 8889:8888 --name model-card-gen intel/ai-tools:intel-ai-safety-1.2.0-mcg
 ```
 
 Finally, on your favorite browser, navigate to `<HOST_NAME>:8887` where `HOST_NAME` is the name or IP address of the server that the container is running on. If asked for a token, review the container logs to locate the token for the Jupyter server.

docker/docker-compose.yaml

@@ -18,7 +18,7 @@
 
 services:
   explainers:
-    image: intel/ai-tools:intel-ai-safety-1.1.0-explainers
+    image: intel/ai-tools:intel-ai-safety-1.2.0-explainers
     pull_policy: always
     build:
       dockerfile: docker/Dockerfile.explainers
@@ -46,7 +46,7 @@ services:
       retries: 5
 
   model_card_gen:
-    image: intel/ai-tools:intel-ai-safety-1.1.0-mcg
+    image: intel/ai-tools:intel-ai-safety-1.2.0-mcg
     pull_policy: always
     build:
       dockerfile: docker/Dockerfile.mcg
@@ -74,7 +74,7 @@ services:
       retries: 5
 
   model_card_gen_ui:
-    image: intel/ai-tools:intel-ai-safety-1.1.0-mcg-ui
+    image: intel/ai-tools:intel-ai-safety-1.2.0-mcg-ui
     pull_policy: always
     build:
       dockerfile: docker/Dockerfile.mcg
@@ -99,4 +99,4 @@ services:
       test: ["CMD-SHELL", "curl --fail http://localhost:8501 || exit 1"]
       interval: 1m
       timeout: 5s
-      retries: 5
+      retries: 5

docs/datasets.rst

@@ -1,2 +1,2 @@
 .. include:: ../DATASETS.md
-   :parser: myst_parser.sphinx_
+   :parser: myst_parser.sphinx_

docs/notebooks/ExplainingImageClassification.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/imagenet_with_cam/ExplainingImageClassification.ipynb"
-}
+}

docs/notebooks/Multimodal_Cancer_Detection.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/multimodal_cancer_detection/Multimodal_Cancer_Detection.ipynb"
-}
+}

docs/notebooks/PyTorch_Text_Classifier_fine_tuning_with_Attributions.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/transfer_learning_text_classification/PyTorch_Text_Classifier_fine_tuning_with_Attributions.ipynb"
-}
+}

docs/notebooks/TorchVision_CIFAR_Interpret.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/cifar_with_attributions/TorchVision_CIFAR_Interpret.ipynb"
-}
+}

docs/notebooks/heart_disease.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/heart_disease_with_attributions/heart_disease.ipynb"
-}
+}

docs/notebooks/mnist.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/mnist_with_attributions_and_metrics/mnist.ipynb"
-}
+}

docs/notebooks/partitionexplainer.nblink

@@ -1,3 +1,3 @@
 {
     "path": "../../notebooks/explainer/newsgroups_with_attributions_and_metrics/partitionexplainer.ipynb"
-}
+}

explainer/intel_ai_safety/explainer/base_explainer.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

explainer/intel_ai_safety/explainer/blackbox.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

explainer/intel_ai_safety/explainer/context/agnostic/attributions_explainer.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2022 Intel Corporation
+# Copyright (c) 2024 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

explainer/intel_ai_safety/explainer/utils/graphics/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

explainer/intel_ai_safety/explainer/utils/types.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

explainer/intel_ai_safety/explainer/version.py

@@ -18,4 +18,4 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"

explainer/poetry.toml

@@ -1,2 +1,2 @@
 [virtualenvs]
-in-project = true
+in-project = true

explainer/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "intel-ai-safety-explainer"
-version = "1.1.0"
+version = "1.2.0"
 description = ""
 authors = ["IntelAI <IntelAI@intel.com>"]
 readme = "README.md"

explainer/tests/pytest.ini

@@ -17,4 +17,4 @@ markers =
 #     ignore::ImportWarning
 
 # Temporary solution while we figure out how to filter out shap warning above
-addopts = -p no:warnings
+addopts = -p no:warnings

fuzz/requirements.txt

@@ -6,4 +6,4 @@ pandas==1.5.3
 plotly==5.20.0
 semantic-version==2.10.0
 tensorflow-data-validation==1.14.0
-tensorflow-model-analysis==0.45.0
+tensorflow-model-analysis==0.45.0

intel_ai_safety/common/plugins.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

intel_ai_safety/common/types.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

intel_ai_safety/version.py

@@ -16,4 +16,4 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"

model_card_gen/intel_ai_safety/model_card_gen/__init__.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

model_card_gen/intel_ai_safety/model_card_gen/docs/examples/csv/multiclass_metrics_by_group.csv

@@ -0,0 +1,34 @@
+group,Subgroup,BPSN,BNSP,feature,label
+None,0.663978494623656,0.05175469691598725,0.7976406533575318,target,Hate
+Islam,0.2520812520812521,0.2253081791188027,0.06895346628679963,target,Hate
+Homosexual,0.1665200452431821,0.10495643644336736,0.11497825514795271,target,Hate
+Other,0.06432748538011696,0.036618539443398206,0.1179243103368606,target,Hate
+African,0.17922839813065528,0.2761131918435289,0.04948755257654906,target,Hate
+Caucasian,0.11973392461197341,0.03502420924023711,0.2017069924046668,target,Hate
+Women,0.0915915915915916,0.052774564447716214,0.134178633262043,target,Hate
+Jewish,0.15855072463768116,0.2847087378640777,0.042195048309178744,target,Hate
+Arab,0.19393939393939397,0.20405040504050406,0.07246704331450093,target,Hate
+Refugee,0.21724709784411278,0.050078554595443844,0.26558981889378364,target,Hate
+Overall generalized mean,0.09752439987057404,0.047228721846868965,0.06072769397983537,target,Hate
+None,0.19501844198551305,0.2783760683760684,0.10728397072576173,target,Offensive
+Islam,0.13459950454170108,0.09323060800078603,0.17756539235412475,target,Offensive
+Homosexual,0.15458579881656803,0.1293481839130912,0.14503850579799946,target,Offensive
+Other,0.16952380952380952,0.13625925925925925,0.1718796992481203,target,Offensive
+African,0.2011661807580175,0.06384055038445974,0.4060973226499902,target,Offensive
+Caucasian,0.2236842105263158,0.33749169557835684,0.05688232739904473,target,Offensive
+Women,0.2237339380196523,0.19960429875444044,0.11690237830393559,target,Offensive
+Jewish,0.14197802197802198,0.04984393757503001,0.32289592760180996,target,Offensive
+Arab,0.2738095238095237,0.07928240740740741,0.33589933382679493,target,Offensive
+Refugee,0.17570754716981132,0.23485815121487713,0.08339763374485597,target,Offensive
+Overall generalized mean,0.16778173692851533,0.07335262227529309,0.08644312885987962,target,Offensive
+None,0.14334558310461926,0.2737090148982106,0.11224370922646786,target,Normal
+Islam,0.3830227743271222,0.17038497457715057,0.4388404615577574,target,Normal
+Homosexual,0.31410490446635025,0.3454184704184704,0.18315707652341112,target,Normal
+Other,0.1304945054945055,0.32237929135303833,0.08765844467283318,target,Normal
+African,0.23356751607525597,0.11845338161127635,0.4037692747001713,target,Normal
+Caucasian,0.2518518518518518,0.21071428571428572,0.2659744408945687,target,Normal
+Women,0.28479532163742693,0.33588516746411484,0.1762568306010929,target,Normal
+Jewish,0.2369787005548595,0.14436481296682693,0.3628726417358744,target,Normal
+Arab,0.2205128205128205,0.20050167224080268,0.256,target,Normal
+Refugee,0.3554285714285714,0.35463636363636364,0.2079171032716265,target,Normal
+Overall generalized mean,0.18221056323099738,0.168860094009674,0.13045961001359177,target,Normal