feat: xai samples (#83)

samples/snippets/explain_tabular_sample.py

@@ -0,0 +1,68 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START aiplatform_explain_tabular_sample]
+from typing import Dict
+
+from google.cloud import aiplatform
+from google.protobuf import json_format
+from google.protobuf.struct_pb2 import Value
+
+
+def explain_tabular_sample(
+    project: str,
+    endpoint_id: str,
+    instance_dict: Dict,
+    location: str = "us-central1",
+    api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com",
+):
+    client_options = {"api_endpoint": api_endpoint}
+    # Initialize client that will be used to create and send requests.
+    # This client only needs to be created once, and can be reused for multiple requests.
+    client = aiplatform.gapic.PredictionServiceClient(client_options=client_options)
+    # The format of each instance should conform to the deployed model's prediction input schema.
+    instance = json_format.ParseDict(instance_dict, Value())
+    instances = [instance]
+    # tabular models do not have additional parameters
+    parameters_dict = {}
+    parameters = json_format.ParseDict(parameters_dict, Value())
+    endpoint = client.endpoint_path(
+        project=project, location=location, endpoint=endpoint_id
+    )
+    response = client.explain(
+        endpoint=endpoint, instances=instances, parameters=parameters
+    )
+    print("response")
+    print(" deployed_model_id:", response.deployed_model_id)
+    explanations = response.explanations
+    for explanation in explanations:
+        print(" explanation")
+        # Feature attributions.
+        attributions = explanation.attributions
+        for attribution in attributions:
+            print("  attribution")
+            print("   baseline_output_value:", attribution.baseline_output_value)
+            print("   instance_output_value:", attribution.instance_output_value)
+            print("   output_display_name:", attribution.output_display_name)
+            print("   approximation_error:", attribution.approximation_error)
+            print("   output_name:", attribution.output_name)
+            output_index = attribution.output_index
+            for output_index in output_index:
+                print("   output_index:", output_index)
+    predictions = response.predictions
+    for prediction in predictions:
+        print(" prediction:", dict(prediction))
+
+
+# [END aiplatform_explain_tabular_sample]

samples/snippets/explain_tabular_sample_test.py

@@ -0,0 +1,37 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import explain_tabular_sample
+
+ENDPOINT_ID = "4966625964059525120"  # iris 1000
+PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT")
+
+INSTANCE = {
+    "petal_length": "1.4",
+    "petal_width": "1.3",
+    "sepal_length": "5.1",
+    "sepal_width": "2.8",
+}
+
+
+def test_ucaip_generated_explain_tabular_sample(capsys):
+
+    explain_tabular_sample.explain_tabular_sample(
+        instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID
+    )
+
+    out, _ = capsys.readouterr()
+    assert 'attribution' in out

samples/snippets/upload_model_explain_image_managed_container_sample.py

@@ -0,0 +1,83 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START aiplatform_upload_model_explain_image_managed_container_sample]
+from google.cloud import aiplatform
+
+
+def upload_model_explain_image_managed_container_sample(
+    project: str,
+    display_name: str,
+    container_spec_image_uri: str,
+    artifact_uri: str,
+    input_tensor_name: str,
+    output_tensor_name: str,
+    location: str = "us-central1",
+    api_endpoint: str = "us-central1-aiplatform.googleapis.com",
+    timeout: int = 300,
+):
+    client_options = {"api_endpoint": api_endpoint}
+    # Initialize client that will be used to create and send requests.
+    # This client only needs to be created once, and can be reused for multiple requests.
+    client = aiplatform.gapic.ModelServiceClient(client_options=client_options)
+
+    # Container specification for deploying the model
+    container_spec = {"image_uri": container_spec_image_uri, "command": [], "args": []}
+
+    # The explainabilty method and corresponding parameters
+    parameters = aiplatform.gapic.ExplanationParameters(
+        {"xrai_attribution": {"step_count": 1}}
+    )
+
+    # The input tensor for feature attribution to the output
+    # For single input model, y = f(x), this will be the serving input layer.
+    input_metadata = aiplatform.gapic.ExplanationMetadata.InputMetadata(
+        {
+            "input_tensor_name": input_tensor_name,
+            # Input is image data
+            "modality": "image",
+        }
+    )
+
+    # The output tensor to explain
+    # For single output model, y = f(x), this will be the serving output layer.
+    output_metadata = aiplatform.gapic.ExplanationMetadata.OutputMetadata(
+        {"output_tensor_name": output_tensor_name}
+    )
+
+    # Assemble the explanation metadata
+    metadata = aiplatform.gapic.ExplanationMetadata(
+        inputs={"image": input_metadata}, outputs={"prediction": output_metadata}
+    )
+
+    # Assemble the explanation specification
+    explanation_spec = aiplatform.gapic.ExplanationSpec(
+        parameters=parameters, metadata=metadata
+    )
+
+    model = aiplatform.gapic.Model(
+        display_name=display_name,
+        # The Cloud Storage location of the custom model
+        artifact_uri=artifact_uri,
+        explanation_spec=explanation_spec,
+        container_spec=container_spec,
+    )
+    parent = f"projects/{project}/locations/{location}"
+    response = client.upload_model(parent=parent, model=model)
+    print("Long running operation:", response.operation.name)
+    upload_model_response = response.result(timeout=timeout)
+    print("upload_model_response:", upload_model_response)
+
+
+# [END aiplatform_upload_model_explain_image_managed_container_sample]

samples/snippets/upload_model_explain_image_managed_container_sample_test.py

@@ -0,0 +1,52 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from uuid import uuid4
+
+import pytest
+
+import helpers
+
+import upload_model_explain_image_managed_container_sample
+
+PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT")
+IMAGE_URI = "gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-1:latest"
+ARTIFACT_URI = "gs://ucaip-samples-us-central1/model/cifar"
+DISPLAY_NAME = f"temp_upload_model_explain_image_managed_container_sample_{uuid4()}"
+
+INPUT_TENSOR_NAME = "bytes_inputs"
+OUTPUT_TENSOR_NAME = "output_0"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def teardown(teardown_model):
+    yield
+
+
+def test_ucaip_generated_upload_model_explain_image_managed_container_sample(capsys, shared_state):
+
+    upload_model_explain_image_managed_container_sample.upload_model_explain_image_managed_container_sample(
+        display_name=DISPLAY_NAME,
+        artifact_uri=ARTIFACT_URI,
+        container_spec_image_uri=IMAGE_URI,
+        project=PROJECT_ID,
+        input_tensor_name=INPUT_TENSOR_NAME,
+        output_tensor_name=OUTPUT_TENSOR_NAME
+    )
+
+    out, _ = capsys.readouterr()
+
+    shared_state["model_name"] = helpers.get_name(out, key="model")

samples/snippets/upload_model_explain_tabular_managed_container_sample.py

@@ -0,0 +1,87 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START aiplatform_upload_model_explain_tabular_managed_container_sample]
+from google.cloud import aiplatform
+
+
+def upload_model_explain_tabular_managed_container_sample(
+    project: str,
+    display_name: str,
+    container_spec_image_uri: str,
+    artifact_uri: str,
+    input_tensor_name: str,
+    output_tensor_name: str,
+    feature_names: list,
+    location: str = "us-central1",
+    api_endpoint: str = "us-central1-aiplatform.googleapis.com",
+    timeout: int = 300,
+):
+    client_options = {"api_endpoint": api_endpoint}
+    # Initialize client that will be used to create and send requests.
+    # This client only needs to be created once, and can be reused for multiple requests.
+    client = aiplatform.gapic.ModelServiceClient(client_options=client_options)
+
+    # Container specification for deploying the model
+    container_spec = {"image_uri": container_spec_image_uri, "command": [], "args": []}
+
+    # The explainabilty method and corresponding parameters
+    parameters = aiplatform.gapic.ExplanationParameters(
+        {"xrai_attribution": {"step_count": 1}}
+    )
+
+    # The input tensor for feature attribution to the output
+    # For single input model, y = f(x), this will be the serving input layer.
+    input_metadata = aiplatform.gapic.ExplanationMetadata.InputMetadata(
+        {
+            "input_tensor_name": input_tensor_name,
+            # Input is tabular data
+            "modality": "numeric",
+            # Assign feature names to the inputs for explanation
+            "encoding": "BAG_OF_FEATURES",
+            "index_feature_mapping": feature_names,
+        }
+    )
+
+    # The output tensor to explain
+    # For single output model, y = f(x), this will be the serving output layer.
+    output_metadata = aiplatform.gapic.ExplanationMetadata.OutputMetadata(
+        {"output_tensor_name": output_tensor_name}
+    )
+
+    # Assemble the explanation metadata
+    metadata = aiplatform.gapic.ExplanationMetadata(
+        inputs={"features": input_metadata}, outputs={"prediction": output_metadata}
+    )
+
+    # Assemble the explanation specification
+    explanation_spec = aiplatform.gapic.ExplanationSpec(
+        parameters=parameters, metadata=metadata
+    )
+
+    model = aiplatform.gapic.Model(
+        display_name=display_name,
+        # The Cloud Storage location of the custom model
+        artifact_uri=artifact_uri,
+        explanation_spec=explanation_spec,
+        container_spec=container_spec,
+    )
+    parent = f"projects/{project}/locations/{location}"
+    response = client.upload_model(parent=parent, model=model)
+    print("Long running operation:", response.operation.name)
+    upload_model_response = response.result(timeout=timeout)
+    print("upload_model_response:", upload_model_response)
+
+
+# [END aiplatform_upload_model_explain_tabular_managed_container_sample]

samples/snippets/upload_model_explain_tabular_managed_container_sample_test.py

@@ -0,0 +1,54 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from uuid import uuid4
+
+import pytest
+
+import helpers
+
+import upload_model_explain_tabular_managed_container_sample
+
+PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT")
+IMAGE_URI = "gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-1:latest"
+ARTIFACT_URI = "gs://ucaip-samples-us-central1/model/boston_housing/"
+DISPLAY_NAME = f"temp_upload_model_test_{uuid4()}"
+
+INPUT_TENSOR_NAME = "dense_input"
+OUTPUT_TENSOR_NAME = "dense_2"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def teardown(teardown_model):
+    yield
+
+
+def test_ucaip_generated_upload_model_explain_tabular_managed_constainer_sample(capsys, shared_state):
+
+    upload_model_explain_tabular_managed_container_sample.upload_model_explain_tabular_managed_container_sample(
+        display_name=DISPLAY_NAME,
+        artifact_uri=ARTIFACT_URI,
+        container_spec_image_uri=IMAGE_URI,
+        project=PROJECT_ID,
+        input_tensor_name=INPUT_TENSOR_NAME,
+        output_tensor_name=OUTPUT_TENSOR_NAME,
+        feature_names=["crim", "zn", "indus", "chas", "nox", "rm", "age",
+                       "dis", "rad", "tax", "ptratio", "b", "lstat"]
+    )
+
+    out, _ = capsys.readouterr()
+
+    shared_state["model_name"] = helpers.get_name(out, key="model")