diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 6d13eeceff8..9689dbeb5e0 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,5 +1,8 @@ ## Upcoming release +## Release 2.14.0 +* Fix model name preprocess error, pass correct model to `ModelImportEvaluationOp` component in `v1.model_evaluation.evaluation_llm_text_generation_pipeline` and `v1.model_evaluation.evaluation_llm_classification_pipeline`. + ## Release 2.13.0 * Add support for `text-bison@002` to `preview.llm.rlhf_pipeline`. * Apply latest GCPC image vulnerability resolutions (base OS and software updates). diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_classification_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_classification_pipeline.py index d8780844a19..f528003c8f8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_classification_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_classification_pipeline.py @@ -176,9 +176,18 @@ def evaluation_llm_classification_pipeline( # pylint: disable=dangerous-default encryption_spec_key_name=encryption_spec_key_name, ) + get_vertex_eval_model_task = dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_eval_model_task.set_display_name('get-vertex-eval-model') + import_evaluation_task = ModelImportEvaluationOp( classification_metrics=eval_task.outputs['evaluation_metrics'], - model=get_vertex_model_task.outputs['artifact'], + model=get_vertex_eval_model_task.outputs['artifact'], dataset_type=batch_predict_instances_format, dataset_paths=batch_predict_gcs_source_uris, display_name=evaluation_display_name, diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py index 15963b5196b..58a5f89170c 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -34,7 +34,6 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul location: str, batch_predict_gcs_source_uris: List[str], batch_predict_gcs_destination_output_uri: str, - service_account: str, model_name: str = 'publishers/google/models/text-bison@002', evaluation_task: str = 'text-generation', input_field_name: str = 'input_text', @@ -44,6 +43,7 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul batch_predict_model_parameters: Dict[str, str] = {}, enable_row_based_metrics: bool = False, machine_type: str = 'e2-standard-4', + service_account: str = '', network: str = '', encryption_spec_key_name: str = '', evaluation_display_name: str = 'evaluation-llm-text-generation-pipeline-{{$.pipeline_job_uuid}}', @@ -72,7 +72,6 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul "output_text": "your ground truth output text" } batch_predict_gcs_destination_output_uri: Required. The Google Cloud Storage location of the directory where the eval pipeline output is to be written to. - service_account: Required. Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. model_name: The Model name used to run evaluation. Must be a publisher Model or a managed Model sharing the same ancestor location. Starting this job has no impact on any existing deployments of the Model and their resources. evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported tasks are: `summarization`, `question-answering`, `text-generation`. input_field_name: The field name of the input eval dataset instances that contains the input prompts to the LLM. @@ -82,6 +81,7 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul batch_predict_model_parameters: A map of parameters that govern the predictions. Some acceptable parameters include: maxOutputTokens, topK, topP, and temperature. enable_row_based_metrics: Flag of if row based metrics is enabled, default value is false. machine_type: The machine type of this custom job. If not set, defaulted to `e2-standard-4`. More details: https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code Service Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name, as in `myVPC`. To specify this field, you must have already configured VPC Network Peering for Vertex AI (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left unspecified, the job is not peered with any network. encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. @@ -158,11 +158,20 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul encryption_spec_key_name=encryption_spec_key_name, ) + get_vertex_eval_model_task = dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_eval_model_task.set_display_name('get-vertex-eval-model') + with dsl.If(enable_row_based_metrics == True): import_evaluation_task_with_row_based_metrics = ModelImportEvaluationOp( metrics=eval_task.outputs['evaluation_metrics'], row_based_metrics=eval_task.outputs['row_based_metrics'], - model=get_vertex_model_task.outputs['artifact'], + model=get_vertex_eval_model_task.outputs['artifact'], problem_type=evaluation_task, dataset_type=batch_predict_predictions_format, dataset_paths=batch_predict_gcs_source_uris, @@ -171,7 +180,7 @@ def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-defaul with dsl.Else(): import_evaluation_task = ModelImportEvaluationOp( metrics=eval_task.outputs['evaluation_metrics'], - model=get_vertex_model_task.outputs['artifact'], + model=get_vertex_eval_model_task.outputs['artifact'], problem_type=evaluation_task, dataset_type=batch_predict_predictions_format, dataset_paths=batch_predict_gcs_source_uris,