Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(vertexai): Add fields to Vertex AI FeatureStore EntityType for Feature Value Monitoring #12983

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/6699.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
vertexai: add fields to `vertex_ai_featurestore_entitytype` to support feature value monitoring
```
295 changes: 294 additions & 1 deletion google/resource_vertex_ai_featurestore_entitytype.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,66 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"categorical_threshold_config": {
Type: schema.TypeList,
Optional: true,
Description: `Threshold for categorical features of anomaly detection. This is shared by all types of Featurestore Monitoring for categorical features (i.e. Features with type (Feature.ValueType) BOOL or STRING).`,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"value": {
Type: schema.TypeFloat,
Required: true,
Description: `Specify a threshold value that can trigger the alert. For categorical feature, the distribution distance is calculated by L-inifinity norm. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`,
},
},
},
},
"import_features_analysis": {
Type: schema.TypeList,
Optional: true,
Description: `The config for ImportFeatures Analysis Based Feature Monitoring.`,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"anomaly_detection_baseline": {
Type: schema.TypeString,
Optional: true,
Description: `Defines the baseline to do anomaly detection for feature values imported by each [entityTypes.importFeatureValues][] operation. The value must be one of the values below:
* LATEST_STATS: Choose the later one statistics generated by either most recent snapshot analysis or previous import features analysis. If non of them exists, skip anomaly detection and only generate a statistics.
* MOST_RECENT_SNAPSHOT_STATS: Use the statistics generated by the most recent snapshot analysis if exists.
* PREVIOUS_IMPORT_FEATURES_STATS: Use the statistics generated by the previous import features analysis if exists.`,
},
"state": {
Type: schema.TypeString,
Optional: true,
Description: `Whether to enable / disable / inherite default hebavior for import features analysis. The value must be one of the values below:
* DEFAULT: The default behavior of whether to enable the monitoring. EntityType-level config: disabled.
* ENABLED: Explicitly enables import features analysis. EntityType-level config: by default enables import features analysis for all Features under it.
* DISABLED: Explicitly disables import features analysis. EntityType-level config: by default disables import features analysis for all Features under it.`,
},
},
},
},
"numerical_threshold_config": {
Type: schema.TypeList,
Optional: true,
Description: `Threshold for numerical features of anomaly detection. This is shared by all objectives of Featurestore Monitoring for numerical features (i.e. Features with type (Feature.ValueType) DOUBLE or INT64).`,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"value": {
Type: schema.TypeFloat,
Required: true,
Description: `Specify a threshold value that can trigger the alert. For numerical feature, the distribution distance is calculated by Jensen–Shannon divergence. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`,
},
},
},
},
"snapshot_analysis": {
Type: schema.TypeList,
Optional: true,
Description: `Configuration of how features in Featurestore are monitored.`,
Description: `The config for Snapshot Analysis Based Feature Monitoring.`,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
Expand All @@ -77,6 +133,19 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe
Description: `The monitoring schedule for snapshot analysis. For EntityType-level config: unset / disabled = true indicates disabled by default for Features under it; otherwise by default enable snapshot analysis monitoring with monitoringInterval for Features under it.`,
Default: false,
},
"monitoring_interval_days": {
Type: schema.TypeInt,
Optional: true,
Description: `Configuration of the snapshot analysis based monitoring pipeline running interval. The value indicates number of days. The default value is 1.
If both FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days and [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval][] are set when creating/updating EntityTypes/Features, FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days will be used.`,
Default: 1,
},
"staleness_days": {
Type: schema.TypeInt,
Optional: true,
Description: `Customized export features time window for snapshot analysis. Unit is one day. The default value is 21 days. Minimum value is 1 day. Maximum value is 4000 days.`,
Default: 21,
},
},
},
},
Expand Down Expand Up @@ -403,6 +472,12 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d *sch
transformed := make(map[string]interface{})
transformed["snapshot_analysis"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(original["snapshotAnalysis"], d, config)
transformed["import_features_analysis"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["importFeaturesAnalysis"], d, config)
transformed["numerical_threshold_config"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numericalThresholdConfig"], d, config)
transformed["categorical_threshold_config"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categoricalThresholdConfig"], d, config)
return []interface{}{transformed}
}
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} {
Expand All @@ -416,12 +491,107 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v int
transformed := make(map[string]interface{})
transformed["disabled"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(original["disabled"], d, config)
transformed["monitoring_interval_days"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoringIntervalDays"], d, config)
transformed["staleness_days"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["stalenessDays"], d, config)
return []interface{}{transformed}
}
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(v interface{}, d *schema.ResourceData, config *Config) interface{} {
return v
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d *schema.ResourceData, config *Config) interface{} {
// Handles the string fixed64 format
if strVal, ok := v.(string); ok {
if intVal, err := stringToFixed64(strVal); err == nil {
return intVal
}
}

// number values are represented as float64
if floatVal, ok := v.(float64); ok {
intVal := int(floatVal)
return intVal
}

return v // let terraform core handle it otherwise
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d *schema.ResourceData, config *Config) interface{} {
// Handles the string fixed64 format
if strVal, ok := v.(string); ok {
if intVal, err := stringToFixed64(strVal); err == nil {
return intVal
}
}

// number values are represented as float64
if floatVal, ok := v.(float64); ok {
intVal := int(floatVal)
return intVal
}

return v // let terraform core handle it otherwise
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} {
if v == nil {
return nil
}
original := v.(map[string]interface{})
if len(original) == 0 {
return nil
}
transformed := make(map[string]interface{})
transformed["state"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config)
transformed["anomaly_detection_baseline"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomalyDetectionBaseline"], d, config)
return []interface{}{transformed}
}
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d *schema.ResourceData, config *Config) interface{} {
return v
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d *schema.ResourceData, config *Config) interface{} {
return v
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
if v == nil {
return nil
}
original := v.(map[string]interface{})
if len(original) == 0 {
return nil
}
transformed := make(map[string]interface{})
transformed["value"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config)
return []interface{}{transformed}
}
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} {
return v
}

func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
if v == nil {
return nil
}
original := v.(map[string]interface{})
if len(original) == 0 {
return nil
}
transformed := make(map[string]interface{})
transformed["value"] =
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config)
return []interface{}{transformed}
}
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} {
return v
}

func expandVertexAIFeaturestoreEntitytypeLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
if v == nil {
return map[string]string{}, nil
Expand Down Expand Up @@ -449,6 +619,27 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d Terra
transformed["snapshotAnalysis"] = transformedSnapshotAnalysis
}

transformedImportFeaturesAnalysis, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["import_features_analysis"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedImportFeaturesAnalysis); val.IsValid() && !isEmptyValue(val) {
transformed["importFeaturesAnalysis"] = transformedImportFeaturesAnalysis
}

transformedNumericalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numerical_threshold_config"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedNumericalThresholdConfig); val.IsValid() && !isEmptyValue(val) {
transformed["numericalThresholdConfig"] = transformedNumericalThresholdConfig
}

transformedCategoricalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categorical_threshold_config"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedCategoricalThresholdConfig); val.IsValid() && !isEmptyValue(val) {
transformed["categoricalThresholdConfig"] = transformedCategoricalThresholdConfig
}

return transformed, nil
}

Expand All @@ -468,13 +659,115 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v inte
transformed["disabled"] = transformedDisabled
}

transformedMonitoringIntervalDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoring_interval_days"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedMonitoringIntervalDays); val.IsValid() && !isEmptyValue(val) {
transformed["monitoringIntervalDays"] = transformedMonitoringIntervalDays
}

transformedStalenessDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["staleness_days"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedStalenessDays); val.IsValid() && !isEmptyValue(val) {
transformed["stalenessDays"] = transformedStalenessDays
}

return transformed, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
l := v.([]interface{})
if len(l) == 0 || l[0] == nil {
return nil, nil
}
raw := l[0]
original := raw.(map[string]interface{})
transformed := make(map[string]interface{})

transformedState, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedState); val.IsValid() && !isEmptyValue(val) {
transformed["state"] = transformedState
}

transformedAnomalyDetectionBaseline, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomaly_detection_baseline"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedAnomalyDetectionBaseline); val.IsValid() && !isEmptyValue(val) {
transformed["anomalyDetectionBaseline"] = transformedAnomalyDetectionBaseline
}

return transformed, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
l := v.([]interface{})
if len(l) == 0 || l[0] == nil {
return nil, nil
}
raw := l[0]
original := raw.(map[string]interface{})
transformed := make(map[string]interface{})

transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) {
transformed["value"] = transformedValue
}

return transformed, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
l := v.([]interface{})
if len(l) == 0 || l[0] == nil {
return nil, nil
}
raw := l[0]
original := raw.(map[string]interface{})
transformed := make(map[string]interface{})

transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config)
if err != nil {
return nil, err
} else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) {
transformed["value"] = transformedValue
}

return transformed, nil
}

func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
return v, nil
}

func resourceVertexAIFeaturestoreEntitytypeEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) {
if v, ok := d.GetOk("featurestore"); ok {
re := regexp.MustCompile("projects/(.+)/locations/(.+)/featurestores/(.+)$")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,23 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" {
foo = "bar"
}
featurestore = google_vertex_ai_featurestore.featurestore.id
monitoring_config {
snapshot_analysis {
disabled = false
monitoring_interval_days = 1
staleness_days = 21
}
numerical_threshold_config {
value = 0.8
}
categorical_threshold_config {
value = 10.0
}
import_features_analysis {
state = "ENABLED"
anomaly_detection_baseline = "PREVIOUS_IMPORT_FEATURES_STATS"
}
}
}
`, context)
}
Expand Down
Loading