Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

User metadata on workflows/events for user interface usage #371

Merged
merged 9 commits into from
May 17, 2024
Merged

User metadata on workflows/events for user interface usage #371

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
db555cc
User metadata on workflows/events for user interface usage
cretz Mar 18, 2024
2da3c29
Line ending fixes
cretz Mar 18, 2024
d5b9376
PR suggested fixes
cretz Mar 18, 2024
3839519
PR-suggested comments
cretz Mar 19, 2024
cb07811
Move metadata from execution info to execution config
cretz Apr 3, 2024
3d93adc
Merge remote-tracking branch 'remotes/origin/master' into user-metadata
cretz Apr 3, 2024
c89cf6e
Merge remote-tracking branch 'remotes/origin/master' into user-metadata
cretz Apr 29, 2024
ae77172
Merge branch 'master' into user-metadata
cretz May 17, 2024
1e8c2d4
Regen OpenAPI specs
cretz May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions buf.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ breaking:
- temporal/api/operatorservice/v1/request_response.proto
- temporal/api/operatorservice/v1/service.proto
- temporal/api/nexus/v1/message.proto
# Remove this after user metadata is merged
- temporal/api/sdk/v1/workflow_metadata.proto
lint:
use:
- DEFAULT
Expand Down
38 changes: 38 additions & 0 deletions openapi/openapiv2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2477,6 +2477,10 @@
"skipGenerateWorkflowTask": {
"type": "boolean",
"description": "Indicates that a new workflow task should not be generated when this signal is received."
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo\nfor use by user interfaces to display the fixed as-of-start summary and details of the\nworkflow."
}
}
},
Expand Down Expand Up @@ -2639,6 +2643,10 @@
"$ref": "#/definitions/v1Callback"
},
"description": "Callbacks to be called by the server when this workflow reaches a terminal state.\nIf the workflow continues-as-new, these callbacks will be carried over to the new execution.\nCallback addresses must be whitelisted in the server's dynamic configuration."
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo\nfor use by user interfaces to display the fixed as-of-start summary and details of the\nworkflow."
}
}
},
Expand Down Expand Up @@ -3807,6 +3815,10 @@
"commandType": {
"$ref": "#/definitions/v1CommandType"
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "Metadata on the command. This is sometimes carried over to the history event if one is\ncreated as a result of the command. Most commands won't have this information, and how this\ninformation is used is dependent upon the interface that reads it.\n\nCurrent well-known uses:\n * start_child_workflow_execution_command_attributes - populates\n temporal.api.workflow.v1.WorkflowExecutionInfo.user_metadata where the summary and details\n are used by user interfaces to show fixed as-of-start workflow summary and details.\n * start_timer_command_attributes - populates temporal.api.history.v1.HistoryEvent for timer\n started where the summary is used to identify the timer."
},
"scheduleActivityTaskCommandAttributes": {
"$ref": "#/definitions/v1ScheduleActivityTaskCommandAttributes"
},
Expand Down Expand Up @@ -4585,6 +4597,10 @@
"type": "boolean",
"description": "Set to true when the SDK may ignore the event as it does not impact workflow state or\ninformation in any way that the SDK need be concerned with. If an SDK encounters an event\ntype which it does not understand, it must error unless this is true. If it is true, it's\nacceptable for the event type and/or attributes to be uninterpretable."
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "Metadata on the event. This is often carried over from commands and client calls. Most events\nwon't have this information, and how this information is used is dependent upon the interface\nthat reads it.\n\nCurrent well-known uses:\n * workflow_execution_started_event_attributes - summary and details from start workflow.\n * timer_started_event_attributes - summary represents an identifier for the timer for use by\n user interfaces."
},
"workflowExecutionStartedEventAttributes": {
"$ref": "#/definitions/v1WorkflowExecutionStartedEventAttributes"
},
Expand Down Expand Up @@ -5330,6 +5346,10 @@
},
"header": {
"$ref": "#/definitions/v1Header"
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionConfig\nfor use by user interfaces to display the fixed as-of-start summary and details of the\nworkflow."
}
},
"description": "NewWorkflowExecutionInfo is a shared message that encapsulates all the\nrequired arguments to starting a workflow in different contexts."
Expand Down Expand Up @@ -7540,6 +7560,20 @@
}
}
},
"v1UserMetadata": {
"type": "object",
"properties": {
"summary": {
"$ref": "#/definitions/v1Payload",
"description": "Short-form text that provides a summary. This payload should be a \"json/plain\"-encoded payload\nthat is a single JSON string for use in user interfaces. User interface formatting may not\napply to this text when used in \"title\" situations. The payload data section is limited to 400\nbytes by default."
},
"details": {
"$ref": "#/definitions/v1Payload",
"description": "Long-form text that provides details. This payload should be a \"json/plain\"-encoded payload\nthat is a single JSON string for use in user interfaces. User interface formatting may apply to\nthis text in common use. The payload data section is limited to 20000 bytes by default."
}
},
"description": "Information a user can set, often for use by user interfaces."
},
"v1VersionInfo": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -7686,6 +7720,10 @@
},
"defaultWorkflowTaskTimeout": {
"type": "string"
},
"userMetadata": {
"$ref": "#/definitions/v1UserMetadata",
"description": "User metadata provided on start workflow."
}
}
},
Expand Down
56 changes: 56 additions & 0 deletions openapi/openapiv3.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2976,6 +2976,18 @@ components:
information in any way that the SDK need be concerned with. If an SDK encounters an event
type which it does not understand, it must error unless this is true. If it is true, it's
acceptable for the event type and/or attributes to be uninterpretable.
userMetadata:
allOf:
- $ref: '#/components/schemas/UserMetadata'
description: |-
Metadata on the event. This is often carried over from commands and client calls. Most events
won't have this information, and how this information is used is dependent upon the interface
that reads it.

Current well-known uses:
* workflow_execution_started_event_attributes - summary and details from start workflow.
* timer_started_event_attributes - summary represents an identifier for the timer for use by
user interfaces.
workflowExecutionStartedEventAttributes:
$ref: '#/components/schemas/WorkflowExecutionStartedEventAttributes'
workflowExecutionCompletedEventAttributes:
Expand Down Expand Up @@ -3426,6 +3438,13 @@ components:
$ref: '#/components/schemas/SearchAttributes'
header:
$ref: '#/components/schemas/Header'
userMetadata:
allOf:
- $ref: '#/components/schemas/UserMetadata'
description: |-
Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionConfig
for use by user interfaces to display the fixed as-of-start summary and details of the
workflow.
description: |-
NewWorkflowExecutionInfo is a shared message that encapsulates all the
required arguments to starting a workflow in different contexts.
Expand Down Expand Up @@ -4757,6 +4776,13 @@ components:
skipGenerateWorkflowTask:
type: boolean
description: Indicates that a new workflow task should not be generated when this signal is received.
userMetadata:
allOf:
- $ref: '#/components/schemas/UserMetadata'
description: |-
Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo
for use by user interfaces to display the fixed as-of-start summary and details of the
workflow.
SignalWithStartWorkflowExecutionResponse:
type: object
properties:
Expand Down Expand Up @@ -5055,6 +5081,13 @@ components:
Callbacks to be called by the server when this workflow reaches a terminal state.
If the workflow continues-as-new, these callbacks will be carried over to the new execution.
Callback addresses must be whitelisted in the server's dynamic configuration.
userMetadata:
allOf:
- $ref: '#/components/schemas/UserMetadata'
description: |-
Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo
for use by user interfaces to display the fixed as-of-start summary and details of the
workflow.
StartWorkflowExecutionResponse:
type: object
properties:
Expand Down Expand Up @@ -5492,6 +5525,25 @@ components:
description: The `WORKFLOW_TASK_COMPLETED` event which this command was reported with
searchAttributes:
$ref: '#/components/schemas/SearchAttributes'
UserMetadata:
type: object
properties:
summary:
allOf:
- $ref: '#/components/schemas/Payload'
description: |-
Short-form text that provides a summary. This payload should be a "json/plain"-encoded payload
that is a single JSON string for use in user interfaces. User interface formatting may not
apply to this text when used in "title" situations. The payload data section is limited to 400
bytes by default.
details:
allOf:
- $ref: '#/components/schemas/Payload'
description: |-
Long-form text that provides details. This payload should be a "json/plain"-encoded payload
that is a single JSON string for use in user interfaces. User interface formatting may apply to
this text in common use. The payload data section is limited to 20000 bytes by default.
description: Information a user can set, often for use by user interfaces.
VersionInfo:
type: object
properties:
Expand Down Expand Up @@ -5622,6 +5674,10 @@ components:
defaultWorkflowTaskTimeout:
pattern: ^-?(?:0|[1-9][0-9]{0,11})(?:\.[0-9]{1,9})?s$
type: string
userMetadata:
allOf:
- $ref: '#/components/schemas/UserMetadata'
description: User metadata provided on start workflow.
WorkflowExecutionContinuedAsNewEventAttributes:
type: object
properties:
Expand Down
13 changes: 13 additions & 0 deletions temporal/api/command/v1/message.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import "temporal/api/enums/v1/command_type.proto";
import "temporal/api/common/v1/message.proto";
import "temporal/api/failure/v1/message.proto";
import "temporal/api/taskqueue/v1/message.proto";
import "temporal/api/sdk/v1/user_metadata.proto";

message ScheduleActivityTaskCommandAttributes {
string activity_id = 1;
Expand Down Expand Up @@ -236,6 +237,18 @@ message ProtocolMessageCommandAttributes {

message Command {
temporal.api.enums.v1.CommandType command_type = 1;
// Metadata on the command. This is sometimes carried over to the history event if one is
// created as a result of the command. Most commands won't have this information, and how this
// information is used is dependent upon the interface that reads it.
//
// Current well-known uses:
yiminc marked this conversation as resolved.
Show resolved Hide resolved
// * start_child_workflow_execution_command_attributes - populates
// temporal.api.workflow.v1.WorkflowExecutionInfo.user_metadata where the summary and details
// are used by user interfaces to show fixed as-of-start workflow summary and details.
// * start_timer_command_attributes - populates temporal.api.history.v1.HistoryEvent for timer
// started where the summary is used to identify the timer.
temporal.api.sdk.v1.UserMetadata user_metadata = 301;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why choose the field number 301? Why not 18 (next immediate available number) or 100 (if we want to group in hundreds)?
Just curious about the convention we use when picking field numbers in proto.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same reason why the person who added HistoryEvent.worker_may_ignore as 300, so that it would not interfere with our continually added, incrementally numbered attribute fields below. I chose 301 instead of 300 just to match history event's equivalent, but don't have to. The field number doesn't really matter much here, so it leaves room for attributes to keep adding.

// The command details. The type must match that in `command_type`.
oneof attributes {
ScheduleActivityTaskCommandAttributes schedule_activity_task_command_attributes = 2;
StartTimerCommandAttributes start_timer_command_attributes = 3;
Expand Down
10 changes: 10 additions & 0 deletions temporal/api/history/v1/message.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import "temporal/api/taskqueue/v1/message.proto";
import "temporal/api/update/v1/message.proto";
import "temporal/api/workflow/v1/message.proto";
import "temporal/api/sdk/v1/task_complete_metadata.proto";
import "temporal/api/sdk/v1/user_metadata.proto";

// Always the first event in workflow history
message WorkflowExecutionStartedEventAttributes {
Expand Down Expand Up @@ -776,6 +777,15 @@ message HistoryEvent {
// type which it does not understand, it must error unless this is true. If it is true, it's
// acceptable for the event type and/or attributes to be uninterpretable.
bool worker_may_ignore = 300;
// Metadata on the event. This is often carried over from commands and client calls. Most events
// won't have this information, and how this information is used is dependent upon the interface
// that reads it.
//
// Current well-known uses:
// * workflow_execution_started_event_attributes - summary and details from start workflow.
// * timer_started_event_attributes - summary represents an identifier for the timer for use by
// user interfaces.
temporal.api.sdk.v1.UserMetadata user_metadata = 301;
// The event details. The type must match that in `event_type`.
oneof attributes {
WorkflowExecutionStartedEventAttributes workflow_execution_started_event_attributes = 6;
Expand Down
49 changes: 49 additions & 0 deletions temporal/api/sdk/v1/user_metadata.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// The MIT License
//
// Copyright (c) 2024 Temporal Technologies Inc. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

syntax = "proto3";

package temporal.api.sdk.v1;

option go_package = "go.temporal.io/api/sdk/v1;sdk";
option java_package = "io.temporal.api.sdk.v1";
option java_multiple_files = true;
option java_outer_classname = "UserMetadataProto";
option ruby_package = "Temporalio::Api::Sdk::V1";
option csharp_namespace = "Temporalio.Api.Sdk.V1";


import "temporal/api/common/v1/message.proto";

// Information a user can set, often for use by user interfaces.
message UserMetadata {
// Short-form text that provides a summary. This payload should be a "json/plain"-encoded payload
// that is a single JSON string for use in user interfaces. User interface formatting may not
// apply to this text when used in "title" situations. The payload data section is limited to 400
// bytes by default.
temporal.api.common.v1.Payload summary = 1;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume we considered this as a map from string to Payload.. why not do that? (For the record I like this better, just wondering what the reasoning was.)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There were a few of reasons that was rejected (there's an internal document discussing this). The primary reasons were that 1) the stringly-typed contract was too loose, 2) the keys are not user-defined, and 3) it unnecessarily increased history size for fixed keys.


// Long-form text that provides details. This payload should be a "json/plain"-encoded payload
// that is a single JSON string for use in user interfaces. User interface formatting may apply to
// this text in common use. The payload data section is limited to 20000 bytes by default.
temporal.api.common.v1.Payload details = 2;
}
13 changes: 6 additions & 7 deletions temporal/api/sdk/v1/workflow_metadata.proto
View file
Open in desktop
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file is safe for reworking in an incompatible way because it's just models for a well-known workflow query. See description to understand the changes.

Original file line number Diff line number Diff line change
Expand Up @@ -35,20 +35,19 @@ option csharp_namespace = "Temporalio.Api.Sdk.V1";
message WorkflowMetadata {
// Metadata provided at declaration or creation time.
WorkflowDefinition definition = 1;
// Current long-form details of the workflow's state. This is used by user interfaces to show
// long-form text. This text may be formatted by the user interface.
string current_details = 2;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is always returned in query responses and goes through the user's DC, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why a Payload in history and a string here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because this object is a query result and query results are payloads already (so already subject to encryption in transit) whereas in history if you want something encrypted at rest it needs to be a payload.

}

// (-- api-linter: core::0203::optional=disabled --)
message WorkflowDefinition {
// A name scoped by the task queue that maps to this workflow definition.
// If missing, this workflow is a dynamic workflow.
string type = 1;
// An optional workflow description provided by the application.
// By convention, external tools may interpret its first part,
// i.e., ending with a line break, as a summary of the description.
string description = 2;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It this okay to break? Do we have any consumers of this as protos?

Copy link
Member Author

@cretz cretz Mar 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it's ok to break, no we don't have any consumers of it except TypeScript SDK internally (and not this field and we only use this as JSON not proto). Explained in description and in comment.

repeated WorkflowInteractionDefinition query_definitions = 3;
repeated WorkflowInteractionDefinition signal_definitions = 4;
repeated WorkflowInteractionDefinition update_definitions = 5;
repeated WorkflowInteractionDefinition query_definitions = 2;
repeated WorkflowInteractionDefinition signal_definitions = 3;
repeated WorkflowInteractionDefinition update_definitions = 4;
}

// (-- api-linter: core::0123::resource-annotation=disabled
Expand Down
7 changes: 7 additions & 0 deletions temporal/api/workflow/v1/message.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import "temporal/api/enums/v1/workflow.proto";
import "temporal/api/common/v1/message.proto";
import "temporal/api/failure/v1/message.proto";
import "temporal/api/taskqueue/v1/message.proto";
import "temporal/api/sdk/v1/user_metadata.proto";

message WorkflowExecutionInfo {
temporal.api.common.v1.WorkflowExecution execution = 1;
Expand All @@ -65,6 +66,8 @@ message WorkflowExecutionConfig {
google.protobuf.Duration workflow_execution_timeout = 2;
google.protobuf.Duration workflow_run_timeout = 3;
google.protobuf.Duration default_workflow_task_timeout = 4;
// User metadata provided on start workflow.
temporal.api.sdk.v1.UserMetadata user_metadata = 5;
}

message PendingActivityInfo {
Expand Down Expand Up @@ -151,6 +154,10 @@ message NewWorkflowExecutionInfo {
temporal.api.common.v1.Memo memo = 11;
temporal.api.common.v1.SearchAttributes search_attributes = 12;
temporal.api.common.v1.Header header = 13;
// Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionConfig
// for use by user interfaces to display the fixed as-of-start summary and details of the
// workflow.
temporal.api.sdk.v1.UserMetadata user_metadata = 14;
}

// CallbackInfo contains the state of an attached workflow callback.
Expand Down
9 changes: 9 additions & 0 deletions temporal/api/workflowservice/v1/request_response.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import "temporal/api/update/v1/message.proto";
import "temporal/api/version/v1/message.proto";
import "temporal/api/batch/v1/message.proto";
import "temporal/api/sdk/v1/task_complete_metadata.proto";
import "temporal/api/sdk/v1/user_metadata.proto";
import "temporal/api/nexus/v1/message.proto";

import "google/protobuf/duration.proto";
Expand Down Expand Up @@ -192,6 +193,10 @@ message StartWorkflowExecutionRequest {
// If the workflow continues-as-new, these callbacks will be carried over to the new execution.
// Callback addresses must be whitelisted in the server's dynamic configuration.
repeated temporal.api.common.v1.Callback completion_callbacks = 21;
// Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo
// for use by user interfaces to display the fixed as-of-start summary and details of the
// workflow.
temporal.api.sdk.v1.UserMetadata user_metadata = 23;
}

message StartWorkflowExecutionResponse {
Expand Down Expand Up @@ -676,6 +681,10 @@ message SignalWithStartWorkflowExecutionRequest {
google.protobuf.Duration workflow_start_delay = 20;
// Indicates that a new workflow task should not be generated when this signal is received.
bool skip_generate_workflow_task = 21;
// Metadata on the workflow if it is started. This is carried over to the WorkflowExecutionInfo
// for use by user interfaces to display the fixed as-of-start summary and details of the
// workflow.
temporal.api.sdk.v1.UserMetadata user_metadata = 23;
}

message SignalWithStartWorkflowExecutionResponse {
Expand Down
Loading