Internal Change

PiperOrigin-RevId: 581813470

lit_nlp/api/dtypes.py

@@ -112,6 +112,14 @@ class FeatureSalience(DataTuple):
   salience: dict[str, float]
 
 
+@attr.s(auto_attribs=True, frozen=True, slots=True)
+class FrameSalience(DataTuple):
+  """Dataclass for a salience map over image frames in a video."""
+
+  # A map of salience score and image string by frame number
+  salience: dict[str, tuple[float, Sequence[str]]]
+
+
 # TODO(b/196886684): document API for salience interpreters.
 @attr.s(auto_attribs=True, frozen=True, slots=True)
 class SequenceSalienceMap(DataTuple):

lit_nlp/api/types.py

@@ -849,6 +849,13 @@ class FeatureSalience(Salience):
   default: dtypes.FeatureSalience = None
 
 
+@attr.s(auto_attribs=True, frozen=True, kw_only=True)
+class FrameSalience(Salience):
+  """Metadata about a returned frame salience map."""
+
+  default: dtypes.FrameSalience = None
+
+
 @attr.s(auto_attribs=True, frozen=True, kw_only=True)
 class ImageSalience(Salience):
   """Metadata about a returned image saliency.

lit_nlp/client/elements/frames_window.css

@@ -0,0 +1,13 @@
+.frames-window-image {
+  flex: 16.5%;
+  padding: 5px;
+}
+
+#frame {
+  max-width: 202px;
+  max-height: 360px;
+}
+
+.frames-window {
+  display: flex;
+}

lit_nlp/client/elements/frames_window.ts

@@ -0,0 +1,58 @@
+/**
+ * LIT module for displaying a variable size window of image frames.
+ */
+
+
+
+import {html, LitElement} from 'lit';
+import {customElement, property} from 'lit/decorators.js';
+import {classMap} from 'lit/directives/class-map.js';
+import {styles as sharedStyles} from '../lib/shared_styles.css';
+
+import {styles} from './frames_window.css';
+
+/**
+ * A LIT module to display variable size list of image frames within a window.
+ */
+@customElement('lit-frames-window')
+export class FramesWindow extends LitElement {
+  @property({type: Array}) frames: string[] = [];
+
+  static override get styles() {
+    return [
+      sharedStyles,
+      styles,
+    ];
+  }
+
+
+  private renderImage(imgSrc: string) {
+    const imageClass = classMap({
+      'frames-window-image': true,
+      'table-image': true,
+    });
+
+    return html`
+     <div class=${imageClass}>
+       <img id='frame' src="${imgSrc}"></img>
+     </div>`;
+  }
+
+
+  override render() {
+    const framesDOM =
+        this.frames.map((imageSrc: string) => this.renderImage(imageSrc));
+
+    return html`
+     <div class="frames-window">
+       ${framesDOM}
+     </div>`;
+  }
+}
+
+
+declare global {
+  interface HTMLElementTagNameMap {
+    'lit-frames-window': FramesWindow;
+  }
+}

lit_nlp/client/elements/table_types.ts

@@ -36,7 +36,7 @@ export interface SortableTemplateResult {
   value: SortableTableEntry;
 }
 /** Wrapper types for the data supplied to the data table */
-export type TableEntry = string|number|TemplateResult|SortableTemplateResult;
+export type TableEntry = string|number|string[]|TemplateResult|SortableTemplateResult;
 
 /** Wrapper types for the rows of data supplied to the data table */
 export type TableData = TableEntry[]|{[key: string]: TableEntry};

lit_nlp/client/lib/dtypes.ts

@@ -60,6 +60,11 @@ export interface FeatureSalience extends DataTuple {
   salience: {[key: string]: number};
 }
 
+/** Dataclass for a salience map over image frames in a video. */
+export interface FrameSalience extends DataTuple {
+  salience: {[key: string]: [number, string[]]};
+}
+
 // TODO(b/196886684): document API for salience interpreters.
 /** Dataclass for a salience map over a target sequence. */
 export interface SequenceSalienceMap extends DataTuple {

lit_nlp/client/lib/lit_types.ts

@@ -20,7 +20,7 @@
 // their Python counterparts.
 // tslint:disable:no-new-decorators class-name enforce-name-casing
 
-import {AnnotationCluster, EdgeLabel, FeatureSalience as FeatureSalienceDType, ScoredTextCandidates, SequenceSalienceMap, SpanLabel, TokenSalience as TokenSalienceDType} from './dtypes';
+import {AnnotationCluster, EdgeLabel, FeatureSalience as FeatureSalienceDType, ScoredTextCandidates, SequenceSalienceMap, SpanLabel, TokenSalience as TokenSalienceDType, FrameSalience as FrameSalienceDType} from './dtypes';
 
 /**
  * A dictionary of registered LitType names mapped to their constructor.
@@ -544,6 +544,14 @@ export class FeatureSalience extends Salience {
   override default: FeatureSalienceDType|undefined = undefined;
 }
 
+/**
+ * Metadata about a returned frame salience map.
+ */
+@registered
+export class FrameSalience extends Salience {
+  override default: FrameSalienceDType|undefined = undefined;
+}
+
 /**
  * Metadata about a returned image saliency.
  * The data is returned as an image in the base64 URL encoded format, e.g.,

lit_nlp/client/modules/salience_map_module.ts

@@ -35,7 +35,7 @@ import {LitModule} from '../core/lit_module';
 import {LegendType} from '../elements/color_legend';
 import {InterpreterClick} from '../elements/interpreter_controls';
 import {TokenWithWeight} from '../elements/token_chips';
-import {FeatureSalience, FieldMatcher, ImageGradients, ImageSalience, LitTypeTypesList, LitTypeWithParent, MulticlassPreds, RegressionScore, Salience, SalienceTargetInfo, TokenGradients, TokenSalience} from '../lib/lit_types';
+import {FeatureSalience, FieldMatcher, ImageGradients, ImageSalience, LitTypeTypesList, LitTypeWithParent, MulticlassPreds, RegressionScore, Salience, SalienceTargetInfo, TokenGradients, TokenSalience, FrameSalience} from '../lib/lit_types';
 import {styles as sharedStyles} from '../lib/shared_styles.css';
 import {CallConfig, IndexedInput, ModelInfoMap, Preds, SCROLL_SYNC_CSS_CLASS, Spec} from '../lib/types';
 import {argmax, cloneSpec, findSpecKeys, makeModifiedInput} from '../lib/utils';
@@ -64,12 +64,11 @@ interface FeatureSalienceResult {
   [key: string]: {salience: FeatureSalienceMap};
 }
 
-type SalienceResult = TokenSalienceResult | ImageSalienceResult |
+/** Different types of salience results supported in this module. */
+export type SalienceResult = TokenSalienceResult | ImageSalienceResult |
                       FeatureSalienceResult;
 
-// Notably, not SequenceSalience as that is handled by a different module.
-const SUPPORTED_SALIENCE_TYPES =
-    [TokenSalience, FeatureSalience, ImageSalience];
+const SUPPORTED_SALIENCE_TYPES = [TokenSalience, FeatureSalience, ImageSalience, FrameSalience];
 
 const TARGET_SELECTOR_SUPPORTED_TYPES: LitTypeTypesList =
     [MulticlassPreds, RegressionScore];