preprocessing_investigation.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Anaconda: learn-env",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "\n",
    "import apache_beam as beam\n",
    "import tensorflow as tf\n",
    "from apache_beam.options.pipeline_options import PipelineOptions\n",
    "import apache_beam.runners.interactive.interactive_beam as ib\n",
    "import apache_beam.transforms.sql\n",
    "\n",
    "import beam__common\n",
    "import fidscs_globals\n",
    "import random\n",
    "\n",
    "import data_extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/tmp/fids-capstone-data/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "use_beam: True\n",
      "INFO:tensorflow:Using MirroredStrategy with devices ('/device:CPU:0',)\n",
      "INFO:tensorflow:Single-worker MultiWorkerMirroredStrategy with local_devices = ('/device:CPU:0',), communication = CollectiveCommunication.AUTO\n",
      "Number of devices available for parallel processing: 1\n",
      "PipelineOptions:\n",
      "{'runner': 'DirectRunner', 'streaming': False, 'beam_services': {}, 'type_check_strictness': 'DEFAULT_TO_ANY', 'type_check_additional': '', 'pipeline_type_check': True, 'runtime_type_check': False, 'performance_runtime_type_check': False, 'direct_runner_use_stacked_bundle': True, 'direct_runner_bundle_repeat': 0, 'direct_num_workers': 0, 'direct_running_mode': 'multi_threading', 'dataflow_endpoint': 'https://dataflow.googleapis.com', 'project': 'my-project', 'job_name': None, 'staging_location': None, 'temp_location': None, 'region': None, 'service_account_email': None, 'no_auth': False, 'template_location': None, 'labels': None, 'update': False, 'transform_name_mapping': None, 'enable_streaming_engine': False, 'dataflow_kms_key': None, 'flexrs_goal': None, 'hdfs_host': None, 'hdfs_port': None, 'hdfs_user': None, 'hdfs_full_urls': False, 'num_workers': None, 'max_num_workers': None, 'autoscaling_algorithm': None, 'machine_type': None, 'disk_size_gb': None, 'disk_type': None, 'worker_region': None, 'worker_zone': None, 'zone': None, 'network': None, 'subnetwork': None, 'worker_harness_container_image': None, 'sdk_harness_container_image_overrides': None, 'use_public_ips': None, 'min_cpu_platform': None, 'dataflow_worker_jar': None, 'dataflow_job_file': None, 'experiments': None, 'number_of_worker_harness_threads': None, 'profile_cpu': False, 'profile_memory': False, 'profile_location': None, 'profile_sample_rate': 1.0, 'requirements_file': None, 'requirements_cache': None, 'setup_file': None, 'beam_plugins': None, 'save_main_session': False, 'sdk_location': 'default', 'extra_packages': None, 'prebuild_sdk_container_engine': None, 'prebuild_sdk_container_base_image': None, 'docker_registry_push_url': None, 'job_endpoint': None, 'artifact_endpoint': None, 'job_server_timeout': 60, 'environment_type': None, 'environment_config': None, 'environment_options': None, 'sdk_worker_parallelism': 1, 'environment_cache_millis': 0, 'output_executable_path': None, 'artifacts_dir': None, 'job_port': 0, 'artifact_port': 0, 'expansion_port': 0, 'flink_master': '[auto]', 'flink_version': '1.10', 'flink_job_server_jar': None, 'flink_submit_uber_jar': False, 'spark_master_url': 'local[4]', 'spark_job_server_jar': None, 'spark_submit_uber_jar': False, 'spark_rest_url': None, 'on_success_matcher': None, 'dry_run': False, 'wait_until_finish_duration': None, 'pubsubRootUrl': None, 's3_access_key_id': None, 's3_secret_access_key': None, 's3_session_token': None, 's3_endpoint_url': None, 's3_region_name': None, 's3_api_version': None, 's3_verify': None, 's3_disable_ssl': False}\n",
      "\n",
      "Found dataset /tmp/fids-capstone-data/data/consultant-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-targetvideo-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-utterance-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-utterance-targetvideo-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-utterance-token-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/ncslgr-corpus-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/document-consultant-targetvideo-utterance-token-frame-index.csv\n",
      "Found dataset /tmp/fids-capstone-data/data/vocabulary-index.csv\n",
      "Beam PL: ALL DONE!\n"
     ]
    }
   ],
   "source": [
    "data_extractor.run(max_target_videos=-1, data_dir=data_dir, use_beam=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "PipelineOptions:\n{'runner': 'InteractiveRunner', 'streaming': False, 'beam_services': {}, 'type_check_strictness': 'DEFAULT_TO_ANY', 'type_check_additional': '', 'pipeline_type_check': True, 'runtime_type_check': False, 'performance_runtime_type_check': False, 'direct_runner_use_stacked_bundle': True, 'direct_runner_bundle_repeat': 0, 'direct_num_workers': 0, 'direct_running_mode': 'multi_threading', 'dataflow_endpoint': 'https://dataflow.googleapis.com', 'project': 'my-project', 'job_name': None, 'staging_location': None, 'temp_location': None, 'region': None, 'service_account_email': None, 'no_auth': False, 'template_location': None, 'labels': None, 'update': False, 'transform_name_mapping': None, 'enable_streaming_engine': False, 'dataflow_kms_key': None, 'flexrs_goal': None, 'hdfs_host': None, 'hdfs_port': None, 'hdfs_user': None, 'hdfs_full_urls': False, 'num_workers': None, 'max_num_workers': None, 'autoscaling_algorithm': None, 'machine_type': None, 'disk_size_gb': None, 'disk_type': None, 'worker_region': None, 'worker_zone': None, 'zone': None, 'network': None, 'subnetwork': None, 'worker_harness_container_image': None, 'sdk_harness_container_image_overrides': None, 'use_public_ips': None, 'min_cpu_platform': None, 'dataflow_worker_jar': None, 'dataflow_job_file': None, 'experiments': None, 'number_of_worker_harness_threads': None, 'profile_cpu': False, 'profile_memory': False, 'profile_location': None, 'profile_sample_rate': 1.0, 'requirements_file': None, 'requirements_cache': None, 'setup_file': None, 'beam_plugins': None, 'save_main_session': False, 'sdk_location': 'default', 'extra_packages': None, 'prebuild_sdk_container_engine': None, 'prebuild_sdk_container_base_image': None, 'docker_registry_push_url': None, 'job_endpoint': None, 'artifact_endpoint': None, 'job_server_timeout': 60, 'environment_type': None, 'environment_config': None, 'environment_options': None, 'sdk_worker_parallelism': 1, 'environment_cache_millis': 0, 'output_executable_path': None, 'artifacts_dir': None, 'job_port': 0, 'artifact_port': 0, 'expansion_port': 0, 'flink_master': '[auto]', 'flink_version': '1.10', 'flink_job_server_jar': None, 'flink_submit_uber_jar': False, 'spark_master_url': 'local[4]', 'spark_job_server_jar': None, 'spark_submit_uber_jar': False, 'spark_rest_url': None, 'on_success_matcher': None, 'dry_run': False, 'wait_until_finish_duration': None, 'pubsubRootUrl': None, 's3_access_key_id': None, 's3_secret_access_key': None, 's3_session_token': None, 's3_endpoint_url': None, 's3_region_name': None, 's3_api_version': None, 's3_verify': None, 's3_disable_ssl': False}\n\n"
     ]
    }
   ],
   "source": [
    "options = {\n",
    "    'project': 'my-project', # change\n",
    "    'runner': 'InteractiveRunner',\n",
    "    'direct_num_workers': 0, # 0 is use all available cores\n",
    "    'direct_running_mode': 'multi_threading', # ['in_memory', 'multi_threading', 'multi_processing'] # 'multi_processing' doesn't seem to work for DirectRunner?\n",
    "    'streaming': False # set to True if data source is unbounded (e.g. GCP PubSub)\n",
    "}\n",
    "pipeline_options = PipelineOptions(flags=[], **options) # easier to pass in options from command-line this way\n",
    "print(f\"PipelineOptions:\\n{pipeline_options.get_all_options()}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fidscs_globals.DATA_ROOT_DIR = data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "can_proceed = True\n",
    "\n",
    "if not data_extractor__common.path_exists(fidscs_globals.DATA_ROOT_DIR) or len(beam__common.list_dir(fidscs_globals.DATA_ROOT_DIR))==0:\n",
    "    print(f\"{fidscs_globals.VALIDATION_FATAL_ERROR_TEXT} data directory does not exist or is empty!\")\n",
    "    can_proceed = False\n",
    "else:\n",
    "    fidscs_globals.VIDEO_DIR = os.path.join(fidscs_globals.DATA_ROOT_DIR, 'videos')\n",
    "    fidscs_globals.STICHED_VIDEO_FRAMES_DIR = os.path.join(fidscs_globals.DATA_ROOT_DIR, 'stitched_video_frames')\n",
    "    fidscs_globals.CORPUS_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.CORPUS_DS_FNAME)\n",
    "    fidscs_globals.DOCUMENT_ASL_CONSULTANT_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.DOCUMENT_ASL_CONSULTANT_DS_FNAME)\n",
    "    fidscs_globals.ASL_CONSULTANT_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.ASL_CONSULTANT_DS_FNAME)\n",
    "    fidscs_globals.VIDEO_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.VIDEO_DS_FNAME)\n",
    "    fidscs_globals.VIDEO_SEGMENT_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.VIDEO_SEGMENT_DS_FNAME)\n",
    "    fidscs_globals.VIDEO_FRAME_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.VIDEO_FRAME_DS_FNAME)\n",
    "    fidscs_globals.UTTERANCE_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.UTTERANCE_DS_FNAME)\n",
    "    fidscs_globals.UTTERANCE_VIDEO_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.UTTERANCE_VIDEO_DS_FNAME)\n",
    "    fidscs_globals.UTTERANCE_TOKEN_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.UTTERANCE_TOKEN_DS_FNAME)\n",
    "    fidscs_globals.UTTERANCE_TOKEN_FRAME_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.UTTERANCE_TOKEN_FRAME_DS_FNAME)\n",
    "    fidscs_globals.VOCABULARY_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.VOCABULARY_DS_FNAME)\n",
    "    fidscs_globals.TRAIN_ASSOC_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.TRAIN_FRAME_SEQ_ASSOC_DS_FNAME)\n",
    "    fidscs_globals.VAL_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.VAL_FRAME_SEQ_DS_FNAME)\n",
    "    fidscs_globals.TRAIN_DS_PATH = os.path.join(fidscs_globals.DATA_ROOT_DIR, fidscs_globals.TRAIN_FRAME_SEQ_DS_FNAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "pl = beam.Pipeline(options=pipeline_options)\n",
    "\n",
    "# full_target_vid_index_schemad_pcoll = beam__common.pl__1__read_target_vid_index_csv(pl)\n",
    "# corpus_index_schemad_pcoll = beam__common.pl__1__read_corpus_index_csv(pl) # XML is base-64 encode but we no longer need it (to decode it) since it is only used to create the datasets\n",
    "# # corpus_index_decoded_XML_pcoll = pl__2__decode_XML(corpus_index_schemad_pcoll) # see above\n",
    "\n",
    "# asl_consultant_index_schemad_pcoll = beam__common.pl__1__read_asl_consultant_index_csv(pl)\n",
    "# document_asl_consultant_utterance_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_utterance_index_csv(pl)\n",
    "# document_asl_consultant_target_video_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_target_video_index_csv(pl)\n",
    "# document_asl_consultant_utterance_video_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_utterance_video_index_csv(pl)\n",
    "# document_target_video_segment_index_schemad_pcoll = beam__common.pl__1__read_document_target_video_segment_index_csv(pl)\n",
    "# vocabulary_index_schemad_pcoll = beam__common.pl__1__read_vocabulary_index_csv(pl)\n",
    "# document_asl_consultant_utterance_token_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_utterance_token_index_csv(pl)\n",
    "# document_asl_consultant_target_video_frame_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_target_video_frame_index_csv(pl)\n",
    "\n",
    "# as it turns it, this is all we need\n",
    "document_asl_consultant_target_video_utterance_token_frame_index_schemad_pcoll = beam__common.pl__1__read_document_asl_consultant_target_video_utterance_token_frame_index_csv(pl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# document_asl_consultant_target_video_utterance_token_frame_index_schemad_pcoll is the main table we use for training.\n",
    "#     This will ultimately provide which frame sequences correspond to individual tokens.\n",
    "\n",
    "# But our first measure is to build train and validation sets (for tokens).\n",
    "#   In order to split up train vs validation sets, we need to compare \"apples to apples\".\n",
    "#   That is, in order for a token (TokenID) to be considered a candidate for the split,\n",
    "#   we require at least two of the same (TokenID, CameraPerspective) wherein the ASL\n",
    "#   consultant for each differs.  We would prefer more than two of these tuples, each\n",
    "#   having unique ASL consultants in the set of occurrences, with the majority of said\n",
    "#   tuples being assigned to the training set and the remainder (at least one) being\n",
    "#   assigned to the validation set.  We would like to achieve a 90/10 split, ideally,\n",
    "#   but we will take what we get.\n",
    "\n",
    "# document_asl_consultant_target_video_utterance_token_frame_index_schemad_pcoll:\n",
    "    # beam.Row(\n",
    "    #   DocumentID=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[0]]),\n",
    "    #   ASLConsultantID=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[1]]),\n",
    "    #   CameraPerspective=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[2]]),\n",
    "    #   TargetVideoFilename=str(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[3]]),\n",
    "    #   UtteranceSequence=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[4]]),\n",
    "    #   TokenSequence=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[5]]),\n",
    "    #   FrameSequence=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[6]]),\n",
    "    #   TokenID=int(d_document_asl_consultant_target_video_utterance_token_frame_info[fidscs_globals.SCHEMA_COL_NAMES__UTTERANCE_TOKEN_FRAME_DS[7]])\n",
    "    # )\n",
    "\n",
    "# We will transform this into tuples of the form:\n",
    "    # [\n",
    "    #     'TokenID', \n",
    "    #     'CameraPerspective', \n",
    "    #     'DocumentID', \n",
    "    #     'ASLConsultantID', \n",
    "    #     'TargetVideoFilename', \n",
    "    #     'UtteranceSequence', \n",
    "    #     'TokenSequence',\n",
    "\n",
    "    #     'FrameSequence'\n",
    "    # ]\n",
    "\n",
    "dctvustsfs = (\n",
    "    document_asl_consultant_target_video_utterance_token_frame_index_schemad_pcoll\n",
    "    | \"Beam PL: extract (TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence,FrameSequence) from dctvustsfs schemad pcoll\" >> beam.Map(\n",
    "            lambda dctvustsfs_row: (\n",
    "                dctvustsfs_row.TokenID,\n",
    "                dctvustsfs_row.CameraPerspective,\n",
    "                dctvustsfs_row.ASLConsultantID,\n",
    "                dctvustsfs_row.TargetVideoFilename,\n",
    "                dctvustsfs_row.UtteranceSequence,\n",
    "                dctvustsfs_row.TokenSequence,\n",
    "                dctvustsfs_row.FrameSequence\n",
    "            )\n",
    "        )\n",
    ")\n",
    "\n",
    "# for train-validation split, we want to key/group by (TokenID, CameraPerspective) with lists of unique (ASLConsultantID, TargetVideoFilename, UtteranceSequence, TokenSequence) > 1\n",
    "ctvusts_by_tcp = (\n",
    "    dctvustsfs\n",
    "    | \"Beam PL: extract ((TokenID,CameraPerspective), (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence)) from dctvustsfs\" >> beam.Map(\n",
    "            lambda dctvustsfs_row_tpl: (\n",
    "                (\n",
    "                    dctvustsfs_row_tpl[0],\n",
    "                    dctvustsfs_row_tpl[1]\n",
    "                ),\n",
    "                (\n",
    "                    dctvustsfs_row_tpl[2],\n",
    "                    dctvustsfs_row_tpl[3],\n",
    "                    dctvustsfs_row_tpl[4],\n",
    "                    dctvustsfs_row_tpl[5]\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "    | \"Beam PL: select distinct ((TokenID,CameraPerspective), (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence)) from ctvusts_by_tcp\" >> beam.Distinct()\n",
    "    | \"Beam PL: group (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence) by key (TokenID,CameraPerspective)\" >> beam.GroupByKey() \n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (\n",
    "        #         TokenID,\n",
    "        #         CameraPerspective\n",
    "        #     ),\n",
    "        #     listof(\n",
    "        #       (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence)\n",
    "        #     )\n",
    "        # )\n",
    ")\n",
    "\n",
    "\n",
    "def flatten_ctvusts_by_tcp(ctvusts_by_tcp_tpl):\n",
    "    return [\n",
    "        (\n",
    "            ctvusts_by_tcp_tpl[0][0],   # TokenID\n",
    "            ctvusts_by_tcp_tpl[0][1],   # CameraPerspective\n",
    "            ctvusts_tpl[0],             # ASLConsultantID\n",
    "            ctvusts_tpl[1],             # TargetVideoFilename\n",
    "            ctvusts_tpl[2],             # UtteranceSequence\n",
    "            ctvusts_tpl[3]              # TokenSequence\n",
    "        ) for ctvusts_tpl in ctvusts_by_tcp_tpl[1]\n",
    "    ]\n",
    "\n",
    "ctvusts_by_tcp__gt_1 = (\n",
    "    ctvusts_by_tcp\n",
    "    | \"Beam PL: filter candidate (TokenID,CameraPerspective) for test-validation split\" >> beam.Filter(\n",
    "            lambda list_ctvusts_by_tcp_tpl: len(set(list_ctvusts_by_tcp_tpl[1])) > 1\n",
    "        )\n",
    "    | \"Beam PL: flatten filtered (TokenID,CameraPerspective) candidates for test-validation split\" >> beam.FlatMap(flatten_ctvusts_by_tcp)\n",
    ")\n",
    "\n",
    "ctvusts_by_tcp__lte_1 = (\n",
    "    ctvusts_by_tcp\n",
    "    | \"Beam PL: filter non-candidate (TokenID,CameraPerspective) for test-validation split\" >> beam.Filter(\n",
    "            lambda list_ctvusts_by_tcp_tpl: len(set(list_ctvusts_by_tcp_tpl[1])) <= 1\n",
    "        )\n",
    "    | \"Beam PL: flatten filtered (TokenID,CameraPerspective) non-candidates for test-validation split\" >> beam.FlatMap(flatten_ctvusts_by_tcp)\n",
    ")"
   ]
  },
  {
   "source": [
    "<p><br>\n",
    "\n",
    "#### Finally, execute validation/train split on ctvusts_by_tcp__gt_1"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first, we need to put ctvusts_by_tcp__gt_1 back into ((TokenID, CameraPerspective), (ASLConsultantID, TargetVideoFilename, UtteranceSequence, TokenSequence)) form\n",
    "def rekey_ctvusts_by_tcp(ctvusts_by_tcp_tpl):\n",
    "    return (\n",
    "        (\n",
    "            ctvusts_by_tcp_tpl[0],  # TokenID\n",
    "            ctvusts_by_tcp_tpl[1]   # CameraPerspective\n",
    "        ),\n",
    "        (\n",
    "            ctvusts_by_tcp_tpl[2],  # ASLConsultantID\n",
    "            ctvusts_by_tcp_tpl[3],  # TargetVideoFilename\n",
    "            ctvusts_by_tcp_tpl[4],  # UtteranceSequence\n",
    "            ctvusts_by_tcp_tpl[5]   # TokenSequence\n",
    "        )\n",
    "    )\n",
    "\n",
    "def val_train_split__ctvusts_by_tcp__gt_1__tpl(ctvusts_list__by__tcp__gt_1__tpl):\n",
    "    \"\"\"\n",
    "    ctvusts_list__by__tcp__gt_1__tpl\n",
    "        (\n",
    "            (TokenID,CameraPerspective), # key\n",
    "            listof(\n",
    "                (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence)\n",
    "            )\n",
    "        )\n",
    "    \"\"\"\n",
    "    ctvusts_list = ctvusts_list__by__tcp__gt_1__tpl[1].copy() # we need a copy since we want to shuffle\n",
    "    random.shuffle(ctvusts_list)\n",
    "    len_ctvusts_list = len(ctvusts_list)\n",
    "    val_len_ctvusts_list = int(len_ctvusts_list*fidscs_globals.VALIDATION_SIZE_RATIO) if len_ctvusts_list > int(((1-fidscs_globals.VALIDATION_SIZE_RATIO)*100)/10) else 1\n",
    "    train__ctvusts_list, val__ctvusts_list = ctvusts_list[val_len_ctvusts_list:], ctvusts_list[:val_len_ctvusts_list]\n",
    "    return (\n",
    "        (\n",
    "            ctvusts_list__by__tcp__gt_1__tpl[0][0],    # TokenID\n",
    "            ctvusts_list__by__tcp__gt_1__tpl[0][1]     # CameraPerspective\n",
    "        ),\n",
    "        (\n",
    "            train__ctvusts_list,\n",
    "            val__ctvusts_list\n",
    "        )\n",
    "    )\n",
    "\n",
    "val_train_split_basis__ctvusts_by_tcp__gt_1 = (\n",
    "    ctvusts_by_tcp__gt_1\n",
    "    | \"Beam PL: rekey ctvusts_by_tcp__gt_1 for validation/train split\" >> beam.Map(rekey_ctvusts_by_tcp)\n",
    "    | \"Beam PL: group (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence) rekeyed by (TokenID,CameraPerspective)\" >> beam.GroupByKey()\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (TokenID,CameraPerspective), # key\n",
    "        #     listof(\n",
    "        #       (ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence)\n",
    "        #     )\n",
    "        # )\n",
    "    | \"Beam PL: split rekeyed ctvusts_list_by_tcp__gt_1\" >> beam.Map(val_train_split__ctvusts_by_tcp__gt_1__tpl)\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (TokenID,CameraPerspective), # key\n",
    "        #     (\n",
    "        #       test_list_of(ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence),\n",
    "        #       val_list_of(ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence),\n",
    "        #     )\n",
    "        # )\n",
    ")\n",
    "\n",
    "train__ctvusts_by_tcp__gt_1 = (\n",
    "    val_train_split_basis__ctvusts_by_tcp__gt_1\n",
    "    | \"Beam PL: select train sublist from val_train_split_basis__ctvusts_by_tcp__gt_1\" >> beam.Map(\n",
    "            lambda val_train_split_basis__ctvusts_by_tcp__gt_1_tpl: [\n",
    "                (\n",
    "                    val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[0][0],  # TokenID\n",
    "                    val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[0][1],  # CameraPerspective\n",
    "                    train_ctvusts_tpl[0],                                   # ASLConsultantID\n",
    "                    train_ctvusts_tpl[1],                                   # TargetVideoFilename\n",
    "                    train_ctvusts_tpl[2],                                   # UtteranceSequence\n",
    "                    train_ctvusts_tpl[3]                                    # TokenSequence\n",
    "                ) for train_ctvusts_tpl in val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[1][0] # index [1][0] points to train sublist\n",
    "            ]\n",
    "        )\n",
    "    | \"Beam PL: 'explode list_train__ctvusts_by_tcp__gt_1_tpl\" >> beam.FlatMap(lambda list_train__ctvusts_by_tcp__gt_1_tpl: list_train__ctvusts_by_tcp__gt_1_tpl)\n",
    ")\n",
    "\n",
    "val__ctvusts_by_tcp__gt_1 = (\n",
    "    val_train_split_basis__ctvusts_by_tcp__gt_1\n",
    "    | \"Beam PL: select validation sublist from val_train_split_basis__ctvusts_by_tcp__gt_1\" >> beam.Map(\n",
    "            lambda val_train_split_basis__ctvusts_by_tcp__gt_1_tpl: [\n",
    "                (\n",
    "                    val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[0][0],  # TokenID\n",
    "                    val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[0][1],  # CameraPerspective\n",
    "                    val_ctvusts_tpl[0],                                     # ASLConsultantID\n",
    "                    val_ctvusts_tpl[1],                                     # TargetVideoFilename\n",
    "                    val_ctvusts_tpl[2],                                     # UtteranceSequence\n",
    "                    val_ctvusts_tpl[3]                                      # TokenSequence\n",
    "                ) for val_ctvusts_tpl in val_train_split_basis__ctvusts_by_tcp__gt_1_tpl[1][1] # index [1][1] points to validation sublist\n",
    "            ]\n",
    "        )\n",
    "    | \"Beam PL: 'explode list_val__ctvusts_by_tcp__gt_1_tpl\" >> beam.FlatMap(lambda list_val__ctvusts_by_tcp__gt_1_tpl: list_val__ctvusts_by_tcp__gt_1_tpl)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join train__ctvusts_by_tcp__gt_1 to dctvustsfs\n",
    "train__ctvusts_by_tcp__gt_1__keys = (\n",
    "    train__ctvusts_by_tcp__gt_1\n",
    "    | \"Beam PL: extract ((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence), '<train__ctvusts_by_tcp__gt_1__has_key>') for join to dctvustsfs\" >> beam.Map(\n",
    "            lambda train__ctvusts_by_tcp__gt_1_tpl : (\n",
    "                (\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[0], # TokenID\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[1], # CameraPerspective\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[2], # ASLConsultantID\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[3], # TargetVideoFilename\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[4], # UtteranceSequence\n",
    "                    train__ctvusts_by_tcp__gt_1_tpl[5]  # TokenSequence\n",
    "                ),\n",
    "                \"<train__ctvusts_by_tcp__gt_1__has_key>\"\n",
    "            )\n",
    "        )\n",
    ")\n",
    "\n",
    "dctvustsfs__frame_sequences = (\n",
    "    dctvustsfs\n",
    "    | \"Beam PL: extract ((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence), FrameSequence) for join to train__ctvusts_by_tcp__gt_1/val__ctvusts_by_tcp__gt_1\" >> beam.Map(\n",
    "            lambda dctvustsfs_tpl: (\n",
    "                (\n",
    "                    dctvustsfs_tpl[0],  # TokenID\n",
    "                    dctvustsfs_tpl[1],  # CameraPerspective\n",
    "                    dctvustsfs_tpl[2],  # ASLConsultantID\n",
    "                    dctvustsfs_tpl[3],  # TargetVideoFilename\n",
    "                    dctvustsfs_tpl[4],  # UtteranceSequence\n",
    "                    dctvustsfs_tpl[5]   # TokenSequence\n",
    "                ),\n",
    "                dctvustsfs_tpl[6]       # FrameSequence\n",
    "            )\n",
    "        )\n",
    ")\n",
    "\n",
    "train_dctvustsfs__gt__1 = (\n",
    "    ({\n",
    "      'has_key': train__ctvusts_by_tcp__gt_1__keys,\n",
    "      'frame_sequences': dctvustsfs__frame_sequences\n",
    "    })\n",
    "    | \"Beam PL: join train__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.CoGroupByKey()\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (\n",
    "        #         <TokenID>,\n",
    "        #         <CameraPerspective>,\n",
    "        #         <ASLConsultantID>,\n",
    "        #         <TargetVideoFilename>,\n",
    "        #         <UtteranceSequence>,\n",
    "        #         <TokenSequence>\n",
    "        #     ),\n",
    "        #     {\n",
    "        #         'has_key': listof('<train__ctvusts_by_tcp__gt_1__has_key>'),    # should have only one/single element\n",
    "        #         'frame_sequences': listof(<FrameSequence>)                      # many\n",
    "        #     }\n",
    "        # )\n",
    "    | \"Beam PL: filter out mismatches from joined train__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.Filter(\n",
    "            lambda joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: \n",
    "                len(joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['has_key'])>0 and \\\n",
    "                    len(joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['frame_sequences'])>0\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof(<FrameSequence>) from joined train__ctvusts_by_tcp__gt_1 to dctvustsfs to list of tuples\" >> beam.Map(\n",
    "            lambda joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: [\n",
    "                (\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][0], # TokenID\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][1], # CameraPerspective\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][2], # ASLConsultantID\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][3], # TargetVideoFilename\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][4], # UtteranceSequence\n",
    "                    joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][5], # TokenSequence\n",
    "                    frame_seq\n",
    "                ) for frame_seq in sorted(joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['frame_sequences'])\n",
    "            ]\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence, FrameSequence)) from joined train__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.FlatMap(\n",
    "            lambda list_joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: list_joined__train__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join val__ctvusts_by_tcp__gt_1 to dctvustsfs\n",
    "val__ctvusts_by_tcp__gt_1__keys = (\n",
    "    val__ctvusts_by_tcp__gt_1\n",
    "    | \"Beam PL: extract ((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence), '<val__ctvusts_by_tcp__gt_1__has_key>') for join to dctvustsfs\" >> beam.Map(\n",
    "            lambda val__ctvusts_by_tcp__gt_1_tpl : (\n",
    "                (\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[0], # TokenID\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[1], # CameraPerspective\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[2], # ASLConsultantID\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[3], # TargetVideoFilename\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[4], # UtteranceSequence\n",
    "                    val__ctvusts_by_tcp__gt_1_tpl[5]  # TokenSequence\n",
    "                ),\n",
    "                \"<val__ctvusts_by_tcp__gt_1__has_key>\"\n",
    "            )\n",
    "        )\n",
    ")\n",
    "\n",
    "val_dctvustsfs__gt__1 = (\n",
    "    ({\n",
    "      'has_key': val__ctvusts_by_tcp__gt_1__keys,\n",
    "      'frame_sequences': dctvustsfs__frame_sequences\n",
    "    })\n",
    "    | \"Beam PL: join val__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.CoGroupByKey()\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (\n",
    "        #         <TokenID>,\n",
    "        #         <CameraPerspective>,\n",
    "        #         <ASLConsultantID>,\n",
    "        #         <TargetVideoFilename>,\n",
    "        #         <UtteranceSequence>,\n",
    "        #         <TokenSequence>\n",
    "        #     ),\n",
    "        #     {\n",
    "        #         'has_key': listof('<val__ctvusts_by_tcp__gt_1__has_key>'),    # should have only one/single element\n",
    "        #         'frame_sequences': listof(<FrameSequence>)                      # many\n",
    "        #     }\n",
    "        # )\n",
    "    | \"Beam PL: filter out mismatches from joined val__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.Filter(\n",
    "            lambda joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: \n",
    "                len(joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['has_key'])>0 and \\\n",
    "                    len(joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['frame_sequences'])>0\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof(<FrameSequence>) from joined val__ctvusts_by_tcp__gt_1 to dctvustsfs to list of tuples\" >> beam.Map(\n",
    "            lambda joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: [\n",
    "                (\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][0],   # TokenID\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][1],   # CameraPerspective\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][2],   # ASLConsultantID\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][3],   # TargetVideoFilename\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][4],   # UtteranceSequence\n",
    "                    joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[0][5],   # TokenSequence\n",
    "                    frame_seq                                                       # FrameSequence\n",
    "                ) for frame_seq in sorted(joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl[1]['frame_sequences'])\n",
    "            ]\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence, FrameSequence)) from joined val__ctvusts_by_tcp__gt_1 to dctvustsfs\" >> beam.FlatMap(\n",
    "            lambda list_joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl: list_joined__val__ctvusts_by_tcp__gt_1__to__dctvustsfs__tpl\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train__ctvusts_by_tcp__lte_1__keys = (\n",
    "    ctvusts_by_tcp__lte_1\n",
    "    | \"Beam PL: extract ((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence), '<ctvusts_by_tcp__lte_1_tpl__has_key>') for join to dctvustsfs\" >> beam.Map(\n",
    "            lambda ctvusts_by_tcp__lte_1_tpl : (\n",
    "                (\n",
    "                    ctvusts_by_tcp__lte_1_tpl[0], # TokenID\n",
    "                    ctvusts_by_tcp__lte_1_tpl[1], # CameraPerspective\n",
    "                    ctvusts_by_tcp__lte_1_tpl[2], # ASLConsultantID\n",
    "                    ctvusts_by_tcp__lte_1_tpl[3], # TargetVideoFilename\n",
    "                    ctvusts_by_tcp__lte_1_tpl[4], # UtteranceSequence\n",
    "                    ctvusts_by_tcp__lte_1_tpl[5]  # TokenSequence\n",
    "                ),\n",
    "                \"<ctvusts_by_tcp__lte_1_tpl__has_key>\"\n",
    "            )\n",
    "        )\n",
    ")\n",
    "\n",
    "train_dctvustsfs__lte_1 = (\n",
    "    ({\n",
    "      'has_key': train__ctvusts_by_tcp__lte_1__keys,\n",
    "      'frame_sequences': dctvustsfs__frame_sequences\n",
    "    })\n",
    "    | \"Beam PL: join ctvusts_by_tcp__lte_1 to dctvustsfs\" >> beam.CoGroupByKey()\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (\n",
    "        #         <TokenID>,\n",
    "        #         <CameraPerspective>,\n",
    "        #         <ASLConsultantID>,\n",
    "        #         <TargetVideoFilename>,\n",
    "        #         <UtteranceSequence>,\n",
    "        #         <TokenSequence>\n",
    "        #     ),\n",
    "        #     {\n",
    "        #         'has_key': listof('<ctvusts_by_tcp__lte_1_tpl__has_key>'),    # should have only one/single element\n",
    "        #         'frame_sequences': listof(<FrameSequence>)                      # many\n",
    "        #     }\n",
    "        # )\n",
    "    | \"Beam PL: filter out mismatches from joined train__ctvusts_by_tcp__lte_1 to dctvustsfs\" >> beam.Filter(\n",
    "            lambda joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl: \n",
    "                len(joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[1]['has_key'])>0 and \\\n",
    "                    len(joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[1]['frame_sequences'])>0\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof(<FrameSequence>) from joined train__ctvusts_by_tcp__lte_1 to dctvustsfs to list of tuples\" >> beam.Map(\n",
    "            lambda joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl: [\n",
    "                (\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][0], # TokenID\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][1], # CameraPerspective\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][2], # ASLConsultantID\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][3], # TargetVideoFilename\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][4], # UtteranceSequence\n",
    "                    joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[0][5], # TokenSequence\n",
    "                    frame_seq\n",
    "                ) for frame_seq in sorted(joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl[1]['frame_sequences'])\n",
    "            ]\n",
    "        )\n",
    "    | \"Beam PL: 'explode' listof((TokenID,CameraPerspective,ASLConsultantID,TargetVideoFilename,UtteranceSequence,TokenSequence, FrameSequence)) from joined ttrain__ctvusts_by_tcp__lte_1 to dctvustsfs\" >> beam.FlatMap(\n",
    "            lambda list_joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl: list_joined__train__ctvusts_by_tcp__lte_1__to__dctvustsfs__tpl\n",
    "        )\n",
    ")\n",
    "\n",
    "train_dctvustsfs__all = (\n",
    "    (train_dctvustsfs__gt__1, train_dctvustsfs__lte_1) \n",
    "    | f\"Beam PL: merge train_dctvustsfs__gt__1 with train_dctvustsfs__lte_1\" >> beam.Flatten() \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find all COMPLETE utterances that can be formed with token-cameraperspective pairs from the validation set\n",
    "\n",
    "val_tcp__gt__1 = (\n",
    "    val_dctvustsfs__gt__1\n",
    "    | \"Beam PL: extract (TokenID, CameraPerspective) from val_dctvustsfs__gt__1\" >> beam.Map(\n",
    "            lambda tpl: (\n",
    "                tpl[0], # TokenID\n",
    "                tpl[1]  # CameraPerspective\n",
    "            )\n",
    "        )\n",
    "    | \"Beam PL: select distinct (TokenID, CameraPerspective) from val_dctvustsfs__gt__1\" >> beam.Distinct()\n",
    ")\n",
    "\n",
    "complete_utterances__with__val_tcp__gt__1 = (\n",
    "    dctvustsfs\n",
    "    | \"Beam PL: extract (ASLConsultantID,TargetVideoFilename,CameraPerspective,UtteranceSequence,TokenSequence,TokenID) from dctvustsfs\" >> beam.Map(\n",
    "            lambda tpl: (\n",
    "                tpl[2], # <ASLConsultantID>\n",
    "                tpl[3], # <TargetVideoFilename>\n",
    "                tpl[4], # <UtteranceSequence>\n",
    "                tpl[1], # <CameraPerspective>\n",
    "\n",
    "                tpl[5], # <TokenSequence>\n",
    "                tpl[0]  # <TokenID>\n",
    "            )\n",
    "        )\n",
    "    | \"Beam PL: select distinct (ASLConsultantID,TargetVideoFilename,CameraPerspective,UtteranceSequence,TokenSequence,TokenID) from dctvustsfs\" >> beam.Distinct()\n",
    "    | \"Beam PL: transform distinct ctvcpustst tuples to tst_by_ctvuscp\" >> beam.Map(\n",
    "            lambda tpl: (\n",
    "                (\n",
    "                    tpl[0], # <ASLConsultantID>\n",
    "                    tpl[1], # <TargetVideoFilename>\n",
    "                    tpl[2], # <UtteranceSequence>\n",
    "                    tpl[3]  # <CameraPerspective>\n",
    "                ),\n",
    "                (\n",
    "                    tpl[4], # <TokenSequence>\n",
    "                    tpl[5]  # <TokenID>\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "    | \"Beam PL: collect list of tokenseq-tokenid for each (<ASLConsultantID>, <TargetVideoFilename>, <UtteranceSequence>, <CameraPerspective>)\" >> beam.GroupByKey()\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (<ASLConsultantID>,<TargetVideoFilename>,<UtteranceSequence>,<CameraPerspective>), # key\n",
    "        #     listof((<TokenSequence>,<TokenID>))\n",
    "        # )\n",
    "    | \"Beam PL: sort list of tokenseq-tokenid by tokenseq for each (<ASLConsultantID>, <TargetVideoFilename>, <UtteranceSequence>, <CameraPerspective>)\" >> beam.Map(\n",
    "            lambda tpl: (\n",
    "                (\n",
    "                    tpl[0][0], # <ASLConsultantID>\n",
    "                    tpl[0][1], # <TargetVideoFilename>\n",
    "                    tpl[0][2], # <UtteranceSequence>\n",
    "                    tpl[0][3]  # <CameraPerspective>\n",
    "                ),\n",
    "                [(tst_tpl[1], tpl[0][3]) for tst_tpl in sorted(tpl[1], key=lambda tst_tpl: tst_tpl[0])]\n",
    "            )\n",
    "        )\n",
    "    # the above produces tuples of the form:\n",
    "        # (\n",
    "        #     (<ASLConsultantID>,<TargetVideoFilename>,<UtteranceSequence>,<CameraPerspective>), # key\n",
    "        #     listof((<TokenID>, <CameraPerspective>)) # sorted by <TokenSequence>\n",
    "        # )\n",
    "\n",
    "    # now we need to filter all of the above (<ASLConsultantID>,<TargetVideoFilename>,<UtteranceSequence>,<CameraPerspective>) where every (<TokenID>, <CameraPerspective>) in the corresponding list exists in val_tcp__gt__1\n",
    "    | \"Beam PL: filter matching rows from vid index\" >> beam.Filter(\n",
    "        lambda list_tcp_tpl__by__ctvuscp__tpl, existing_val_tcp_tpls: all(tcp_tpl in existing_val_tcp_tpls for tcp_tpl in list_tcp_tpl__by__ctvuscp__tpl[1]),\n",
    "        existing_val_tcp_tpls=beam.pvalue.AsIter(val_tcp__gt__1)\n",
    "      )\n",
    "    | \"Beam PL: extract (<ASLConsultantID>,<TargetVideoFilename>,<UtteranceSequence>,<CameraPerspective>,listof(<TokenID>))\" >> beam.Map(\n",
    "            lambda tpl: (\n",
    "                tpl[0][0],  # <ASLConsultantID>\n",
    "                tpl[0][1],  # <TargetVideoFilename>\n",
    "                tpl[0][2],  # <UtteranceSequence>\n",
    "                tpl[0][3],  # <CameraPerspective>\n",
    "                [tcp_tpl[0] for tcp_tpl in tpl[1]] # listof(<TokenID>)\n",
    "            ) \n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we require this in order to make use of ib.show() (which provides visualization of the pcolls specified) or ib.collect() (which creates a pandas dataframe from a pcoll)\n",
    "    # but all pcolls we wish to visualize must be created prior to executing the following line\n",
    "ib.watch(locals())"
   ]
  },
  {
   "source": [
    "#### Show those with counts > 1"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_18b0cbf2adf98115d89f3435ae6b0e33\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_18b0cbf2adf98115d89f3435ae6b0e33\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_18b0cbf2adf98115d89f3435ae6b0e33\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__gt_1 = ib.collect(ctvusts_by_tcp__gt_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID           TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                  \n",
       "1       0                                3   dorm_prank_1053_small_0.mov   \n",
       "        0                                3   lapd_story_1083_small_0.mov   \n",
       "        0                                7     ben_story_439_small_0.mov   \n",
       "        0                                7     ben_story_439_small_0.mov   \n",
       "        1                                7     ben_story_439_small_1.mov   \n",
       "...                                    ...                           ...   \n",
       "2406    3                                3  scary_story_1048_small_3.mov   \n",
       "2409    0                                3    boston-la_1088_small_0.mov   \n",
       "        0                                3    boston-la_1088_small_0.mov   \n",
       "        2                                3    boston-la_1088_small_2.mov   \n",
       "        2                                3    boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "1       0                                 32              1  \n",
       "        0                                 15              2  \n",
       "        0                                 10              1  \n",
       "        0                                 29              1  \n",
       "        1                                 10              1  \n",
       "...                                      ...            ...  \n",
       "2406    3                                 35              0  \n",
       "2409    0                                  4             12  \n",
       "        0                                  6              5  \n",
       "        2                                  4             12  \n",
       "        2                                  6              5  \n",
       "\n",
       "[29554 rows x 4 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">1</th>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>10</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>29</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>ben_story_439_small_1.mov</td>\n      <td>10</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2406</th>\n      <th>3</th>\n      <td>3</td>\n      <td>scary_story_1048_small_3.mov</td>\n      <td>35</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">2409</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>4</td>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n<p>29554 rows × 4 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__gt_1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence']\n",
    "df_ctvusts_by_tcp__gt_1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_ctvusts_by_tcp__gt_1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence'], ignore_index=False, inplace=True)\n",
    "df_ctvusts_by_tcp__gt_1.sort_index(inplace=True)\n",
    "df_ctvusts_by_tcp__gt_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_ctvusts_by_tcp__gt_1.loc[\n",
    "#     (\n",
    "#         [2369],         # TokenID\n",
    "#         [2]             # CameraPerspective\n",
    "#     ), \n",
    "#     :\n",
    "# ].sort_index(ascending=[True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           count\n",
       "TokenID CameraPerspective       \n",
       "365     0                    685\n",
       "1809    0                    621\n",
       "365     2                    574\n",
       "1809    2                    553\n",
       "380     0                    413\n",
       "...                          ...\n",
       "1221    0                      2\n",
       "1751    0                      2\n",
       "1759    0                      2\n",
       "        2                      2\n",
       "2409    2                      2\n",
       "\n",
       "[3074 rows x 1 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>count</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>365</th>\n      <th>0</th>\n      <td>685</td>\n    </tr>\n    <tr>\n      <th>1809</th>\n      <th>0</th>\n      <td>621</td>\n    </tr>\n    <tr>\n      <th>365</th>\n      <th>2</th>\n      <td>574</td>\n    </tr>\n    <tr>\n      <th>1809</th>\n      <th>2</th>\n      <td>553</td>\n    </tr>\n    <tr>\n      <th>380</th>\n      <th>0</th>\n      <td>413</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1221</th>\n      <th>0</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>1751</th>\n      <th>0</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">1759</th>\n      <th>0</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2409</th>\n      <th>2</th>\n      <td>2</td>\n    </tr>\n  </tbody>\n</table>\n<p>3074 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__gt_1__count = df_ctvusts_by_tcp__gt_1.reset_index().groupby(['TokenID', 'CameraPerspective']).count()\n",
    "df_ctvusts_by_tcp__gt_1__count = df_ctvusts_by_tcp__gt_1__count[['ASLConsultantID']]\n",
    "df_ctvusts_by_tcp__gt_1__count.columns = ['count']\n",
    "df_ctvusts_by_tcp__gt_1__count.sort_values(axis=0, by=['count'], ascending=False, inplace=True)\n",
    "# df_ctvusts_by_tcp__gt_1__count.sort_index(inplace=True)\n",
    "df_ctvusts_by_tcp__gt_1__count"
   ]
  },
  {
   "source": [
    "#### Now show those with counts <= 1"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_f3047c82296d7dd1ff5429f85b5aea3c\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_f3047c82296d7dd1ff5429f85b5aea3c\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_f3047c82296d7dd1ff5429f85b5aea3c\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__lte_1 = ib.collect(ctvusts_by_tcp__lte_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID         TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                \n",
       "0       0                                7   ben_story_439_small_0.mov   \n",
       "        1                                7   ben_story_439_small_1.mov   \n",
       "        2                                7   ben_story_439_small_2.mov   \n",
       "        3                                7   ben_story_439_small_3.mov   \n",
       "4       0                                3   siblings_1066_small_0.mov   \n",
       "...                                    ...                         ...   \n",
       "2408    0                                0      DSP%2520Immigrants.mov   \n",
       "2410    0                                3  boston-la_1088_small_0.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "2411    0                                3  boston-la_1088_small_0.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "0       0                                  7              0  \n",
       "        1                                  7              0  \n",
       "        2                                  7              0  \n",
       "        3                                  7              0  \n",
       "4       0                                 21              1  \n",
       "...                                      ...            ...  \n",
       "2408    0                                 16              5  \n",
       "2410    0                                 39              5  \n",
       "        2                                 39              5  \n",
       "2411    0                                 81              5  \n",
       "        2                                 81              5  \n",
       "\n",
       "[3682 rows x 4 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">0</th>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>ben_story_439_small_1.mov</td>\n      <td>7</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7</td>\n      <td>ben_story_439_small_2.mov</td>\n      <td>7</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>7</td>\n      <td>ben_story_439_small_3.mov</td>\n      <td>7</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <th>0</th>\n      <td>3</td>\n      <td>siblings_1066_small_0.mov</td>\n      <td>21</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2408</th>\n      <th>0</th>\n      <td>0</td>\n      <td>DSP%2520Immigrants.mov</td>\n      <td>16</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">2410</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>39</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>39</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">2411</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>81</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n<p>3682 rows × 4 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__lte_1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence']\n",
    "df_ctvusts_by_tcp__lte_1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_ctvusts_by_tcp__lte_1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence'], ignore_index=False, inplace=True)\n",
    "df_ctvusts_by_tcp__lte_1.sort_index(inplace=True)\n",
    "df_ctvusts_by_tcp__lte_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           count\n",
       "TokenID CameraPerspective       \n",
       "0       0                      1\n",
       "1637    0                      1\n",
       "1638    3                      1\n",
       "1639    0                      1\n",
       "        2                      1\n",
       "...                          ...\n",
       "888     2                      1\n",
       "        3                      1\n",
       "889     0                      1\n",
       "        1                      1\n",
       "2411    2                      1\n",
       "\n",
       "[3682 rows x 1 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>count</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <th>0</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1637</th>\n      <th>0</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1638</th>\n      <th>3</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">1639</th>\n      <th>0</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">888</th>\n      <th>2</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">889</th>\n      <th>0</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2411</th>\n      <th>2</th>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>3682 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 20
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__lte_1__count = df_ctvusts_by_tcp__lte_1.reset_index().groupby(['TokenID', 'CameraPerspective']).count()\n",
    "df_ctvusts_by_tcp__lte_1__count = df_ctvusts_by_tcp__lte_1__count[['ASLConsultantID']]\n",
    "df_ctvusts_by_tcp__lte_1__count.columns = ['count']\n",
    "df_ctvusts_by_tcp__lte_1__count.sort_values(axis=0, by=['count'], ascending=False, inplace=True)\n",
    "# df_ctvusts_by_tcp__gt_1__count.sort_index(inplace=True)\n",
    "df_ctvusts_by_tcp__lte_1__count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [ASLConsultantID_left, TargetVideoFilename_left, UtteranceSequence_left, TokenSequence_left, ASLConsultantID_right, TargetVideoFilename_right, UtteranceSequence_right, TokenSequence_right]\n",
       "Index: []"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID_left</th>\n      <th>TargetVideoFilename_left</th>\n      <th>UtteranceSequence_left</th>\n      <th>TokenSequence_left</th>\n      <th>ASLConsultantID_right</th>\n      <th>TargetVideoFilename_right</th>\n      <th>UtteranceSequence_right</th>\n      <th>TokenSequence_right</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "df_ctvusts_by_tcp__intersection = df_ctvusts_by_tcp__gt_1.join(df_ctvusts_by_tcp__lte_1, how='inner', lsuffix='_left', rsuffix='_right')\n",
    "df_ctvusts_by_tcp__intersection"
   ]
  },
  {
   "source": [],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "#### Now show train/validation split"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_e37b1badc6be47fb1490004b08d4536b\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_e37b1badc6be47fb1490004b08d4536b\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_e37b1badc6be47fb1490004b08d4536b\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_train__ctvusts_by_tcp__gt_1 = ib.collect(train__ctvusts_by_tcp__gt_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID           TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                  \n",
       "1       0                                3   dorm_prank_1053_small_0.mov   \n",
       "        0                                7     ben_story_439_small_0.mov   \n",
       "        0                                7     ben_story_439_small_0.mov   \n",
       "        1                                7     ben_story_439_small_1.mov   \n",
       "        2                                3   dorm_prank_1053_small_2.mov   \n",
       "...                                    ...                           ...   \n",
       "2406    0                                3  scary_story_1048_small_0.mov   \n",
       "        2                                3  scary_story_1048_small_2.mov   \n",
       "        3                                3  scary_story_1048_small_3.mov   \n",
       "2409    0                                3    boston-la_1088_small_0.mov   \n",
       "        2                                3    boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "1       0                                 32              1  \n",
       "        0                                 10              1  \n",
       "        0                                 29              1  \n",
       "        1                                 29              1  \n",
       "        2                                 32              1  \n",
       "...                                      ...            ...  \n",
       "2406    0                                 35              0  \n",
       "        2                                 21              0  \n",
       "        3                                 35              0  \n",
       "2409    0                                  4             12  \n",
       "        2                                  4             12  \n",
       "\n",
       "[25190 rows x 4 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">1</th>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>10</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>29</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>ben_story_439_small_1.mov</td>\n      <td>29</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_2.mov</td>\n      <td>32</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">2406</th>\n      <th>0</th>\n      <td>3</td>\n      <td>scary_story_1048_small_0.mov</td>\n      <td>35</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>scary_story_1048_small_2.mov</td>\n      <td>21</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>scary_story_1048_small_3.mov</td>\n      <td>35</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">2409</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>4</td>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n    </tr>\n  </tbody>\n</table>\n<p>25190 rows × 4 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "df_train__ctvusts_by_tcp__gt_1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence']\n",
    "df_train__ctvusts_by_tcp__gt_1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_train__ctvusts_by_tcp__gt_1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence'], ignore_index=False, inplace=True)\n",
    "df_train__ctvusts_by_tcp__gt_1.sort_index(inplace=True)\n",
    "df_train__ctvusts_by_tcp__gt_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_180acb0956a02a9fce65a129c7d106b1\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_180acb0956a02a9fce65a129c7d106b1\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_180acb0956a02a9fce65a129c7d106b1\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_val__ctvusts_by_tcp__gt_1 = ib.collect(val__ctvusts_by_tcp__gt_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID               TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                      \n",
       "1       0                                3       lapd_story_1083_small_0.mov   \n",
       "        1                                7         ben_story_439_small_1.mov   \n",
       "        2                                3       lapd_story_1083_small_2.mov   \n",
       "        3                                3  dorm_prank_1053_small_3.%2520mov   \n",
       "2       0                                7         ben_story_439_small_0.mov   \n",
       "...                                    ...                               ...   \n",
       "2406    0                                3      scary_story_1048_small_0.mov   \n",
       "        2                                3      scary_story_1048_small_2.mov   \n",
       "        3                                3      scary_story_1048_small_3.mov   \n",
       "2409    0                                3        boston-la_1088_small_0.mov   \n",
       "        2                                3        boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "1       0                                 15              2  \n",
       "        1                                 10              1  \n",
       "        2                                 15              2  \n",
       "        3                                 32              1  \n",
       "2       0                                 43              8  \n",
       "...                                      ...            ...  \n",
       "2406    0                                 21              0  \n",
       "        2                                 35              0  \n",
       "        3                                 21              0  \n",
       "2409    0                                  6              5  \n",
       "        2                                  6              5  \n",
       "\n",
       "[4364 rows x 4 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">1</th>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>ben_story_439_small_1.mov</td>\n      <td>10</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_2.mov</td>\n      <td>15</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_3.%2520mov</td>\n      <td>32</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>43</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">2406</th>\n      <th>0</th>\n      <td>3</td>\n      <td>scary_story_1048_small_0.mov</td>\n      <td>21</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>scary_story_1048_small_2.mov</td>\n      <td>35</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>scary_story_1048_small_3.mov</td>\n      <td>21</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">2409</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n<p>4364 rows × 4 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "df_val__ctvusts_by_tcp__gt_1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence']\n",
    "df_val__ctvusts_by_tcp__gt_1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_val__ctvusts_by_tcp__gt_1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence'], ignore_index=False, inplace=True)\n",
    "df_val__ctvusts_by_tcp__gt_1.sort_index(inplace=True)\n",
    "df_val__ctvusts_by_tcp__gt_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID         TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                \n",
       "2409    0                                3  boston-la_1088_small_0.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "2409    0                                  4             12  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2409</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>4</td>\n      <td>12</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "df_train__ctvusts_by_tcp__gt_1.loc[\n",
    "    (\n",
    "        [2409],         # TokenID\n",
    "        [0]             # CameraPerspective\n",
    "    ), \n",
    "    :\n",
    "].sort_index(ascending=[True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID         TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                \n",
       "2409    0                                3  boston-la_1088_small_0.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  \n",
       "TokenID CameraPerspective                                    \n",
       "2409    0                                  6              5  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2409</th>\n      <th>0</th>\n      <td>3</td>\n      <td>boston-la_1088_small_0.mov</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "df_val__ctvusts_by_tcp__gt_1.loc[\n",
    "    (\n",
    "        [2409],         # TokenID\n",
    "        [0]             # CameraPerspective\n",
    "    ), \n",
    "    :\n",
    "].sort_index(ascending=[True, True])"
   ]
  },
  {
   "source": [
    "<p><br>\n",
    "\n",
    "#### View final training/validation sets (with associated frame sequences)\n",
    "\n",
    "<p><br>\n",
    "\n",
    "##### Training (sub) set (that has at least one corresponding token/camera perspective in the validation set)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_e63f92b31826ce4801ad3919d2b55cc9\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_e63f92b31826ce4801ad3919d2b55cc9\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_e63f92b31826ce4801ad3919d2b55cc9\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_train_dctvustsfs__gt__1 = ib.collect(train_dctvustsfs__gt__1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID          TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                 \n",
       "1       0                                3  dorm_prank_1053_small_0.mov   \n",
       "        0                                3  dorm_prank_1053_small_0.mov   \n",
       "        0                                3  dorm_prank_1053_small_0.mov   \n",
       "        0                                3  dorm_prank_1053_small_0.mov   \n",
       "        0                                3  dorm_prank_1053_small_0.mov   \n",
       "...                                    ...                          ...   \n",
       "2409    2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  FrameSequence  \n",
       "TokenID CameraPerspective                                                   \n",
       "1       0                                 32              1           3206  \n",
       "        0                                 32              1           3207  \n",
       "        0                                 32              1           3208  \n",
       "        0                                 32              1           3209  \n",
       "        0                                 32              1           3210  \n",
       "...                                      ...            ...            ...  \n",
       "2409    2                                  4             12            884  \n",
       "        2                                  4             12            885  \n",
       "        2                                  4             12            886  \n",
       "        2                                  4             12            887  \n",
       "        2                                  4             12            888  \n",
       "\n",
       "[193148 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n      <th>FrameSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">1</th>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n      <td>3206</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n      <td>3207</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n      <td>3208</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n      <td>3209</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>dorm_prank_1053_small_0.mov</td>\n      <td>32</td>\n      <td>1</td>\n      <td>3210</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">2409</th>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n      <td>884</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n      <td>885</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n      <td>886</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n      <td>887</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>4</td>\n      <td>12</td>\n      <td>888</td>\n    </tr>\n  </tbody>\n</table>\n<p>193148 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 29
    }
   ],
   "source": [
    "df_train_dctvustsfs__gt__1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence']\n",
    "df_train_dctvustsfs__gt__1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_train_dctvustsfs__gt__1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence'], ignore_index=False, inplace=True)\n",
    "df_train_dctvustsfs__gt__1.sort_index(inplace=True)\n",
    "df_train_dctvustsfs__gt__1"
   ]
  },
  {
   "source": [
    "<p><br>\n",
    "\n",
    "##### Validation set"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_21507a5b8e3a58175cd652de067fff12\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_21507a5b8e3a58175cd652de067fff12\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_21507a5b8e3a58175cd652de067fff12\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_val_dctvustsfs__gt__1 = ib.collect(val_dctvustsfs__gt__1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID          TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                 \n",
       "1       0                                3  lapd_story_1083_small_0.mov   \n",
       "        0                                3  lapd_story_1083_small_0.mov   \n",
       "        0                                3  lapd_story_1083_small_0.mov   \n",
       "        0                                3  lapd_story_1083_small_0.mov   \n",
       "        0                                3  lapd_story_1083_small_0.mov   \n",
       "...                                    ...                          ...   \n",
       "2409    2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "        2                                3   boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  FrameSequence  \n",
       "TokenID CameraPerspective                                                   \n",
       "1       0                                 15              2           1218  \n",
       "        0                                 15              2           1219  \n",
       "        0                                 15              2           1220  \n",
       "        0                                 15              2           1221  \n",
       "        0                                 15              2           1222  \n",
       "...                                      ...            ...            ...  \n",
       "2409    2                                  6              5           1189  \n",
       "        2                                  6              5           1190  \n",
       "        2                                  6              5           1191  \n",
       "        2                                  6              5           1192  \n",
       "        2                                  6              5           1193  \n",
       "\n",
       "[40141 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n      <th>FrameSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">1</th>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n      <td>1218</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n      <td>1219</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n      <td>1220</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n      <td>1221</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>3</td>\n      <td>lapd_story_1083_small_0.mov</td>\n      <td>15</td>\n      <td>2</td>\n      <td>1222</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">2409</th>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1189</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1190</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1191</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1192</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1193</td>\n    </tr>\n  </tbody>\n</table>\n<p>40141 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "df_val_dctvustsfs__gt__1.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence']\n",
    "df_val_dctvustsfs__gt__1.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_val_dctvustsfs__gt__1.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence'], ignore_index=False, inplace=True)\n",
    "df_val_dctvustsfs__gt__1.sort_index(inplace=True)\n",
    "df_val_dctvustsfs__gt__1"
   ]
  },
  {
   "source": [
    "##### The complete training set (union of training subset - token/camera perspectives with corresponding validation set tuples - with training subset with no corresponding validation set tuples)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_ac866530aca304a384ec123e8b90a891\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_ac866530aca304a384ec123e8b90a891\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_ac866530aca304a384ec123e8b90a891\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_train_dctvustsfs__all = ib.collect(train_dctvustsfs__all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                           ASLConsultantID         TargetVideoFilename  \\\n",
       "TokenID CameraPerspective                                                \n",
       "0       0                                7   ben_story_439_small_0.mov   \n",
       "        0                                7   ben_story_439_small_0.mov   \n",
       "        0                                7   ben_story_439_small_0.mov   \n",
       "        0                                7   ben_story_439_small_0.mov   \n",
       "        0                                7   ben_story_439_small_0.mov   \n",
       "...                                    ...                         ...   \n",
       "2411    2                                3  boston-la_1088_small_2.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "        2                                3  boston-la_1088_small_2.mov   \n",
       "\n",
       "                           UtteranceSequence  TokenSequence  FrameSequence  \n",
       "TokenID CameraPerspective                                                   \n",
       "0       0                                  7              0            585  \n",
       "        0                                  7              0            586  \n",
       "        0                                  7              0            587  \n",
       "        0                                  7              0            588  \n",
       "        0                                  7              0            589  \n",
       "...                                      ...            ...            ...  \n",
       "2411    2                                 81              5          13186  \n",
       "        2                                 81              5          13187  \n",
       "        2                                 81              5          13188  \n",
       "        2                                 81              5          13189  \n",
       "        2                                 81              5          13190  \n",
       "\n",
       "[247102 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>TokenSequence</th>\n      <th>FrameSequence</th>\n    </tr>\n    <tr>\n      <th>TokenID</th>\n      <th>CameraPerspective</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">0</th>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n      <td>585</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n      <td>586</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n      <td>587</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n      <td>588</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>ben_story_439_small_0.mov</td>\n      <td>7</td>\n      <td>0</td>\n      <td>589</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">2411</th>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n      <td>13186</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n      <td>13187</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n      <td>13188</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n      <td>13189</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>boston-la_1088_small_2.mov</td>\n      <td>81</td>\n      <td>5</td>\n      <td>13190</td>\n    </tr>\n  </tbody>\n</table>\n<p>247102 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 33
    }
   ],
   "source": [
    "df_train_dctvustsfs__all.columns = ['TokenID', 'CameraPerspective', 'ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence']\n",
    "df_train_dctvustsfs__all.set_index(['TokenID', 'CameraPerspective'], inplace=True)\n",
    "df_train_dctvustsfs__all.sort_values(axis=0, by=['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'TokenSequence', 'FrameSequence'], ignore_index=False, inplace=True)\n",
    "df_train_dctvustsfs__all.sort_index(inplace=True)\n",
    "df_train_dctvustsfs__all"
   ]
  },
  {
   "source": [
    "<p><br>\n",
    "\n",
    "##### Show (complete) utterances that can be represented by token-cameraperspective tuples from the validation set"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n            <link rel=\"stylesheet\" href=\"https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css\" integrity=\"sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh\" crossorigin=\"anonymous\">\n            <div id=\"progress_indicator_07ce04c62455948a9215d8845dbaabbd\" class=\"spinner-border text-info\" role=\"status\">\n            </div>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "application/javascript": "\n        if (typeof window.interactive_beam_jquery == 'undefined') {\n          var jqueryScript = document.createElement('script');\n          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n          jqueryScript.type = 'text/javascript';\n          jqueryScript.onload = function() {\n            var datatableScript = document.createElement('script');\n            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n            datatableScript.type = 'text/javascript';\n            datatableScript.onload = function() {\n              window.interactive_beam_jquery = jQuery.noConflict(true);\n              window.interactive_beam_jquery(document).ready(function($){\n                \n            $(\"#progress_indicator_07ce04c62455948a9215d8845dbaabbd\").remove();\n              });\n            }\n            document.head.appendChild(datatableScript);\n          };\n          document.head.appendChild(jqueryScript);\n        } else {\n          window.interactive_beam_jquery(document).ready(function($){\n            \n            $(\"#progress_indicator_07ce04c62455948a9215d8845dbaabbd\").remove();\n          });\n        }"
     },
     "metadata": {}
    }
   ],
   "source": [
    "df_complete_utterances__with__val_tcp__gt__1 = ib.collect(complete_utterances__with__val_tcp__gt__1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                                                                                TokenIDSequence\n",
       "ASLConsultantID TargetVideoFilename     UtteranceSequence CameraPerspective                                    \n",
       "0               DSP%2520Immigrants.mov  4                 0                                    [846, 1512, 357]\n",
       "                                        11                0                               [846, 1512, 357, 365]\n",
       "                                        17                0                  [367, 2156, 1673, 846, 1814, 1250]\n",
       "1               640_master_small.mov    0                 0                     [380, 234, 1835, 17, 845, 2150]\n",
       "                                        3                 0                        [382, 1190, 264, 2097, 2150]\n",
       "...                                                                                                         ...\n",
       "7               ch7-607_197_small_1.mov 7                 1                                     [1210, 380, 24]\n",
       "                ch7-607_197_small_2.mov 7                 2                                     [1210, 380, 24]\n",
       "                ch7-608_198_small_0.mov 24                0                                     [1210, 380, 24]\n",
       "                ch7-608_198_small_1.mov 24                1                                     [1210, 380, 24]\n",
       "                ch7-608_198_small_2.mov 24                2                                     [1210, 380, 24]\n",
       "\n",
       "[2978 rows x 1 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th>TokenIDSequence</th>\n    </tr>\n    <tr>\n      <th>ASLConsultantID</th>\n      <th>TargetVideoFilename</th>\n      <th>UtteranceSequence</th>\n      <th>CameraPerspective</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">0</th>\n      <th rowspan=\"3\" valign=\"top\">DSP%2520Immigrants.mov</th>\n      <th>4</th>\n      <th>0</th>\n      <td>[846, 1512, 357]</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <th>0</th>\n      <td>[846, 1512, 357, 365]</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <th>0</th>\n      <td>[367, 2156, 1673, 846, 1814, 1250]</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">1</th>\n      <th rowspan=\"2\" valign=\"top\">640_master_small.mov</th>\n      <th>0</th>\n      <th>0</th>\n      <td>[380, 234, 1835, 17, 845, 2150]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <th>0</th>\n      <td>[382, 1190, 264, 2097, 2150]</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">7</th>\n      <th>ch7-607_197_small_1.mov</th>\n      <th>7</th>\n      <th>1</th>\n      <td>[1210, 380, 24]</td>\n    </tr>\n    <tr>\n      <th>ch7-607_197_small_2.mov</th>\n      <th>7</th>\n      <th>2</th>\n      <td>[1210, 380, 24]</td>\n    </tr>\n    <tr>\n      <th>ch7-608_198_small_0.mov</th>\n      <th>24</th>\n      <th>0</th>\n      <td>[1210, 380, 24]</td>\n    </tr>\n    <tr>\n      <th>ch7-608_198_small_1.mov</th>\n      <th>24</th>\n      <th>1</th>\n      <td>[1210, 380, 24]</td>\n    </tr>\n    <tr>\n      <th>ch7-608_198_small_2.mov</th>\n      <th>24</th>\n      <th>2</th>\n      <td>[1210, 380, 24]</td>\n    </tr>\n  </tbody>\n</table>\n<p>2978 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "df_complete_utterances__with__val_tcp__gt__1.columns = ['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'CameraPerspective', 'TokenIDSequence']\n",
    "df_complete_utterances__with__val_tcp__gt__1.set_index(['ASLConsultantID', 'TargetVideoFilename', 'UtteranceSequence', 'CameraPerspective'], inplace=True)\n",
    "df_complete_utterances__with__val_tcp__gt__1.sort_index(inplace=True)\n",
    "df_complete_utterances__with__val_tcp__gt__1"
   ]
  }
 ]
}