transformer_using_java_jar_library.json

{
"pipelineConfig": {
  "schemaVersion" : 6,
  "version" : 32,
  "pipelineId" : "sparkstredfed13c6-5548-48da-8879-4a722a387ef7",
  "title" : "spark_stream",
  "description" : "",
  "uuid" : "ee91d02d-cf1e-4f05-bd9e-4256652756f8",
  "configuration" : [ {
    "name" : "executionMode",
    "value" : "STREAMING"
  }, {
    "name" : "connectionString",
    "value" : null
  }, {
    "name" : "db",
    "value" : null
  }, {
    "name" : "warehouse",
    "value" : null
  }, {
    "name" : "schema",
    "value" : null
  }, {
    "name" : "edgeHttpUrl",
    "value" : "http://localhost:18633"
  }, {
    "name" : "deliveryGuarantee",
    "value" : "AT_LEAST_ONCE"
  }, {
    "name" : "testOriginStage",
    "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
  }, {
    "name" : "startEventStage",
    "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
  }, {
    "name" : "stopEventStage",
    "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
  }, {
    "name" : "shouldRetry",
    "value" : true
  }, {
    "name" : "triggerInterval",
    "value" : 5000
  }, {
    "name" : "retryAttempts",
    "value" : -1
  }, {
    "name" : "ludicrousMode",
    "value" : false
  }, {
    "name" : "ludicrousModeInputCount",
    "value" : false
  }, {
    "name" : "advancedErrorHandling",
    "value" : false
  }, {
    "name" : "notifyOnStates",
    "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
  }, {
    "name" : "emailIDs",
    "value" : [ ]
  }, {
    "name" : "constants",
    "value" : [ {
      "key" : "px",
      "value" : "'ttutuutututut'"
    } ]
  }, {
    "name" : "errorRecordPolicy",
    "value" : "ORIGINAL_RECORD"
  }, {
    "name" : "workerCount",
    "value" : 0
  }, {
    "name" : "clusterSlaveMemory",
    "value" : 2048
  }, {
    "name" : "clusterSlaveJavaOpts",
    "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
  }, {
    "name" : "clusterLauncherEnv",
    "value" : [ ]
  }, {
    "name" : "mesosDispatcherURL",
    "value" : null
  }, {
    "name" : "logLevel",
    "value" : "INFO"
  }, {
    "name" : "hdfsS3ConfDir",
    "value" : null
  }, {
    "name" : "rateLimit",
    "value" : 0
  }, {
    "name" : "maxRunners",
    "value" : 0
  }, {
    "name" : "runnerIdleTIme",
    "value" : 60
  }, {
    "name" : "shouldCreateFailureSnapshot",
    "value" : true
  }, {
    "name" : "webhookConfigs",
    "value" : [ ]
  }, {
    "name" : "sparkConfigs",
    "value" : [ {
      "key" : "spark.driver.memory",
      "value" : "2G"
    }, {
      "key" : "spark.driver.cores",
      "value" : "1"
    }, {
      "key" : "spark.executor.memory",
      "value" : "2G"
    }, {
      "key" : "spark.executor.cores",
      "value" : "1"
    }, {
      "key" : "spark.dynamicAllocation.enabled",
      "value" : "true"
    }, {
      "key" : "spark.shuffle.service.enabled",
      "value" : "true"
    }, {
      "key" : "spark.dynamicAllocation.minExecutors",
      "value" : "1"
    }, {
      "key" : "spark.jars.package",
      "value" : "org.apache.spark:spark-avro_2.12:3.2.1"
    }, {
      "key" : "spark.jars",
      "value" : "/opt/jars/lib4Spark.jar"
    }, {
      "key" : "spark.driver.extraLibraryPath",
      "value" : "/opt/jars/*.*"
    }, {
      "key" : "spark.driver.extraClassPath",
      "value" : "/opt/jars/lib4Spark.jar"
    }, {
      "key" : "spark.executor.extraClassPath",
      "value" : "/opt/jars/lib4Spark.jar"
    }, {
      "key" : "spark.executor.extraLibraryPath",
      "value" : "/opt/jars/*.*"
    } ]
  }, {
    "name" : "preprocessScript",
    "value" : "import spark.implicits._\nimport org.apache.spark.sql.functions._\nimport lib4Spark._\n\ndef Lib4SparkAdd(a: Integer, b: Integer): Integer = {\n  val x = lib4Spark.Functions.add(a,b)\n  return x\n}\nspark.udf.register (\"lib4SparkAdd\", Lib4SparkAdd _)"
  }, {
    "name" : "udfConfigBeans",
    "value" : [ ]
  }, {
    "name" : "clusterConfig.clusterType",
    "value" : "LOCAL"
  }, {
    "name" : "clusterConfig.sparkMasterUrl",
    "value" : "local[*]"
  }, {
    "name" : "clusterConfig.deployMode",
    "value" : "CLIENT"
  }, {
    "name" : "clusterConfig.hadoopUserName",
    "value" : null
  }, {
    "name" : "clusterConfig.sparkAppName",
    "value" : "${pipeline:title()}"
  }, {
    "name" : "clusterConfig.stagingDir",
    "value" : "/streamsets"
  }, {
    "name" : "clusterConfig.useYarnKerberosKeytab",
    "value" : false
  }, {
    "name" : "clusterConfig.yarnKerberosKeytabSource",
    "value" : "PROPERTIES_FILE"
  }, {
    "name" : "clusterConfig.yarnKerberosKeytab",
    "value" : null
  }, {
    "name" : "clusterConfig.yarnKerberosKeytabBase64Bytes",
    "value" : null
  }, {
    "name" : "clusterConfig.yarnKerberosPrincipal",
    "value" : "name@DOMAIN"
  }, {
    "name" : "clusterConfig.callbackUrl",
    "value" : null
  }, {
    "name" : "databricksConfig.baseUrl",
    "value" : null
  }, {
    "name" : "databricksConfig.credentialType",
    "value" : "TOKEN"
  }, {
    "name" : "databricksConfig.username",
    "value" : null
  }, {
    "name" : "databricksConfig.password",
    "value" : null
  }, {
    "name" : "databricksConfig.token",
    "value" : null
  }, {
    "name" : "databricksConfig.provisionNewCluster",
    "value" : true
  }, {
    "name" : "databricksConfig.initScriptsContents",
    "value" : [ ]
  }, {
    "name" : "databricksConfig.clusterId",
    "value" : null
  }, {
    "name" : "databricksConfig.clusterConfig",
    "value" : "{\n    \"num_workers\": 8,\n    \"spark_version\": \"5.3.x-scala2.11\",\n    \"node_type_id\": \"i3.xlarge\"\n}"
  }, {
    "name" : "databricksConfig.terminateCluster",
    "value" : false
  }, {
    "name" : "livyConfig.baseUrlBDC",
    "value" : "https://localhost:30443/gateway/default/livy/v1/"
  }, {
    "name" : "livyConfig.baseUrlHD",
    "value" : "https://localhost/livy/"
  }, {
    "name" : "livyConfig.username",
    "value" : null
  }, {
    "name" : "livyConfig.password",
    "value" : null
  }, {
    "name" : "sdcEmrConnectionSelection",
    "value" : "MANUAL"
  }, {
    "name" : "sdcEmrConnection.awsConfig.credentialMode",
    "value" : "WITH_CREDENTIALS"
  }, {
    "name" : "sdcEmrConnection.awsConfig.awsAccessKeyId",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.awsConfig.awsSecretAccessKey",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.awsConfig.roleARN",
    "value" : "arn:aws:iam::<account-id>:role/role-name"
  }, {
    "name" : "sdcEmrConnection.awsConfig.roleSessionName",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.awsConfig.sessionDuration",
    "value" : 3600
  }, {
    "name" : "sdcEmrConnection.awsConfig.setSessionTags",
    "value" : true
  }, {
    "name" : "sdcEmrConnection.region",
    "value" : "US_WEST_2"
  }, {
    "name" : "sdcEmrConnection.customRegion",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.s3StagingUri",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.provisionNewCluster",
    "value" : false
  }, {
    "name" : "sdcEmrConnection.clusterId",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.emrVersion",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.clusterPrefix",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.terminateCluster",
    "value" : true
  }, {
    "name" : "sdcEmrConnection.loggingEnabled",
    "value" : true
  }, {
    "name" : "sdcEmrConnection.s3LogUri",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.serviceRole",
    "value" : "EMR_DefaultRole"
  }, {
    "name" : "sdcEmrConnection.jobFlowRole",
    "value" : "EMR_EC2_DefaultRole"
  }, {
    "name" : "sdcEmrConnection.visibleToAllUsers",
    "value" : true
  }, {
    "name" : "sdcEmrConnection.ec2SubnetId",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.masterSecurityGroup",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.slaveSecurityGroup",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.serviceAccessSecurityGroup",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.instanceCount",
    "value" : 2
  }, {
    "name" : "sdcEmrConnection.stepConcurrency",
    "value" : 10
  }, {
    "name" : "sdcEmrConnection.masterInstanceType",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.masterInstanceTypeCustom",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.slaveInstanceType",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.slaveInstanceTypeCustom",
    "value" : null
  }, {
    "name" : "sdcEmrConnection.customTags",
    "value" : [ ]
  }, {
    "name" : "enableEMRDebugging",
    "value" : true
  }, {
    "name" : "transformerEmrConnectionSelection",
    "value" : "MANUAL"
  }, {
    "name" : "transformerEmrConnection.awsConfig.credentialMode",
    "value" : "WITH_CREDENTIALS"
  }, {
    "name" : "transformerEmrConnection.awsConfig.awsAccessKeyId",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.awsConfig.awsSecretAccessKey",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.awsConfig.isAssumeRole",
    "value" : false
  }, {
    "name" : "transformerEmrConnection.awsConfig.roleARN",
    "value" : "arn:aws:iam::<account-id>:role/role-name"
  }, {
    "name" : "transformerEmrConnection.awsConfig.roleSessionName",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.awsConfig.sessionDuration",
    "value" : 3600
  }, {
    "name" : "transformerEmrConnection.awsConfig.setSessionTags",
    "value" : true
  }, {
    "name" : "transformerEmrConnection.region",
    "value" : "US_WEST_2"
  }, {
    "name" : "transformerEmrConnection.customRegion",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.s3StagingUri",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.provisionNewCluster",
    "value" : false
  }, {
    "name" : "transformerEmrConnection.defineBootstrapActions",
    "value" : false
  }, {
    "name" : "transformerEmrConnection.bootstrapActionSource",
    "value" : "IN_S3"
  }, {
    "name" : "transformerEmrConnection.bootstrapActions",
    "value" : [ ]
  }, {
    "name" : "transformerEmrConnection.bootstrapActionScripts",
    "value" : [ ]
  }, {
    "name" : "transformerEmrConnection.clusterId",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.emrVersion",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.clusterPrefix",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.terminateCluster",
    "value" : true
  }, {
    "name" : "transformerEmrConnection.loggingEnabled",
    "value" : true
  }, {
    "name" : "transformerEmrConnection.s3LogUri",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.serviceRole",
    "value" : "EMR_DefaultRole"
  }, {
    "name" : "transformerEmrConnection.jobFlowRole",
    "value" : "EMR_EC2_DefaultRole"
  }, {
    "name" : "transformerEmrConnection.sshEc2KeyId",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.visibleToAllUsers",
    "value" : true
  }, {
    "name" : "transformerEmrConnection.ec2SubnetId",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.masterSecurityGroup",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.slaveSecurityGroup",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.serviceAccessSecurityGroup",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.instanceCount",
    "value" : 2
  }, {
    "name" : "transformerEmrConnection.stepConcurrency",
    "value" : 10
  }, {
    "name" : "transformerEmrConnection.masterInstanceType",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.masterInstanceTypeCustom",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.slaveInstanceType",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.slaveInstanceTypeCustom",
    "value" : null
  }, {
    "name" : "transformerEmrConnection.customTags",
    "value" : [ ]
  }, {
    "name" : "transformerEMRConfig.encryption",
    "value" : "NONE"
  }, {
    "name" : "transformerEMRConfig.kmsKeyId",
    "value" : null
  }, {
    "name" : "googleCloudCredentialsConfig.projectId",
    "value" : null
  }, {
    "name" : "googleCloudCredentialsConfig.credentialsProvider",
    "value" : "DEFAULT_PROVIDER"
  }, {
    "name" : "googleCloudCredentialsConfig.path",
    "value" : null
  }, {
    "name" : "googleCloudCredentialsConfig.credentialsFileContent",
    "value" : null
  }, {
    "name" : "googleCloudConfig.region",
    "value" : null
  }, {
    "name" : "googleCloudConfig.customRegion",
    "value" : null
  }, {
    "name" : "googleCloudConfig.gcsStagingUri",
    "value" : null
  }, {
    "name" : "googleCloudConfig.create",
    "value" : false
  }, {
    "name" : "googleCloudConfig.clusterPrefix",
    "value" : null
  }, {
    "name" : "googleCloudConfig.version",
    "value" : "1.4-ubuntu18"
  }, {
    "name" : "googleCloudConfig.masterType",
    "value" : null
  }, {
    "name" : "googleCloudConfig.workerType",
    "value" : null
  }, {
    "name" : "googleCloudConfig.networkType",
    "value" : null
  }, {
    "name" : "googleCloudConfig.network",
    "value" : null
  }, {
    "name" : "googleCloudConfig.subnet",
    "value" : null
  }, {
    "name" : "googleCloudConfig.tags",
    "value" : [ ]
  }, {
    "name" : "googleCloudConfig.workerCount",
    "value" : 2
  }, {
    "name" : "googleCloudConfig.clusterName",
    "value" : null
  }, {
    "name" : "googleCloudConfig.terminate",
    "value" : null
  }, {
    "name" : "googleCloudConfig.clusterLabels",
    "value" : [ ]
  }, {
    "name" : "badRecordsHandling",
    "value" : ""
  }, {
    "name" : "statsAggregatorStage",
    "value" : "streamsets-spark-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
  } ],
  "uiInfo" : {
    "schemaFromPreview" : {
      "SparkSQLExpression_01OutputLane16469242265980" : {
        "header" : { },
        "value" : {
          "type" : "LIST_MAP",
          "value" : [ {
            "type" : "STRING",
            "sqpath" : "p1",
            "dqpath" : "p1"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p2",
            "dqpath" : "p2"
          }, {
            "type" : "LONG",
            "sqpath" : "p3",
            "dqpath" : "p3"
          }, {
            "type" : "FLOAT",
            "sqpath" : "p4",
            "dqpath" : "p4"
          }, {
            "type" : "DOUBLE",
            "sqpath" : "p5",
            "dqpath" : "p5"
          }, {
            "type" : "DATE",
            "sqpath" : "p6",
            "dqpath" : "p6"
          }, {
            "type" : "DATETIME",
            "sqpath" : "p7",
            "dqpath" : "p7"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p9",
            "dqpath" : "p9"
          }, {
            "type" : "BOOLEAN",
            "sqpath" : "p10",
            "dqpath" : "p10"
          }, {
            "type" : "DECIMAL",
            "sqpath" : "p11",
            "dqpath" : "p11"
          }, {
            "type" : "BYTE_ARRAY",
            "sqpath" : "p12",
            "dqpath" : "p12"
          }, {
            "type" : "INTEGER",
            "sqpath" : "yyyy",
            "dqpath" : "yyyy"
          }, {
            "type" : "STRING",
            "sqpath" : "tttt",
            "dqpath" : "tttt"
          } ],
          "sqpath" : "",
          "dqpath" : ""
        }
      },
      "File_02OutputLane16458169067120" : {
        "header" : { },
        "value" : {
          "type" : "LIST_MAP",
          "value" : [ {
            "type" : "STRING",
            "sqpath" : "p1",
            "dqpath" : "p1"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p2",
            "dqpath" : "p2"
          }, {
            "type" : "LONG",
            "sqpath" : "p3",
            "dqpath" : "p3"
          }, {
            "type" : "FLOAT",
            "sqpath" : "p4",
            "dqpath" : "p4"
          }, {
            "type" : "DOUBLE",
            "sqpath" : "p5",
            "dqpath" : "p5"
          }, {
            "type" : "DATE",
            "sqpath" : "p6",
            "dqpath" : "p6"
          }, {
            "type" : "DATETIME",
            "sqpath" : "p7",
            "dqpath" : "p7"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p9",
            "dqpath" : "p9"
          }, {
            "type" : "BOOLEAN",
            "sqpath" : "p10",
            "dqpath" : "p10"
          }, {
            "type" : "DECIMAL",
            "sqpath" : "p11",
            "dqpath" : "p11"
          }, {
            "type" : "BYTE_ARRAY",
            "sqpath" : "p12",
            "dqpath" : "p12"
          } ],
          "sqpath" : "",
          "dqpath" : ""
        }
      },
      "Scala_01OutputLane16467660366310" : {
        "header" : { },
        "value" : {
          "type" : "LIST_MAP",
          "value" : [ {
            "type" : "STRING",
            "sqpath" : "p1",
            "dqpath" : "p1"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p2",
            "dqpath" : "p2"
          }, {
            "type" : "LONG",
            "sqpath" : "p3",
            "dqpath" : "p3"
          }, {
            "type" : "FLOAT",
            "sqpath" : "p4",
            "dqpath" : "p4"
          }, {
            "type" : "DOUBLE",
            "sqpath" : "p5",
            "dqpath" : "p5"
          }, {
            "type" : "DATE",
            "sqpath" : "p6",
            "dqpath" : "p6"
          }, {
            "type" : "DATETIME",
            "sqpath" : "p7",
            "dqpath" : "p7"
          }, {
            "type" : "INTEGER",
            "sqpath" : "p9",
            "dqpath" : "p9"
          }, {
            "type" : "BOOLEAN",
            "sqpath" : "p10",
            "dqpath" : "p10"
          }, {
            "type" : "DECIMAL",
            "sqpath" : "p11",
            "dqpath" : "p11"
          }, {
            "type" : "BYTE_ARRAY",
            "sqpath" : "p12",
            "dqpath" : "p12"
          }, {
            "type" : "INTEGER",
            "sqpath" : "yyyy",
            "dqpath" : "yyyy"
          }, {
            "type" : "STRING",
            "sqpath" : "tttt",
            "dqpath" : "tttt"
          }, {
            "type" : "INTEGER",
            "sqpath" : "xxxxxx",
            "dqpath" : "xxxxxx"
          } ],
          "sqpath" : "",
          "dqpath" : ""
        }
      }
    },
    "displayMode" : "ADVANCED",
    "schemaFromPreviewUpdateTime" : 1647439823278,
    "previewConfig" : {
      "previewSource" : "CONFIGURED_SOURCE",
      "batchSize" : 10,
      "timeout" : 120000,
      "writeToDestinations" : false,
      "executeLifecycleEvents" : false,
      "showHeader" : true,
      "showFieldType" : true,
      "rememberMe" : true,
      "timeZone" : "Browser",
      "clusterManagerType" : "CONFIGURED_CLUSTER",
      "saveSchema" : true,
      "endStage" : ""
    }
  },
  "fragments" : [ ],
  "stages" : [ {
    "instanceName" : "File_02",
    "library" : "streamsets-spark-file-lib",
    "stageName" : "com_streamsets_pipeline_spark_origin_fs_FileDOrigin",
    "stageVersion" : "2",
    "configuration" : [ {
      "name" : "configBean.fileNamePattern",
      "value" : "*"
    }, {
      "name" : "configBean.fileNameExcludePattern",
      "value" : null
    }, {
      "name" : "configBean.batchNumFileLimit",
      "value" : 0
    }, {
      "name" : "configBean.schemaConfigBean.schemaMode",
      "value" : "INFER"
    }, {
      "name" : "configBean.schemaConfigBean.customSchemaString",
      "value" : null
    }, {
      "name" : "configBean.schemaConfigBean.parseErrorMode",
      "value" : "PERMISSIVE"
    }, {
      "name" : "configBean.schemaConfigBean.columnNameOfCorruptRecord",
      "value" : null
    }, {
      "name" : "configBean.postProcessType",
      "value" : "DELETE"
    }, {
      "name" : "configBean.archiveDir",
      "value" : null
    }, {
      "name" : "configBean.dataFormatConfig.csvHeader",
      "value" : true
    }, {
      "name" : "configBean.dataFormatConfig.csvDelimiter",
      "value" : ","
    }, {
      "name" : "configBean.dataFormatConfig.csvQuote",
      "value" : "\""
    }, {
      "name" : "configBean.dataFormatConfig.csvEscape",
      "value" : "\\"
    }, {
      "name" : "configBean.dataFormatConfig.multiLineCsv",
      "value" : false
    }, {
      "name" : "configBean.dataFormatConfig.rootTag",
      "value" : "ROWS"
    }, {
      "name" : "configBean.dataFormatConfig.rowTag",
      "value" : "ROW"
    }, {
      "name" : "configBean.dataFormatConfig.multiLineJson",
      "value" : false
    }, {
      "name" : "configBean.dataFormatConfig.avroSchema",
      "value" : null
    }, {
      "name" : "configBean.dataFormatConfig.avroIgnoreExtension",
      "value" : true
    }, {
      "name" : "configBean.path",
      "value" : "/media/psf/Downloads/performance/spark/avro"
    }, {
      "name" : "configBean.dataFormat",
      "value" : "AVRO_OLD"
    }, {
      "name" : "configBean.hdfsProperties",
      "value" : [ ]
    }, {
      "name" : "noReload",
      "value" : false
    }, {
      "name" : "cache",
      "value" : false
    }, {
      "name" : "skipOffsetTracking",
      "value" : false
    } ],
    "uiInfo" : {
      "outputStreamLabels" : null,
      "yPos" : 50,
      "stageType" : "SOURCE",
      "rawSource" : null,
      "icon" : "directory.png",
      "description" : "",
      "inputStreamLabels" : null,
      "label" : "File 2",
      "xPos" : 60,
      "displayMode" : "ADVANCED"
    },
    "inputLanes" : [ ],
    "outputLanes" : [ "File_02OutputLane16458169067120" ],
    "eventLanes" : [ ],
    "services" : [ ]
  }, {
    "instanceName" : "SparkSQLExpression_01",
    "library" : "streamsets-spark-basic-lib",
    "stageName" : "com_streamsets_pipeline_spark_transform_evaluator_SQLEvaluatorDTransform",
    "stageVersion" : "2",
    "configuration" : [ {
      "name" : "config.expressions",
      "value" : [ {
        "fieldName" : "yyyy",
        "sqlExpression" : "lib4SparkAdd(p2,p9)",
        "enableCastToType" : false,
        "castToType" : "STRING"
      }, {
        "fieldName" : "tttt",
        "sqlExpression" : "${px}",
        "enableCastToType" : false,
        "castToType" : "STRING"
      } ]
    }, {
      "name" : "cache",
      "value" : false
    } ],
    "uiInfo" : {
      "outputStreamLabels" : null,
      "yPos" : 50,
      "stageType" : "PROCESSOR",
      "rawSource" : null,
      "icon" : "expression.png",
      "description" : "",
      "inputStreamLabels" : null,
      "label" : "Spark SQL Expression 1",
      "xPos" : 280.515625,
      "displayMode" : "ADVANCED"
    },
    "inputLanes" : [ "File_02OutputLane16458169067120" ],
    "outputLanes" : [ "SparkSQLExpression_01OutputLane16469242265980" ],
    "eventLanes" : [ ],
    "services" : [ ]
  }, {
    "instanceName" : "Scala_01",
    "library" : "streamsets-spark-basic-lib",
    "stageName" : "com_streamsets_pipeline_spark_transform_scala_ScalaDTransform",
    "stageVersion" : "3",
    "configuration" : [ {
      "name" : "code",
      "value" : "import spark.implicits._\nimport org.apache.spark.sql.functions._\nimport lib4Spark._\n\noutput = inputs(0).withColumn(\"xxxxxx\",typedLit(lib4Spark.Functions.add(5,5)))"
    }, {
      "name" : "skipEmptyBatches",
      "value" : true
    } ],
    "uiInfo" : {
      "outputStreamLabels" : null,
      "yPos" : 50,
      "stageType" : "PROCESSOR",
      "rawSource" : null,
      "icon" : "scala-spiral.png",
      "description" : "",
      "inputStreamLabels" : null,
      "label" : "Scala 1",
      "xPos" : 499,
      "displayMode" : "ADVANCED"
    },
    "inputLanes" : [ "SparkSQLExpression_01OutputLane16469242265980" ],
    "outputLanes" : [ "Scala_01OutputLane16467660366310" ],
    "eventLanes" : [ ],
    "services" : [ ]
  }, {
    "instanceName" : "Trash_01",
    "library" : "streamsets-spark-basic-lib",
    "stageName" : "com_streamsets_pipeline_spark_destination_dev_NullDTarget",
    "stageVersion" : "1",
    "configuration" : [ ],
    "uiInfo" : {
      "outputStreamLabels" : null,
      "yPos" : 50,
      "stageType" : "TARGET",
      "rawSource" : null,
      "icon" : "trash.png",
      "description" : "",
      "inputStreamLabels" : null,
      "label" : "Trash 1",
      "xPos" : 720,
      "displayMode" : "ADVANCED"
    },
    "inputLanes" : [ "Scala_01OutputLane16467660366310" ],
    "outputLanes" : [ ],
    "eventLanes" : [ ],
    "services" : [ ]
  } ],
  "errorStage" : null,
  "info" : {
    "pipelineId" : "sparkstredfed13c6-5548-48da-8879-4a722a387ef7",
    "title" : "spark_stream",
    "description" : "",
    "created" : 1645808236276,
    "lastModified" : 1645808236276,
    "creator" : "admin@sch-cs-jeanmarie",
    "lastModifier" : "admin@sch-cs-jeanmarie",
    "lastRev" : "0",
    "uuid" : "ee91d02d-cf1e-4f05-bd9e-4256652756f8",
    "valid" : false,
    "metadata" : null,
    "name" : "sparkstredfed13c6-5548-48da-8879-4a722a387ef7",
    "sdcVersion" : "4.2.0",
    "sdcId" : "58d57441-8b49-11ec-8e52-8b0b82ac055b"
  },
  "metadata" : {
    "dpm.pipeline.rules.id" : "9aac523b-ab43-4eaa-bc04-2719295e1054:sch-cs-jeanmarie",
    "dpm.pipeline.id" : "965ca4fe-e0ef-4c96-81b0-08073cc7e0e2:sch-cs-jeanmarie",
    "dpm.base.url" : "https://trailer.streamsetscloud.com",
    "dpm.pipeline.version" : "2",
    "dpm.pipeline.commit.id" : "d6b86f35-e777-4735-b51d-d47fa8a234f1:sch-cs-jeanmarie",
    "dpm.pipeline.connections" : "",
    "labels" : [ "architecture" ]
  },
  "statsAggregatorStage" : {
    "instanceName" : "WritetoControlHubdirectly_ErrorStage",
    "library" : "streamsets-spark-basic-lib",
    "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
    "stageVersion" : "1",
    "configuration" : [ ],
    "uiInfo" : {
      "outputStreamLabels" : null,
      "yPos" : 50,
      "stageType" : "TARGET",
      "rawSource" : null,
      "icon" : "",
      "description" : "",
      "inputStreamLabels" : null,
      "label" : "Error Records - Write to Control Hub directly",
      "xPos" : 60
    },
    "inputLanes" : [ ],
    "outputLanes" : [ ],
    "eventLanes" : [ ],
    "services" : [ ]
  },
  "startEventStages" : [ ],
  "stopEventStages" : [ ],
  "testOriginStage" : null,
  "valid" : true,
  "issues" : {
    "pipelineIssues" : [ ],
    "stageIssues" : { },
    "issueCount" : 0
  },
  "previewable" : true
},
"pipelineRules": {"schemaVersion":3,"version":2,"metricsRuleDefinitions":[{"id":"badRecordsAlertID","alertText":"High incidence of Error Records","metricId":"pipeline.batchErrorRecords.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1645808236276,"valid":true},{"id":"stageErrorAlertID","alertText":"High incidence of Stage Errors","metricId":"pipeline.batchErrorMessages.counter","metricType":"COUNTER","metricElement":"COUNTER_COUNT","condition":"${value() > 100}","sendEmail":false,"enabled":false,"timestamp":1645808236276,"valid":true},{"id":"idleGaugeID","alertText":"Pipeline is Idle","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"TIME_OF_LAST_RECEIVED_RECORD","condition":"${time:now() - value() > 120000}","sendEmail":false,"enabled":false,"timestamp":1645808236276,"valid":true},{"id":"batchTimeAlertID","alertText":"Batch taking more time to process","metricId":"RuntimeStatsGauge.gauge","metricType":"GAUGE","metricElement":"CURRENT_BATCH_AGE","condition":"${value() > 200}","sendEmail":false,"enabled":false,"timestamp":1645808236276,"valid":true}],"dataRuleDefinitions":[],"driftRuleDefinitions":[],"uuid":"9fd65f21-a674-42eb-9766-3ce17dccbfcb","configuration":[{"name":"emailIDs","value":[]},{"name":"webhookConfigs","value":[]}],"configIssues":[],"ruleIssues":[]},
"libraryDefinitions": {"schemaVersion":"1","pipeline":[{"configDefinitions":[{"fieldName":"executionMode","label":"Execution Mode","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Standalone","Cluster Batch","Cluster Yarn Streaming","Cluster Mesos Streaming","Edge","Cluster EMR Batch","Batch","Streaming","Snowpark"],"values":["STANDALONE","CLUSTER_BATCH","CLUSTER_YARN_STREAMING","CLUSTER_MESOS_STREAMING","EDGE","EMR_BATCH","BATCH","STREAMING","SNOWPARK"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.ExecutionModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"executionMode","type":"MODEL","defaultValue":"STANDALONE"},{"fieldName":"connectionString","label":"Snowflake URL","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["SNOWPARK"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"connectionString","type":"STRING","defaultValue":null},{"fieldName":"db","label":"Database","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["SNOWPARK"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"db","type":"STRING","defaultValue":null},{"fieldName":"warehouse","label":"Warehouse","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["SNOWPARK"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":35,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"warehouse","type":"STRING","defaultValue":null},{"fieldName":"schema","label":"Schema","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["SNOWPARK"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"schema","type":"STRING","defaultValue":null},{"fieldName":"edgeHttpUrl","label":"Data Collector Edge URL","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"edgeHttpUrl","type":"STRING","defaultValue":"http://localhost:18633"},{"fieldName":"deliveryGuarantee","label":"Delivery Guarantee","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["At Least Once","At Most Once"],"values":["AT_LEAST_ONCE","AT_MOST_ONCE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"deliveryGuarantee","type":"MODEL","defaultValue":"AT_LEAST_ONCE"},{"fieldName":"testOriginStage","label":"Test Origin","description":"Stage used for testing in preview mode.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["ADLS Gen1 (Library: ADLS Transformer-provided libraries for Hadoop 3.2.0)","ADLS Gen1 (Library: ADLS cluster-provided libraries)","ADLS Gen2 (Library: ADLS Transformer-provided libraries for Hadoop 3.2.0)","ADLS Gen2 (Library: ADLS cluster-provided libraries)","Amazon Redshift (Library: AWS Redshift cluster-provided libraries)","Amazon S3 (Library: AWS Transformer-provided libraries for Hadoop 2.7.7)","Amazon S3 (Library: AWS Transformer-provided libraries for Hadoop 3.2.0)","Amazon S3 (Library: AWS cluster-provided libraries)","Azure Event Hubs (Library: Azure Event Hubs)","Delta Lake (Library: Delta Lake Transformer-provided libraries)","Delta Lake (Library: Delta Lake cluster-provided libraries)","Dev Random (Library: Basic)","Dev Raw Data Source (Library: Basic)","File (Library: File)","Fragment Origin (Library: Basic)","Google Big Query (Library: Google Cloud cluster-provided libraries)","Google Cloud Storage (Library: Google Cloud cluster-provided libraries)","Hive (Library: Hive)","JDBC Query (Library: JDBC)","JDBC Table (Library: JDBC)","Kafka (Library: Kafka Transformer-provided libraries)","Kafka (Library: Kafka cluster-provided libraries)","Kudu (Library: Kudu)","MapR Event Store (Library: MapR Event Store libraries)","MapR FS (Library: MapR FS libraries)","MapR Hive (Library: MapR Hive)","MySQL JDBC Table (Library: JDBC)","Oracle JDBC Table (Library: JDBC)","PostgreSQL JDBC Table (Library: JDBC)","Snowflake (Library: Snowflake Transformer-provided libraries)","Snowflake (Library: Snowflake cluster-provided libraries)","SQL Server JDBC Table (Library: JDBC)","Whole Directory (Library: File)"],"values":["streamsets-spark-azure_3_2_0-lib::com_streamsets_pipeline_spark_origin_azure_ADLSGen1DOrigin::2","streamsets-spark-azure-no-dependency-lib::com_streamsets_pipeline_spark_origin_azure_ADLSGen1DOrigin::2","streamsets-spark-azure_3_2_0-lib::com_streamsets_pipeline_spark_origin_azure_ADLSGen2DOrigin::3","streamsets-spark-azure-no-dependency-lib::com_streamsets_pipeline_spark_origin_azure_ADLSGen2DOrigin::3","streamsets-spark-aws-redshift-no-dependency-lib::com_streamsets_pipeline_spark_origin_redshift_RedshiftDOrigin::1","streamsets-spark-aws_2_7_7-lib::com_streamsets_pipeline_spark_origin_s3_AmazonS3DOrigin::4","streamsets-spark-aws_3_2_0-lib::com_streamsets_pipeline_spark_origin_s3_AmazonS3DOrigin::4","streamsets-spark-aws-no-dependency-lib::com_streamsets_pipeline_spark_origin_s3_AmazonS3DOrigin::4","streamsets-spark-azure-eventhub-spark-lib::com_streamsets_pipeline_spark_origin_eventhub_AzureEventHubDOrigin::1","streamsets-spark-delta-lake-lib::com_streamsets_pipeline_spark_origin_delta_DeltaLakeDOrigin::2","streamsets-spark-delta-lake-with-no-dependency-lib::com_streamsets_pipeline_spark_origin_delta_DeltaLakeDOrigin::2","streamsets-spark-basic-lib::com_streamsets_pipeline_spark_origin_dev_DevRandomDOrigin::1","streamsets-spark-basic-lib::com_streamsets_pipeline_spark_origin_dev_RawDataDOrigin::2","streamsets-spark-file-lib::com_streamsets_pipeline_spark_origin_fs_FileDOrigin::2","streamsets-spark-basic-lib::com_streamsets_pipeline_stage_origin_fragment_FragmentSource::1","streamsets-spark-google-cloud-no-dependency-lib::com_streamsets_pipeline_spark_origin_gcloud_bigquery_BigQueryDOrigin::1","streamsets-spark-google-cloud-no-dependency-lib::com_streamsets_pipeline_spark_origin_gcloud_GoogleCloudStorageDOrigin::2","streamsets-spark-hive-lib::com_streamsets_pipeline_spark_origin_hive_HiveDOrigin::1","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_query_JdbcQueryDOrigin::1","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_SparkJDBCDOrigin::3","streamsets-spark-kafka-lib::com_streamsets_pipeline_spark_origin_kafka_KafkaDOrigin::4","streamsets-spark-kafka-no-dependency-lib::com_streamsets_pipeline_spark_origin_kafka_KafkaDOrigin::4","streamsets-spark-kudu-lib::com_streamsets_pipeline_spark_origin_kudu_KuduDOrigin::2","streamsets-spark-mapr-eventstore-lib::com_streamsets_pipeline_spark_origin_eventstore_EventStoreDOrigin::2","streamsets-spark-mapr-fs-lib::com_streamsets_pipeline_spark_origin_maprfs_MapRFSDOrigin::1","streamsets-spark-mapr-hive-lib::com_streamsets_pipeline_spark_origin_hive_MapRHiveDOrigin::1","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_table_branded_mysql_MySQLJdbcTableDOrigin::2","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_table_branded_oracle_OracleJdbcTableDOrigin::2","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_table_branded_postgresql_PostgreJdbcTableDOrigin::2","streamsets-spark-snowflake-lib::com_streamsets_pipeline_spark_origin_snowflake_SnowflakeDOrigin::6","streamsets-spark-snowflake-with-no-dependency-lib::com_streamsets_pipeline_spark_origin_snowflake_SnowflakeDOrigin::6","streamsets-spark-jdbc-lib::com_streamsets_pipeline_spark_origin_jdbc_table_branded_sqlserver_SqlServerJdbcTableDOrigin::2","streamsets-spark-file-lib::com_streamsets_pipeline_spark_origin_fs_WholeDirectoryDOrigin::2"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.PipelineTestStageChooserValues","filteringConfig":""},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":21,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"testOriginStage","type":"MODEL","defaultValue":"streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"},{"fieldName":"startEventStage","label":"Start Event","description":"Stage that should handle pipeline start event.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Discard (Library: Basic)"],"values":["streamsets-spark-basic-lib::com_streamsets_pipeline_spark_destination_dev_ToErrorNullDTarget::1"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":""},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":23,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"startEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"fieldName":"stopEventStage","label":"Stop Event","description":"Stage that should handle pipeline stop event.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Discard (Library: Basic)"],"values":["streamsets-spark-basic-lib::com_streamsets_pipeline_spark_destination_dev_ToErrorNullDTarget::1"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues","filteringConfig":""},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":26,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"stopEventStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"fieldName":"shouldRetry","label":"Retry Pipeline on Error","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"shouldRetry","type":"BOOLEAN","defaultValue":true},{"fieldName":"triggerInterval","label":"Trigger Interval (millis)","description":"Time interval between generation of batches","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":1,"dependsOnMap":{"executionMode":["STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":35,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"triggerInterval","type":"NUMBER","defaultValue":2000},{"fieldName":"retryAttempts","label":"Retry Attempts","description":"Max no of retries. To retry indefinitely, use -1. The wait time between retries starts at 15 seconds and doubles until reaching 5 minutes.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"shouldRetry":[true],"executionMode":["STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"retryAttempts","type":"NUMBER","defaultValue":-1},{"fieldName":"ludicrousMode","label":"Enable Ludicrous Mode","description":"Ludicrous mode may significantly improve performance, but metrics will be limited","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":40,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"ludicrousMode","type":"BOOLEAN","defaultValue":false},{"fieldName":"ludicrousModeInputCount","label":"Collect Input Metrics","description":"Collects and displays input metrics. Can result in rereading data unless origins are configured to cache data","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"],"ludicrousMode":[true]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"ludicrousModeInputCount","type":"BOOLEAN","defaultValue":false},{"fieldName":"advancedErrorHandling","label":"Advanced Error Handling","description":"Reports the record that generates an error, when possible. Supported in single-origin pipelines","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":60,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"advancedErrorHandling","type":"BOOLEAN","defaultValue":false},{"fieldName":"notifyOnStates","label":"Notify on Pipeline State Changes","description":"Notifies via email when pipeline gets to the specified states","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Running","Start Error","Run Error","Running Error","Stopped","Finished","Disconnected","Connecting","Stop Error"],"values":["RUNNING","START_ERROR","RUN_ERROR","RUNNING_ERROR","STOPPED","FINISHED","DISCONNECTED","CONNECTING","STOP_ERROR"],"configDefinitions":null,"modelType":"MULTI_VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.PipelineStateChooserValues","filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":75,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"notifyOnStates","type":"MODEL","defaultValue":["RUN_ERROR","STOPPED","FINISHED"]},{"fieldName":"emailIDs","label":"Email IDs","description":"Email Addresses","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":76,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"emailIDs","type":"LIST","defaultValue":[]},{"fieldName":"constants","label":"Parameters","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"PARAMETERS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"constants","type":"MAP","defaultValue":[]},{"fieldName":"badRecordsHandling","label":"Error Records","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Discard (Library: Basic)"],"values":["streamsets-spark-basic-lib::com_streamsets_pipeline_spark_destination_dev_ToErrorNullDTarget::1"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.ErrorHandlingChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"BAD_RECORDS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"badRecordsHandling","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"},{"fieldName":"errorRecordPolicy","label":"Error Record Policy","description":"Determines which variation of the record is sent to error.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Original record as it was generated by the origin.","Record as it was seen by the stage that sent it to error stream."],"values":["ORIGINAL_RECORD","STAGE_RECORD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"BAD_RECORDS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":93,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"errorRecordPolicy","type":"MODEL","defaultValue":"ORIGINAL_RECORD"},{"fieldName":"statsAggregatorStage","label":"Statistics Aggregator","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Discard (Library: Basic)","Write to Control Hub directly (Library: Basic)"],"values":["streamsets-spark-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1","streamsets-spark-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.StatsTargetChooserValues","filteringConfig":""},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EDGE","STREAMING","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"STATS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":95,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"statsAggregatorStage","type":"MODEL","defaultValue":"streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"},{"fieldName":"workerCount","label":"Worker Count","description":"Number of workers. 0 to start as many workers as Kafka partitions for topic.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":0,"dependsOnMap":{"executionMode":["CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"workerCount","type":"NUMBER","defaultValue":0},{"fieldName":"clusterSlaveMemory","label":"Worker Memory (MB)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterSlaveMemory","type":"NUMBER","defaultValue":2048},{"fieldName":"clusterSlaveJavaOpts","label":"Worker Java Options","description":"Add properties as needed. Changes to default settings are not recommended.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_BATCH","EMR_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":110,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterSlaveJavaOpts","type":"STRING","defaultValue":"-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"},{"fieldName":"clusterLauncherEnv","label":"Launcher ENV","description":"Sets additional environment variables for the cluster launcher","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterLauncherEnv","type":"MAP","defaultValue":[]},{"fieldName":"mesosDispatcherURL","label":"Mesos Dispatcher URL","description":"URL for service which launches Mesos framework","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"mesosDispatcherURL","type":"STRING","defaultValue":null},{"fieldName":"logLevel","label":"Log Level","description":"Log level to use for the launched application","max":9223372036854776000,"mode":"text/plain","model":{"labels":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"],"values":["OFF","FATAL","ERROR","WARN","INFO","DEBUG","TRACE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.LogLevelChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH","EMR_BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":140,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"logLevel","type":"MODEL","defaultValue":"INFO"},{"fieldName":"hdfsS3ConfDir","label":"Checkpoint Configuration Directory","description":"An SDC resource directory or symbolic link with HDFS/S3 configuration files core-site.xml and hdfs-site.xml","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["CLUSTER_MESOS_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"hdfsS3ConfDir","type":"STRING","defaultValue":null},{"fieldName":"rateLimit","label":"Rate Limit (records / sec)","description":"Maximum number of records per second that should be accepted into the pipeline. Rate is not limited if this is not set, or is set to 0","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":180,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"rateLimit","type":"NUMBER","defaultValue":0},{"fieldName":"maxRunners","label":"Max Runners","description":"Maximum number of runners that should be created for this pipeline. Use 0 to not impose limit.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":0,"dependsOnMap":{"executionMode":["STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":190,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"maxRunners","type":"NUMBER","defaultValue":0},{"fieldName":"runnerIdleTIme","label":"Runner Idle Time (sec)","description":"When pipeline runners are idle for at least this time, run an empty batch through the runner to process any events or other time-driven functionality. Value -1 will disable this functionality completely.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"runnerIdleTIme","type":"NUMBER","defaultValue":60},{"fieldName":"shouldCreateFailureSnapshot","label":"Create Failure Snapshot","description":"Attempts to create a partial snapshot with unprocessed records if the pipeline fails","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STANDALONE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":205,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"shouldCreateFailureSnapshot","type":"BOOLEAN","defaultValue":true},{"fieldName":"webhookConfigs","label":"Webhooks","description":"Webhooks","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"payload","label":"Payload","description":"Data that should be included as a part of the Webhook request","max":9223372036854776000,"mode":"application/json","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["DELETE","POST","PUT"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":2,"name":"payload","type":"TEXT","defaultValue":"{\n  \"text\" : \"Pipeline '{{PIPELINE_TITLE}}' state changed to {{PIPELINE_STATE}} at {{TIME}}. \\n <{{PIPELINE_URL}}|Click here> for details!\",\n  \"fields\": [\n    \t{\n\t\t\t\"title\": \"Pipeline Status Message\", \n\t\t\t\"value\": \"{{PIPELINE_STATE_MESSAGE}}\",\n      \t\t\"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Input Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_INPUT_RECORDS_COUNT}}\",\n            \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Output Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_OUTPUT_RECORDS_COUNT}}\",\n            \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_RECORDS_COUNT}}\",\n            \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Messages Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_MESSAGES_COUNT}}\",\n            \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Runtime Parameters\", \n\t\t\t\"value\": \"{{PIPELINE_RUNTIME_PARAMETERS}}\",\n          \t\"short\": true\n\t\t}\n\t]\n}"},{"fieldName":"webhookUrl","label":"Webhook URL","description":"The Webhook HTTP resource URL","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"webhookUrl","type":"STRING","defaultValue":null},{"fieldName":"headers","label":"Headers","description":"Headers to include in the request","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"headers","type":"MAP","defaultValue":[]},{"fieldName":"httpMethod","label":"HTTP Method","description":"HTTP method to send","max":9223372036854776000,"mode":"text/plain","model":{"labels":["GET","PUT","POST","DELETE","HEAD"],"values":["GET","PUT","POST","DELETE","HEAD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"httpMethod","type":"MODEL","defaultValue":"POST"},{"fieldName":"contentType","label":"Content Type","description":"Content-Type header to be sent with the request; used if that header is not already present","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["DELETE","POST","PUT"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"EXPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"contentType","type":"STRING","defaultValue":"application/json"},{"fieldName":"authType","label":"Authentication Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["None","Basic","Digest","Universal"],"values":["NONE","BASIC","DIGEST","UNIVERSAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"authType","type":"MODEL","defaultValue":"NONE"},{"fieldName":"username","label":"Username","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","BASIC","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"username","type":"CREDENTIAL","defaultValue":null},{"fieldName":"password","label":"Password","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","BASIC","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"password","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","STANDALONE","CLUSTER_MESOS_STREAMING","CLUSTER_BATCH","BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":210,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"webhookConfigs","type":"MODEL","defaultValue":[]},{"fieldName":"sparkConfigs","label":"Extra Spark Configuration","description":"Additional Spark Configuration to pass to the spark-submit script, the parameters will be passed as --conf <key>=<value>","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","CLUSTER_BATCH","BATCH","CLUSTER_YARN_STREAMING"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":220,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sparkConfigs","type":"MAP","defaultValue":[]},{"fieldName":"preprocessScript","label":"Preprocessing Script","description":"Scala script to run on the driver before starting the pipeline. Can be used to register user defined functions, etc. Use the 'spark' variable to access the Spark session","max":9223372036854776000,"mode":"text/x-scala","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"preprocessScript","type":"TEXT","defaultValue":"/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n  i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"},{"fieldName":"udfConfigBeans","label":"UDF Configs","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"uploadType","label":"UDF Definition Mode","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Inline","Jar File"],"values":["INLINE","JAR"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.creation.udf.JavaUDFTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"uploadType","type":"MODEL","defaultValue":null},{"fieldName":"sqlName","label":"SQL UDF Name","description":"Name used to refer to this function from SQL/Snowpark","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sqlName","type":"STRING","defaultValue":null},{"fieldName":"javaFunctionName","label":"Java Handler Function (must be static)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"javaFunctionName","type":"STRING","defaultValue":null},{"fieldName":"jarTargetPath","label":"Snowflake Target Path for Jar","description":"Must be in a Snowflake stage accessible to this user. Only include the path, jar name is automatically generated.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"jarTargetPath","type":"STRING","defaultValue":"@~/"},{"fieldName":"javaUDF","label":"Java UDF","description":"Define a Java class with the handler function and any other functions and state required. Include all necessary import statements","max":9223372036854776000,"mode":"text/x-java","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"uploadType":["INLINE"]},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"javaUDF","type":"TEXT","defaultValue":null},{"fieldName":"jarPath","label":"Local Path to JAR File","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"uploadType":["JAR"]},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"jarPath","type":"STRING","defaultValue":"@~/"}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["SNOWPARK"]},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"udfConfigBeans","type":"MODEL","defaultValue":[]},{"fieldName":"clusterType","label":"Cluster Manager Type","description":"Type of cluster manager used by the Spark cluster","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Apache Spark for HDInsight","Databricks","EMR","Dataproc","Hadoop YARN","None (local)","Spark Standalone Cluster","SQL Server 2019 Big Data Cluster"],"values":["AZURE_HD_INSIGHT","DATABRICKS","EMR","DATAPROC","YARN","LOCAL","STANDALONE_SPARK_CLUSTER","SQL_SERVER_BIG_DATA_CLUSTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.SparkClusterTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":101,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.clusterType","type":"MODEL","defaultValue":"LOCAL"},{"fieldName":"sparkMasterUrl","label":"Master URL","description":"Master URL to connect to Spark","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["LOCAL","STANDALONE_SPARK_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":102,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.sparkMasterUrl","type":"STRING","defaultValue":"local[*]"},{"fieldName":"deployMode","label":"Deployment Mode","description":"Mode used to launch the Spark driver process","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Client","Cluster"],"values":["CLIENT","CLUSTER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.SparkDeployModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["STANDALONE_SPARK_CLUSTER","KUBERNETES","YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":103,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterConfig.deployMode","type":"MODEL","defaultValue":"CLIENT"},{"fieldName":"hadoopUserName","label":"Hadoop User Name","description":"Name of the Hadoop user that StreamSets impersonates","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":104,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterConfig.hadoopUserName","type":"STRING","defaultValue":null},{"fieldName":"sparkAppName","label":"Application Name","description":"Name of the launched Spark application","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterConfig.sparkAppName","type":"STRING","defaultValue":"${pipeline:title()}"},{"fieldName":"stagingDir","label":"Staging Directory","description":"Staging directory on the remote system for copying StreamSets resources","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","DATABRICKS","EMR","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":106,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterConfig.stagingDir","type":"STRING","defaultValue":"/streamsets"},{"fieldName":"useYarnKerberosKeytab","label":"Use YARN Kerberos Keytab","description":"Indicates that the Spark application should use a principal and keytab for Kerberos authentication","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.useYarnKerberosKeytab","type":"BOOLEAN","defaultValue":false},{"fieldName":"yarnKerberosKeytabSource","label":"Keytab Source","description":"Source for the Kerberos keytab used to launch the Spark application","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Transformer Configuration File","Pipeline Configuration - File","Pipeline Configuration - Credential Store"],"values":["PROPERTIES_FILE","PIPELINE","PIPELINE_CREDENTIAL_STORE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.KeytabSourceChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1050,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.yarnKerberosKeytabSource","type":"MODEL","defaultValue":"PROPERTIES_FILE"},{"fieldName":"yarnKerberosKeytab","label":"YARN Kerberos Keytab Path","description":"Absolute path to the Kerberos keytab used to launch the Spark application for the pipeline","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.yarnKerberosKeytab","type":"STRING","defaultValue":null},{"fieldName":"yarnKerberosKeytabBase64Bytes","label":"Keytab Credential Function","description":"Credential function to retrieve the base64 encoded keytab from a credential store.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE_CREDENTIAL_STORE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":1150,"connectionType":"","displayMode":"BASIC","upload":"BASE64","lines":0,"name":"clusterConfig.yarnKerberosKeytabBase64Bytes","type":"CREDENTIAL","defaultValue":null},{"fieldName":"yarnKerberosPrincipal","label":"YARN Kerberos Principal","description":"Name of the Kerberos principal used to launch the Spark application for the pipeline.  Must be present in the keytab specified above.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.yarnKerberosKeytabSource":["PIPELINE_CREDENTIAL_STORE","PIPELINE"],"clusterConfig.clusterType":["YARN"],"executionMode":["STREAMING","BATCH"],"clusterConfig.useYarnKerberosKeytab":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"clusterConfig.yarnKerberosPrincipal","type":"STRING","defaultValue":"name@DOMAIN"},{"fieldName":"callbackUrl","label":"Cluster Callback URL","description":"Optional callback URL for the Spark cluster to use to contact this Transformer instance. Use this format: http://<hostname>:<port>","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"ADVANCED","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"clusterConfig.callbackUrl","type":"STRING","defaultValue":null},{"fieldName":"baseUrl","label":"URL to connect to Databricks","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.baseUrl","type":"STRING","defaultValue":null},{"fieldName":"credentialType","label":"Credential Type","description":"Type of credential to use to connect to Databricks","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Username/Password","Token"],"values":["PASSWORD","TOKEN"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.CredentialTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":106,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.credentialType","type":"MODEL","defaultValue":"TOKEN"},{"fieldName":"username","label":"Username","description":"Databricks user name","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.username","type":"CREDENTIAL","defaultValue":null},{"fieldName":"password","label":"Password","description":"Password for the account","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["PASSWORD"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.password","type":"CREDENTIAL","defaultValue":null},{"fieldName":"token","label":"Token","description":"Personal access token for the account","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"],"databricksConfig.credentialType":["TOKEN"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":109,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.token","type":"CREDENTIAL","defaultValue":null},{"fieldName":"provisionNewCluster","label":"Provision a New Cluster","description":"Provisions a new cluster when the pipeline starts","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.provisionNewCluster","type":"BOOLEAN","defaultValue":true},{"fieldName":"initScriptsContents","label":"Init Scripts","description":"Databricks cluster-scoped init scripts to use","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"scriptType","label":"Script Type","description":"Script type and location","max":9223372036854776000,"mode":"text/plain","model":{"labels":["DBFS from pipeline","DBFS from location","S3 from location","ABFSS from location"],"values":["DBFS_PIPELINE","DBFS_LOCATION","S3_LOCATION","ABFSS_LOCATION"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.InitScriptChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":5,"name":"scriptType","type":"MODEL","defaultValue":"DBFS_PIPELINE"},{"fieldName":"script","label":"DBFS Script","description":"Content of a DBFS init script","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"scriptType":["DBFS_PIPELINE"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":5,"name":"script","type":"TEXT","defaultValue":null},{"fieldName":"scriptLocationDBFS","label":"DBFS Script Location","description":"Location of an existing DBFS script","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"scriptType":["DBFS_LOCATION"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"scriptLocationDBFS","type":"STRING","defaultValue":null},{"fieldName":"scriptLocationS3","label":"S3 Script Location","description":"Location of an existing S3 script","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"scriptType":["S3_LOCATION"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"scriptLocationS3","type":"STRING","defaultValue":null},{"fieldName":"scriptLocationAbfss","label":"ABFSS Script Location","description":"Location of an existing ABFSS script","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"scriptType":["ABFSS_LOCATION"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"scriptLocationAbfss","type":"STRING","defaultValue":null},{"fieldName":"region","label":"AWS Region","description":"AWS region for the script","max":9223372036854776000,"mode":"text/plain","model":{"labels":["US East (N. Virginia) us-east-1","US East (Ohio) us-east-2","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US-East) us-gov-east-1","AWS GovCloud (US-West) us-gov-west-1","Africa (Cape Town) af-south-1","Asia Pacific (Hong Kong) ap-east-1","Asia Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","Europe (Milan) eu-south-1","Europe (Stockholm) eu-north-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","Middle East (Bahrain) me-south-1","South America (São Paulo) sa-east-1","Other - specify"],"values":["US_EAST_1","US_EAST_2","US_WEST_1","US_WEST_2","US_GOV_EST_1","US_GOV_WEST_1","AF_SOUTH_1","AP_EAST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_SOUTH_1","EU_NORTH_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","ME_SOUTH_1","SA_EAST_1","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":""},"required":false,"min":-9223372036854776000,"dependsOnMap":{"scriptType":["S3_LOCATION"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"region","type":"MODEL","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":true,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":111,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.initScriptsContents","type":"MODEL","defaultValue":[]},{"fieldName":"clusterId","label":"Cluster ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[false],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":112,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"databricksConfig.clusterId","type":"STRING","defaultValue":null},{"fieldName":"clusterConfig","label":"Cluster Configuration","description":"Configuration for the Databricks cluster","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":113,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"databricksConfig.clusterConfig","type":"TEXT","defaultValue":"{\n    \"num_workers\": 8,\n    \"spark_version\": \"5.3.x-scala2.11\",\n    \"node_type_id\": \"i3.xlarge\"\n}"},{"fieldName":"terminateCluster","label":"Terminate Cluster","description":"Terminates the cluster when the pipeline stops","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"databricksConfig.provisionNewCluster":[true],"clusterConfig.clusterType":["DATABRICKS"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":114,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"databricksConfig.terminateCluster","type":"BOOLEAN","defaultValue":false},{"fieldName":"baseUrlBDC","label":"Livy Endpoint","description":"URL to connect to the Livy Server","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"livyConfig.baseUrlBDC","type":"STRING","defaultValue":"https://localhost:30443/gateway/default/livy/v1/"},{"fieldName":"baseUrlHD","label":"Livy Endpoint","description":"URL to connect to the Livy Server","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"livyConfig.baseUrlHD","type":"STRING","defaultValue":"https://localhost/livy/"},{"fieldName":"username","label":"Username","description":"Livy user name","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":107,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"livyConfig.username","type":"CREDENTIAL","defaultValue":null},{"fieldName":"password","label":"Password","description":"Password for the account","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["AZURE_HD_INSIGHT","SQL_SERVER_BIG_DATA_CLUSTER"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":108,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"livyConfig.password","type":"CREDENTIAL","defaultValue":null},{"fieldName":"sdcEmrConnectionSelection","label":"Connection","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["None"],"values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-500,"connectionType":"NOT-IMPLEMENTED-SDC-EMR-CONNECTION","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"fieldName":"credentialMode","label":"Authentication Method","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["AWS Keys","Instance Profile","None"],"values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS"},{"fieldName":"awsAccessKeyId","label":"Access Key ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null},{"fieldName":"awsSecretAccessKey","label":"Secret Access Key","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null},{"fieldName":"roleARN","label":"Role ARN","description":"Amazon resource name of the role to assume","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name"},{"fieldName":"roleSessionName","label":"Role Session Name","description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null},{"fieldName":"sessionDuration","label":"Session Timeout (secs)","description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","max":43200,"mode":"text/plain","model":null,"required":true,"min":3600,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600},{"fieldName":"setSessionTags","label":"Set Session Tags","description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"sdcEmrConnection.awsConfig.isAssumeRole":[true],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true},{"fieldName":"region","label":"AWS Region","description":"The AWS region for EMR.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["US East (N. Virginia) us-east-1","US East (Ohio) us-east-2","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US-East) us-gov-east-1","AWS GovCloud (US-West) us-gov-west-1","Africa (Cape Town) af-south-1","Asia Pacific (Hong Kong) ap-east-1","Asia Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","Europe (Milan) eu-south-1","Europe (Stockholm) eu-north-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","Middle East (Bahrain) me-south-1","South America (São Paulo) sa-east-1","Other - specify"],"values":["US_EAST_1","US_EAST_2","US_WEST_1","US_WEST_2","US_GOV_EST_1","US_GOV_WEST_1","AF_SOUTH_1","AP_EAST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_SOUTH_1","EU_NORTH_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","ME_SOUTH_1","SA_EAST_1","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2"},{"fieldName":"customRegion","label":"Endpoint","description":"Custom endpoint","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.region":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.customRegion","type":"STRING","defaultValue":null},{"fieldName":"s3StagingUri","label":"S3 Staging URI","description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.s3StagingUri","type":"STRING","defaultValue":null},{"fieldName":"provisionNewCluster","label":"Provision a New Cluster","description":"Provisions a new cluster when the pipeline starts","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false},{"fieldName":"clusterId","label":"Cluster ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[false],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.clusterId","type":"STRING","defaultValue":null},{"fieldName":"emrVersion","label":"EMR Version","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.emrVersion","type":"STRING","defaultValue":null},{"fieldName":"clusterPrefix","label":"Cluster Name Prefix","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.clusterPrefix","type":"STRING","defaultValue":null},{"fieldName":"terminateCluster","label":"Terminate Cluster","description":"Terminates the cluster when the pipeline stops","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true},{"fieldName":"loggingEnabled","label":"Logging Enabled","description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true},{"fieldName":"s3LogUri","label":"S3 Log URI","description":"S3 URI to store log files. Use the format: s3://<path>.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.s3LogUri","type":"STRING","defaultValue":null},{"fieldName":"serviceRole","label":"Service Role","description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole"},{"fieldName":"jobFlowRole","label":"Job Flow Role","description":"EMR role for EC2. Used by EC2 instances within the cluster.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole"},{"fieldName":"visibleToAllUsers","label":"Visible to All Users","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true},{"fieldName":"ec2SubnetId","label":"EC2 Subnet ID","description":"ID of the EC2 subnet to launch the cluster in","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null},{"fieldName":"masterSecurityGroup","label":"Master Security Group","description":"ID of the security group for the master node","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"slaveSecurityGroup","label":"Slave Security Group","description":"ID of the security group for slave nodes","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"serviceAccessSecurityGroup","label":"Service Access Security Group","description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"instanceCount","label":"Instance Count","description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":2,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.instanceCount","type":"NUMBER","defaultValue":2},{"fieldName":"stepConcurrency","label":"Step Concurrency","description":"Number of EMR steps this cluster can run concurrently","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":1,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":10},{"fieldName":"masterInstanceType","label":"Master Instance Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"],"values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null},{"fieldName":"masterInstanceTypeCustom","label":"Master Instance Type (Custom)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.masterInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null},{"fieldName":"slaveInstanceType","label":"Slave Instance Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"],"values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null},{"fieldName":"slaveInstanceTypeCustom","label":"Slave Instance Type (Custom)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnection.slaveInstanceType":["OTHER"],"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sdcEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null},{"fieldName":"customTags","label":"AWS Cluster Tags","description":"AWS tags to assign to all EMR clusters provisioned by the connection.","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"tag","label":"Tag Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"tag","type":"STRING","defaultValue":null},{"fieldName":"value","label":"Tag Value","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"value","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8300,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"sdcEmrConnection.customTags","type":"MODEL","defaultValue":[]},{"fieldName":"enableEMRDebugging","label":"Enable Debugging","description":"Enable console debugging in EMR","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"sdcEmrConnectionSelection":["MANUAL"],"sdcEmrConnection.provisionNewCluster":[true],"executionMode":["EMR_BATCH"],"sdcEmrConnection.loggingEnabled":[true]},"dependsOn":"","group":"EMR","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3310,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"enableEMRDebugging","type":"BOOLEAN","defaultValue":true},{"fieldName":"transformerEmrConnectionSelection","label":"Connection","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["None"],"values":["MANUAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.api.ConnectionDef$Constants$ConnectionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-600,"connectionType":"STREAMSETS_AWS_EMR_CLUSTER","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnectionSelection","type":"MODEL","defaultValue":"MANUAL"},{"fieldName":"credentialMode","label":"Authentication Method","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["AWS Keys","Instance Profile","None"],"values":["WITH_CREDENTIALS","WITH_IAM_ROLES","WITH_ANONYMOUS_CREDENTIALS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AWSCredentialModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-120,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.credentialMode","type":"MODEL","defaultValue":"WITH_CREDENTIALS"},{"fieldName":"awsAccessKeyId","label":"Access Key ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.awsAccessKeyId","type":"CREDENTIAL","defaultValue":null},{"fieldName":"awsSecretAccessKey","label":"Secret Access Key","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.awsSecretAccessKey","type":"CREDENTIAL","defaultValue":null},{"fieldName":"isAssumeRole","label":"Assume Role","description":"Temporarily assume a specified role to connect to Amazon","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.isAssumeRole","type":"BOOLEAN","defaultValue":false},{"fieldName":"roleARN","label":"Role ARN","description":"Amazon resource name of the role to assume","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.roleARN","type":"CREDENTIAL","defaultValue":"arn:aws:iam::<account-id>:role/role-name"},{"fieldName":"roleSessionName","label":"Role Session Name","description":"Optional name for the session created by assuming a role. Overrides the default unique identifier.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.roleSessionName","type":"STRING","defaultValue":null},{"fieldName":"sessionDuration","label":"Session Timeout (secs)","description":"Maximum time for each session. Use a value in the following range: 3600 - 43200. The session is refreshed if the pipeline continues to run.","max":43200,"mode":"text/plain","model":null,"required":true,"min":3600,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":-100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.sessionDuration","type":"NUMBER","defaultValue":3600},{"fieldName":"setSessionTags","label":"Set Session Tags","description":"Specify session tags in the IAM policy to require that the currently logged in user who starts the pipeline has access to assume the specified role","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.awsConfig.isAssumeRole":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.awsConfig.credentialMode":["WITH_IAM_ROLES","WITH_CREDENTIALS"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":-100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.awsConfig.setSessionTags","type":"BOOLEAN","defaultValue":true},{"fieldName":"region","label":"AWS Region","description":"The AWS region for EMR.","max":9223372036854776000,"mode":"text/plain","model":{"labels":["US East (N. Virginia) us-east-1","US East (Ohio) us-east-2","US West (N. California) us-west-1","US West (Oregon) us-west-2","AWS GovCloud (US-East) us-gov-east-1","AWS GovCloud (US-West) us-gov-west-1","Africa (Cape Town) af-south-1","Asia Pacific (Hong Kong) ap-east-1","Asia Pacific (Tokyo) ap-northeast-1","Asia Pacific (Seoul) ap-northeast-2","Asia Pacific (Osaka-Local) ap-northeast-3","Asia Pacific (Mumbai) ap-south-1","Asia Pacific (Singapore) ap-southeast-1","Asia Pacific (Sydney) ap-southeast-2","Canada (Central) ca-central-1","China (Beijing) cn-north-1","China (Ningxia) cn-northwest-1","EU (Frankfurt) eu-central-1","Europe (Milan) eu-south-1","Europe (Stockholm) eu-north-1","EU (Ireland) eu-west-1","EU (London) eu-west-2","EU (Paris) eu-west-3","Middle East (Bahrain) me-south-1","South America (São Paulo) sa-east-1","Other - specify"],"values":["US_EAST_1","US_EAST_2","US_WEST_1","US_WEST_2","US_GOV_EST_1","US_GOV_WEST_1","AF_SOUTH_1","AP_EAST_1","AP_NORTHEAST_1","AP_NORTHEAST_2","AP_NORTHEAST_3","AP_SOUTH_1","AP_SOUTHEAST_1","AP_SOUTHEAST_2","CA_CENTRAL_1","CN_NORTH_1","CN_NORTHWEST_1","EU_CENTRAL_1","EU_SOUTH_1","EU_NORTH_1","EU_WEST_1","EU_WEST_2","EU_WEST_3","ME_SOUTH_1","SA_EAST_1","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.lib.aws.AwsRegionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.region","type":"MODEL","defaultValue":"US_WEST_2"},{"fieldName":"customRegion","label":"Endpoint","description":"Custom endpoint","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.region":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.customRegion","type":"STRING","defaultValue":null},{"fieldName":"s3StagingUri","label":"S3 Staging URI","description":"S3 URI where resources are staged for pipeline execution. Use the format: s3://<path>.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.s3StagingUri","type":"STRING","defaultValue":null},{"fieldName":"provisionNewCluster","label":"Provision a New Cluster","description":"Provisions a new cluster when the pipeline starts","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.provisionNewCluster","type":"BOOLEAN","defaultValue":false},{"fieldName":"defineBootstrapActions","label":"Define Bootstrap Actions","description":"Enables actions to be executed on bootstraping a pipeline","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.defineBootstrapActions","type":"BOOLEAN","defaultValue":false},{"fieldName":"bootstrapActionSource","label":"Bootstrap Actions Source","description":"Defines where bootstrap action should be taken from","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Executable Files in S3","Defined in Pipeline"],"values":["IN_S3","IN_PIPELINE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.common.emr.BootstrapActionSourceChooser","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.bootstrapActionSource","type":"MODEL","defaultValue":"IN_S3"},{"fieldName":"bootstrapActions","label":"Bootstrap Actions","description":"S3 URIs of bootstrap actions and their arguments","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"location","label":"Location","description":"S3 URI where the action script is located","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2121,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"location","type":"STRING","defaultValue":null},{"fieldName":"arguments","label":"Arguments","description":"Arguments of the action script","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2122,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"arguments","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.bootstrapActionSource":["IN_S3"],"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2120,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.bootstrapActions","type":"MODEL","defaultValue":[]},{"fieldName":"bootstrapActionScripts","label":"Bootstrap Action Scripts","description":"Contents of bootstrap actions","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"script","label":"Scripts","description":"Content of a bootstrap action script","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2131,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":5,"name":"script","type":"TEXT","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.bootstrapActionSource":["IN_PIPELINE"],"transformerEmrConnection.defineBootstrapActions":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":2130,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.bootstrapActionScripts","type":"MODEL","defaultValue":[]},{"fieldName":"clusterId","label":"Cluster ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[false]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":2500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.clusterId","type":"STRING","defaultValue":null},{"fieldName":"emrVersion","label":"EMR Version","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.emrVersion","type":"STRING","defaultValue":null},{"fieldName":"clusterPrefix","label":"Cluster Name Prefix","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.clusterPrefix","type":"STRING","defaultValue":null},{"fieldName":"terminateCluster","label":"Terminate Cluster","description":"Terminates the cluster when the pipeline stops","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.terminateCluster","type":"BOOLEAN","defaultValue":true},{"fieldName":"loggingEnabled","label":"Logging Enabled","description":"Copies cluster log files to the specified S3 location. Use to enable continued access to log files.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":3300,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.loggingEnabled","type":"BOOLEAN","defaultValue":true},{"fieldName":"s3LogUri","label":"S3 Log URI","description":"S3 URI to store log files. Use the format: s3://<path>.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.loggingEnabled":[true],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":3500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.s3LogUri","type":"STRING","defaultValue":null},{"fieldName":"serviceRole","label":"Service Role","description":"EMR role used by the cluster when provisioning resources and performing other service-level tasks","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.serviceRole","type":"STRING","defaultValue":"EMR_DefaultRole"},{"fieldName":"jobFlowRole","label":"Job Flow Role","description":"EMR role for EC2. Used by EC2 instances within the cluster.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.jobFlowRole","type":"STRING","defaultValue":"EMR_EC2_DefaultRole"},{"fieldName":"sshEc2KeyId","label":"SSH EC2 Key ID","description":"ID of the EC2 SSH key to be used on nodes of the cluster","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4150,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.sshEc2KeyId","type":"STRING","defaultValue":null},{"fieldName":"visibleToAllUsers","label":"Visible to All Users","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":4200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.visibleToAllUsers","type":"BOOLEAN","defaultValue":true},{"fieldName":"ec2SubnetId","label":"EC2 Subnet ID","description":"ID of the EC2 subnet to launch the cluster in","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.ec2SubnetId","type":"STRING","defaultValue":null},{"fieldName":"masterSecurityGroup","label":"Master Security Group","description":"ID of the security group for the master node","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4700,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.masterSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"slaveSecurityGroup","label":"Slave Security Group","description":"ID of the security group for slave nodes","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4800,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.slaveSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"serviceAccessSecurityGroup","label":"Service Access Security Group","description":"ID of the security group for the Amazon EMR service to access clusters in VPC private subnets","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":4900,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.serviceAccessSecurityGroup","type":"STRING","defaultValue":null},{"fieldName":"instanceCount","label":"Instance Count","description":"EC2 instances in the cluster. Minimum of 2. Adding instances for multiple partitions can improve performance","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":2,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.instanceCount","type":"NUMBER","defaultValue":2},{"fieldName":"stepConcurrency","label":"Step Concurrency","description":"Number of EMR steps this cluster can run concurrently","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":1,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":5000,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.stepConcurrency","type":"NUMBER","defaultValue":10},{"fieldName":"masterInstanceType","label":"Master Instance Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"],"values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":7000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.masterInstanceType","type":"MODEL","defaultValue":null},{"fieldName":"masterInstanceTypeCustom","label":"Master Instance Type (Custom)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.masterInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":7100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.masterInstanceTypeCustom","type":"STRING","defaultValue":null},{"fieldName":"slaveInstanceType","label":"Slave Instance Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["m4.large","m4.xlarge","m4.2xlarge","m4.4xlarge","m4.10xlarge","m4.16xlarge","c4.large","c4.xlarge","c4.2xlarge","c4.4xlarge","c4.8xlarge","r3.xlarge","r3.2xlarge","r3.4xlarge","r3.8xlarge","r4.xlarge","r4.2xlarge","r4.4xlarge","r4.8xlarge","r4.16xlarge","i3.xlarge","i3.2xlarge","i3.4xlarge","i3.8xlarge","i3.16xlarge","d2.xlarge","d2.2xlarge","d2.4xlarge","d2.8xlarge","p2.xlarge","p2.8xlarge","p2.16xlarge","p3.2xlarge","p3.8xlarge","p3.16xlarge","Custom"],"values":["M4_LARGE","M4_XLARGE","M4_2XLARGE","M4_4XLARGE","M4_10XLARGE","M4_16XLARGE","C4_LARGE","C4_XLARGE","C4_2XLARGE","C4_4XLARGE","C4_8XLARGE","R3_XLARGE","R3_2XLARGE","R3_4XLARGE","R3_8XLARGE","R4_XLARGE","R4_2XLARGE","R4_4XLARGE","R4_8XLARGE","R4_16XLARGE","I3_XLARGE","I3_2XLARGE","I3_4XLARGE","I3_8XLARGE","I3_16XLARGE","D2_XLARGE","D2_2XLARGE","D2_4XLARGE","D2_8XLARGE","P2_XLARGE","P2_8XLARGE","P2_16XLARGE","P3_2XLARGE","P3_8XLARGE","P3_16XLARGE","OTHER"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.slaveInstanceType","type":"MODEL","defaultValue":null},{"fieldName":"slaveInstanceTypeCustom","label":"Slave Instance Type (Custom)","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"transformerEmrConnection.slaveInstanceType":["OTHER"],"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":8100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"transformerEmrConnection.slaveInstanceTypeCustom","type":"STRING","defaultValue":null},{"fieldName":"customTags","label":"AWS Cluster Tags","description":"AWS tags to assign to all EMR clusters provisioned by the connection.","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"tag","label":"Tag Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"tag","type":"STRING","defaultValue":null},{"fieldName":"value","label":"Tag Value","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"value","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEmrConnectionSelection":["MANUAL"],"transformerEmrConnection.provisionNewCluster":[true]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":8300,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEmrConnection.customTags","type":"MODEL","defaultValue":[]},{"fieldName":"encryption","label":"Enable Server-Side Encryption","description":"Server-Side Encryption","max":9223372036854776000,"mode":"text/plain","model":{"labels":["None","SSE-S3","SSE-KMS"],"values":["NONE","S3","KMS"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.transformer.config.SseOptionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":50100,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEMRConfig.encryption","type":"MODEL","defaultValue":"NONE"},{"fieldName":"kmsKeyId","label":"AWS KMS Key ARN","description":"AWS KMS master encryption key that was used for the object. The KMS key you specify in the policy must use the \"arn:aws:kms:region:acct-id:key/key-id\" format.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["EMR"],"executionMode":["STREAMING","BATCH"],"transformerEMRConfig.encryption":["KMS"]},"dependsOn":"","group":"CLUSTER","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":50200,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"transformerEMRConfig.kmsKeyId","type":"CREDENTIAL","defaultValue":null},{"fieldName":"projectId","label":"Project ID","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudCredentialsConfig.projectId","type":"STRING","defaultValue":null},{"fieldName":"credentialsProvider","label":"Credentials Provider","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Default Credentials Provider","Service Account Credentials File (JSON)","Service Account Credentials (JSON)"],"values":["DEFAULT_PROVIDER","JSON_PROVIDER","JSON"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.stage.common.CredentialsProviderChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudCredentialsConfig.credentialsProvider","type":"MODEL","defaultValue":"DEFAULT_PROVIDER"},{"fieldName":"path","label":"Credentials File Path (JSON)","description":"Path to the credentials file.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON_PROVIDER"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudCredentialsConfig.path","type":"STRING","defaultValue":null},{"fieldName":"credentialsFileContent","label":"Credentials File Content (JSON)","description":"Content of the credentials file","max":9223372036854776000,"mode":"application/json","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"googleCloudCredentialsConfig.credentialsProvider":["JSON"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":30,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudCredentialsConfig.credentialsFileContent","type":"CREDENTIAL","defaultValue":null},{"fieldName":"region","label":"Region","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["us-west1","us-west2","us-west3","us-east1","us-east4","us-central1","asia-east1","asia-east2","asia-northeast1","asia-northeast2","asia-northeast3","asia-south1","asia-south2","asia-southeast1","asia-southeast2","australia-southeast1","australia-southeast2","europe-central2","europe-north1","europe-west1","europe-west2","europe-west3","europe-west4","europe-west6","northamerica-northeast1","northamerica-northeast2","southamerica-east1","Custom"],"values":["US_WEST_1","US_WEST_2","US_WEST_3","US_EAST_1","US_EAST_4","US_CENTRAL_1","ASIA_EAST_1","ASIA_EAST_2","ASIA_NORTH_EAST_1","ASIA_NORTH_EAST_2","ASIA_NORTH_EAST_3","ASIA_SOUTH_1","ASIA_SOUTH_2","ASIA_SOUTH_EAST_1","ASIA_SOUTH_EAST_2","AUSTRALIA_SOUTH_EAST_1","AUSTRALIA_SOUTH_EAST_2","EUROPE_CENTRAL_2","EUROPE_NORTH_1","EUROPE_WEST_1","EUROPE_WEST_2","EUROPE_WEST_3","EUROPE_WEST_4","EUROPE_WEST_6","NORTH_AMERICA_NORTH_EAST_1","NORTH_AMERICA_NORTH_EAST_2","SOUTH_AMERICA_EAST_1","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.GoogleCloudRegionChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":15,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.region","type":"MODEL","defaultValue":null},{"fieldName":"customRegion","label":"Custom Region","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"googleCloudConfig.region":["CUSTOM"],"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":18,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.customRegion","type":"STRING","defaultValue":null},{"fieldName":"gcsStagingUri","label":"GCS Staging URI","description":"GCS URI where Transformer resources are staged for pipeline execution. Use the format: gs://<path>.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":60,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.gcsStagingUri","type":"STRING","defaultValue":null},{"fieldName":"create","label":"Create Cluster","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":70,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.create","type":"BOOLEAN","defaultValue":false},{"fieldName":"clusterPrefix","label":"Cluster Prefix","description":"Cluster name is generated by suffixing the pipeline ID to this prefix","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":80,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.clusterPrefix","type":"STRING","defaultValue":null},{"fieldName":"version","label":"Image Version","description":"The image version to use for the Dataproc cluster. Example: 1.5-ubuntu18. If not specified, Dataproc default is used","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":85,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"googleCloudConfig.version","type":"STRING","defaultValue":"1.4-ubuntu18"},{"fieldName":"masterType","label":"Master Machine Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"],"values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":90,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.masterType","type":"MODEL","defaultValue":null},{"fieldName":"workerType","label":"Worker Machine Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["n1-standard-2","n1-standard-4","n1-standard-8","n1-standard-16","n1-standard-32","n1-standard-64","n1-standard-96","n1-highmem-2","n1-highmem-4","n1-highmem-8","n1-highmem-16","n1-highmem-32","n1-highmem-64","n1-highmem-96","n1-highcpu-2","n1-highcpu-4","n1-highcpu-8","n1-highcpu-16","n1-highcpu-32","n1-highcpu-64","n1-highcpu-96","n2-standard-2","n2-standard-4","n2-standard-8","n2-standard-16","n2-standard-32","n2-standard-48","n2-standard-64","n2-standard-80","n2-highmem-2","n2-highmem-4","n2-highmem-8","n2-highmem-16","n2-highmem-32","n2-highmem-48","n2-highmem-64","n2-highmem-80","n2-highcpu-2","n2-highcpu-4","n2-highcpu-8","n2-highcpu-16","n2-highcpu-32","n2-highcpu-48","n2-highcpu-64","n2-highcpu-80","n2d-standard-2","n2d-standard-4","n2d-standard-8","n2d-standard-16","n2d-standard-32","nd2-standard-48","n2d-standard-64","n2d-standard-80","n2d-standard-96","n2d-standard-128","n2d-standard-224","n2d-highmem-2","n2d-highmem-4","n2d-highmem-8","n2d-highmem-16","n2d-highmem-32","n2d-highmem-48","n2d-highmem-64","n2d-highmem-80","n2d-highmem-96","n2d-highcpu-2","n2d-highcpu-4","n2d-highcpu-8","n2d-highcpu-16","n2d-highcpu-32","n2d-highcpu-48","n2d-highcpu-64","n2d-highcpu-80","n2d-highcpu-96","n2d-highcpu-128","n2d-highcpu-224","e2-standard-2","e2-standard-4","e2-standard-8","e2-standard-16","e2-highmem-2","e2-highmem-4","e2-highmem-8","e2-highmem-16","e2-highcpu-2","e2-highcpu-4","e2-highcpu-8","e2-highcpu-16","m1-ultramem-40","m1-ultramem-80","m1-ultramem-160","m2-ultramem-208","m2-ultramem-416","c2-standard-4","c2-standard-8","c2-standard-16","c2-standard-30","c2-standard-60","custom"],"values":["N1_STANDARD_2","N1_STANDARD_4","N1_STANDARD_8","N1_STANDARD_16","N1_STANDARD_32","N1_STANDARD_64","N1_STANDARD_96","N1_HIGHMEM_2","N1_HIGHMEM_4","N1_HIGHMEM_8","N1_HIGHMEM_16","N1_HIGHMEM_32","N1_HIGHMEM_64","N1_HIGHMEM_96","N1_HIGHCPU_2","N1_HIGHCPU_4","N1_HIGHCPU_8","N1_HIGHCPU_16","N1_HIGHCPU_32","N1_HIGHCPU_64","N1_HIGHCPU_96","N2_STANDARD_2","N2_STANDARD_4","N2_STANDARD_8","N2_STANDARD_16","N2_STANDARD_32","N2_STANDARD_48","N2_STANDARD_64","N2_STANDARD_80","N2_HIGHMEM_2","N2_HIGHMEM_4","N2_HIGHMEM_8","N2_HIGHMEM_16","N2_HIGHMEM_32","N2_HIGHMEM_48","N2_HIGHMEM_64","N2_HIGHMEM_80","N2_HIGHCPU_2","N2_HIGHCPU_4","N2_HIGHCPU_8","N2_HIGHCPU_16","N2_HIGHCPU_32","N2_HIGHCPU_48","N2_HIGHCPU_64","N2_HIGHCPU_80","N2D_STANDARD_2","N2D_STANDARD_4","N2D_STANDARD_8","N2D_STANDARD_16","N2D_STANDARD_32","ND2_STANDARD_48","N2D_STANDARD_64","N2D_STANDARD_80","N2D_STANDARD_96","N2D_STANDARD_128","N2D_STANDARD_224","N2D_HIGHMEM_2","N2D_HIGHMEM_4","N2D_HIGHMEM_8","N2D_HIGHMEM_16","N2D_HIGHMEM_32","N2D_HIGHMEM_48","N2D_HIGHMEM_64","N2D_HIGHMEM_80","N2D_HIGHMEM_96","N2D_HIGHCPU_2","N2D_HIGHCPU_4","N2D_HIGHCPU_8","N2D_HIGHCPU_16","N2D_HIGHCPU_32","N2D_HIGHCPU_48","N2D_HIGHCPU_64","N2D_HIGHCPU_80","N2D_HIGHCPU_96","N2D_HIGHCPU_128","N2D_HIGHCPU_224","E2_STANDARD_2","E2_STANDARD_4","E2_STANDARD_8","E2_STANDARD_16","E2_HIGHMEM_2","E2_HIGHMEM_4","E2_HIGHMEM_8","E2_HIGHMEM_16","E2_HIGHCPU_2","E2_HIGHCPU_4","E2_HIGHCPU_8","E2_HIGHCPU_16","M1_ULTRAMEM_40","M1_ULTRAMEM_80","M1_ULTRAMEM_160","M2_ULTRAMEM_208","M2_ULTRAMEM_416","C2_STANDARD_4","C2_STANDARD_8","C2_STANDARD_16","C2_STANDARD_30","C2_STANDARD_60","CUSTOM"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.MachineTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.workerType","type":"MODEL","defaultValue":null},{"fieldName":"networkType","label":"Network Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Auto - VPC created in Auto mode","Custom - VPC with custom subnets","Default VPC for project and region"],"values":["AUTO","CUSTOM","NONE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.lib.googlecloud.NetworkChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.networkType","type":"MODEL","defaultValue":null},{"fieldName":"network","label":"Network Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["AUTO"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.network","type":"STRING","defaultValue":null},{"fieldName":"subnet","label":"Subnet Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true],"googleCloudConfig.networkType":["CUSTOM"]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.subnet","type":"STRING","defaultValue":null},{"fieldName":"tags","label":"Network Tags","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":140,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.tags","type":"LIST","defaultValue":[]},{"fieldName":"workerCount","label":"Worker Count","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":2,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":150,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.workerCount","type":"NUMBER","defaultValue":2},{"fieldName":"clusterName","label":"Cluster Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[false]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":160,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.clusterName","type":"STRING","defaultValue":null},{"fieldName":"terminate","label":"Terminate Cluster","description":"Terminates the cluster when the pipeline stops","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":170,"connectionType":"","displayMode":"ADVANCED","upload":"NO","lines":0,"name":"googleCloudConfig.terminate","type":"BOOLEAN","defaultValue":null},{"fieldName":"clusterLabels","label":"Cluster Labels","description":"Labels to associate with the new cluster","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"key","label":"Label","description":"Label key","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":181,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"key","type":"STRING","defaultValue":null},{"fieldName":"value","label":"Value","description":"Label value","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":182,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"value","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{"clusterConfig.clusterType":["DATAPROC"],"executionMode":["STREAMING","BATCH"],"googleCloudConfig.create":[true]},"dependsOn":"","group":"DATAPROC","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":180,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"googleCloudConfig.clusterLabels","type":"MODEL","defaultValue":[]}],"configGroupDefinition":{"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineGroups":["CLUSTER","PARAMETERS","NOTIFICATIONS","BAD_RECORDS","STATS","EMR","DATAPROC","ADVANCED"]},"groupNameToLabelMapList":[{"name":"CLUSTER","label":"Cluster"},{"name":"PARAMETERS","label":"Parameters"},{"name":"NOTIFICATIONS","label":"Notifications"},{"name":"BAD_RECORDS","label":"Error Records"},{"name":"STATS","label":"Statistics"},{"name":"EMR","label":"EMR"},{"name":"DATAPROC","label":"Dataproc"},{"name":"ADVANCED","label":"Advanced"}]}}],"pipelineFragment":null,"pipelineRules":[{"configDefinitions":[{"fieldName":"emailIDs","label":"Email IDs","description":"Email Addresses","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":76,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"emailIDs","type":"LIST","defaultValue":[]},{"fieldName":"webhookConfigs","label":"Webhooks","description":"Webhooks","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"payload","label":"Payload","description":"Data that should be included as a part of the Webhook request","max":9223372036854776000,"mode":"application/json","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["DELETE","POST","PUT"]},"dependsOn":"","group":"WEBHOOK","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":240,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":2,"name":"payload","type":"TEXT","defaultValue":"{\n  \"text\" : \"{{ALERT_TEXT}}. The alert, {{ALERT_NAME}},  was triggered for pipeline '{{PIPELINE_TITLE}}'. \\n The threshold of {{ALERT_VALUE}} records, for the following condition was reached at {{TIME}}: {{ALERT_CONDITION}} \\n <{{PIPELINE_URL}}|Click here> for details!\"\n}"},{"fieldName":"webhookUrl","label":"Webhook URL","description":"The Webhook HTTP resource URL","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"webhookUrl","type":"STRING","defaultValue":null},{"fieldName":"headers","label":"Headers","description":"Headers to include in the request","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":210,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"headers","type":"MAP","defaultValue":[]},{"fieldName":"httpMethod","label":"HTTP Method","description":"HTTP method to send","max":9223372036854776000,"mode":"text/plain","model":{"labels":["GET","PUT","POST","DELETE","HEAD"],"values":["GET","PUT","POST","DELETE","HEAD"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.HttpMethodChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":220,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"httpMethod","type":"MODEL","defaultValue":"POST"},{"fieldName":"contentType","label":"Content Type","description":"Content-Type header to be sent with the request; used if that header is not already present","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"httpMethod":["DELETE","POST","PUT"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"EXPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":250,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"contentType","type":"STRING","defaultValue":"application/json"},{"fieldName":"authType","label":"Authentication Type","description":"","max":9223372036854776000,"mode":"text/plain","model":{"labels":["None","Basic","Digest","Universal"],"values":["NONE","BASIC","DIGEST","UNIVERSAL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.datacollector.config.AuthenticationTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":260,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"authType","type":"MODEL","defaultValue":"NONE"},{"fieldName":"username","label":"Username","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","BASIC","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":270,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"username","type":"CREDENTIAL","defaultValue":null},{"fieldName":"password","label":"Password","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"authType":["DIGEST","BASIC","UNIVERSAL"]},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","65","66","67","68"],"elConstantDefinitionsIdx":["64"],"displayPosition":280,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"password","type":"CREDENTIAL","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"NOTIFICATIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":200,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"webhookConfigs","type":"MODEL","defaultValue":[]}],"configGroupDefinition":{"classNameToGroupsMap":{"com.streamsets.datacollector.config.PipelineRulesGroups":["NOTIFICATIONS"]},"groupNameToLabelMapList":[{"name":"NOTIFICATIONS","label":"Notifications"}]}}],"stages":[{"icon":"directory.png","version":"2","privateClassLoader":false,"services":[],"library":"streamsets-spark-file-lib","label":"File","libraryLabel":"File","description":"Reads data from HDFS-compatible file systems","configDefinitions":[{"fieldName":"fileNamePattern","label":"File Name Pattern","description":"A glob that defines a pattern of file names to process","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":12,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.fileNamePattern","type":"STRING","defaultValue":"*"},{"fieldName":"fileNameExcludePattern","label":"Exclusion Pattern","description":"A glob that defines a pattern of file names to exclude from processing","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":13,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.fileNameExcludePattern","type":"STRING","defaultValue":null},{"fieldName":"batchNumFileLimit","label":"Max Files Per Batch","description":"Limits the number of files to be processed in a single batch. Use 0 for unlimited.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":0,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":15,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.batchNumFileLimit","type":"NUMBER","defaultValue":0},{"fieldName":"schemaMode","label":"Schema Mode","description":"Determines the schema to use when reading data","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Infer from Data","Use Custom Schema - JSON Format","Use Custom Schema - DDL Format"],"values":["INFER","JSON","DDL"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.spark.origin.schema.SchemaModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV","JSON"]},"dependsOn":"","group":"SCHEMA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.schemaConfigBean.schemaMode","type":"MODEL","defaultValue":"INFER"},{"fieldName":"customSchemaString","label":"Schema","description":"Custom schema to use when parsing data","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.schemaConfigBean.schemaMode":["DDL","JSON"],"configBean.dataFormat":["CSV","JSON"]},"dependsOn":"","group":"SCHEMA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.schemaConfigBean.customSchemaString","type":"TEXT","defaultValue":null},{"fieldName":"parseErrorMode","label":"Error Handling","description":"Behavior when data does not conform to the schema","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Permissive - Creates a record with null values, optionally places original data in a field","Drop Malformed - Drops the record from the pipeline","Fail Fast - Stops the pipeline"],"values":["PERMISSIVE","DROPMALFORMED","FAILFAST"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.spark.origin.schema.ParseErrorModeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.schemaConfigBean.schemaMode":["DDL","JSON"],"configBean.dataFormat":["CSV","JSON"]},"dependsOn":"","group":"SCHEMA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":50,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.schemaConfigBean.parseErrorMode","type":"MODEL","defaultValue":"PERMISSIVE"},{"fieldName":"columnNameOfCorruptRecord","label":"Original Data Field","description":"String field to contain the original record data when parsing errors occur. Include the field in the custom schema. When not used, the original data for error records is not captured.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.schemaConfigBean.schemaMode":["DDL","JSON"],"configBean.schemaConfigBean.parseErrorMode":["PERMISSIVE"],"configBean.dataFormat":["CSV","JSON"]},"dependsOn":"","group":"SCHEMA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":60,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.schemaConfigBean.columnNameOfCorruptRecord","type":"STRING","defaultValue":null},{"fieldName":"postProcessType","label":"Post Processing","description":"Action to take on successfully read files","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Delete","Archive","None"],"values":["DELETE","ARCHIVE","NONE"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.spark.lib.fs.PostProcessTypeValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.postProcessType","type":"MODEL","defaultValue":"NONE"},{"fieldName":"archiveDir","label":"Archive Directory","description":"Directory to store successfully read files","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.postProcessType":["ARCHIVE"]},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":40,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.archiveDir","type":"STRING","defaultValue":null},{"fieldName":"csvHeader","label":"Includes Header","description":"Uses the first row of the first file to name record fields. Skips processing the first row of each file, assuming it is a header row.","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":400,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.csvHeader","type":"BOOLEAN","defaultValue":true},{"fieldName":"csvDelimiter","label":"Delimiter Character","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":410,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.csvDelimiter","type":"CHARACTER","defaultValue":","},{"fieldName":"csvQuote","label":"Quote Character","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":420,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.csvQuote","type":"CHARACTER","defaultValue":"\""},{"fieldName":"csvEscape","label":"Escape Character","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":430,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.csvEscape","type":"CHARACTER","defaultValue":"\\"},{"fieldName":"multiLineCsv","label":"Multi-Line CSV","description":"Parse multiple lines of CSV as one record based on the configured record delimiter. Can negatively affect performance","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["CSV"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":440,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.multiLineCsv","type":"BOOLEAN","defaultValue":false},{"fieldName":"rootTag","label":"Root Tag","description":"Root tag of the XML files","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":500,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.rootTag","type":"STRING","defaultValue":"ROWS"},{"fieldName":"rowTag","label":"Row Tag","description":"Tag used to identify each row","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["XML"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":510,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.rowTag","type":"STRING","defaultValue":"ROW"},{"fieldName":"multiLineJson","label":"Multi-Line JSON","description":"Parse JSON in multi-line mode","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["JSON"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":1000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.multiLineJson","type":"BOOLEAN","defaultValue":false},{"fieldName":"avroSchema","label":"Avro Schema","description":"The Avro schema (in JSON format) to use when reading data","max":9223372036854776000,"mode":"text/plain","model":null,"required":false,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.avroSchema","type":"STRING","defaultValue":null},{"fieldName":"avroIgnoreExtension","label":"Ignore extension","description":"If enabled, all files (not just those ending in .avro) are read","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"configBean.dataFormat":["AVRO"]},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormatConfig.avroIgnoreExtension","type":"BOOLEAN","defaultValue":true},{"fieldName":"path","label":"Directory Path","description":"Path to the directory to use","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.path","type":"STRING","defaultValue":null},{"fieldName":"dataFormat","label":"Data Format","description":"Format of the data","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Avro (Spark 2.4 or later)","Avro (Spark 2.3)","Delimited","JSON","ORC","Parquet","Text","XML"],"values":["AVRO","AVRO_OLD","CSV","JSON","ORC","PARQUET","TEXT","XML"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.spark.lib.fs.DataFormatChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"DATA_FORMAT","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.dataFormat","type":"MODEL","defaultValue":null},{"fieldName":"hdfsProperties","label":"Additional Configuration","description":"Additional properties to pass to a Hadoop-compatible file system. These properties take precedence over those defined in HDFS configuration files.","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"key","label":"Name","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"key","type":"STRING","defaultValue":null},{"fieldName":"value","label":"Value","description":"","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"value","type":"STRING","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":false,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"FILE","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":999,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"configBean.hdfsProperties","type":"MODEL","defaultValue":[]},{"fieldName":"noReload","label":"Load Data Only Once","description":"Do not reload data every micro-batch in streaming mode. Has no effect in batch mode. Data will be cached on Spark executors","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"noReload","type":"BOOLEAN","defaultValue":false},{"fieldName":"cache","label":"Cache Data","description":"If only a small amount of data is read from the source system due to pushdown of operators  from other stages, it might be beneficial to not cache. Caching will cause all data to be read once","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"noReload":[false]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"cache","type":"BOOLEAN","defaultValue":true},{"fieldName":"skipOffsetTracking","label":"Skip Offset Tracking","description":"Do not track offsets between batches. Data will be completely reloaded from the source on  batch.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10000000,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"skipOffsetTracking","type":"BOOLEAN","defaultValue":false}],"configGroupDefinition":{"classNameToGroupsMap":{"com.streamsets.pipeline.spark.origin.fs.Groups":["FILE","DATA_FORMAT","SCHEMA"]},"groupNameToLabelMapList":[{"name":"FILE","label":"File"},{"name":"DATA_FORMAT","label":"Data Format"},{"name":"SCHEMA","label":"Schema"}]},"pipelineLifecycleStage":false,"statsAggregatorStage":false,"errorStage":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["BATCH","STREAMING"],"resetOffset":true,"inputStreamLabels":null,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","onlineHelpRefUrl":"index.html?contextID=task_gvq_jrd_1hb","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"beta":false,"bisectable":false,"eventDefs":[],"yamlUpgrader":"upgrader/FileDOrigin.yaml","name":"com_streamsets_pipeline_spark_origin_fs_FileDOrigin","type":"SOURCE","className":"com.streamsets.pipeline.spark.origin.fs.FileDOrigin","onRecordError":false,"producingEvents":false,"preconditions":false},{"icon":"expression.png","version":"2","privateClassLoader":false,"services":[],"library":"streamsets-spark-basic-lib","label":"Spark SQL Expression","libraryLabel":"Basic","description":"Evaluates Spark SQL expressions on input data","configDefinitions":[{"fieldName":"expressions","label":"Expressions","description":"Configure Spark SQL expressions","max":9223372036854776000,"mode":"text/plain","model":{"labels":null,"values":null,"configDefinitions":[{"fieldName":"fieldName","label":"Output Field","description":"Output field for the results of the calculation","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"fieldName","type":"STRING","defaultValue":null},{"fieldName":"sqlExpression","label":"SQL Expression","description":"Spark SQL expression","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":20,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"sqlExpression","type":"STRING","defaultValue":null},{"fieldName":"enableCastToType","label":"Cast to Type","description":"Cast the expression to the specified type. Leave clear to use the inferred type.","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":105,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"enableCastToType","type":"BOOLEAN","defaultValue":false},{"fieldName":"castToType","label":"Type","description":"Data type to cast the expression to","max":9223372036854776000,"mode":"text/plain","model":{"labels":["Binary","Byte","Boolean","Date","Decimal","Double","Float","Integer","Long","Timestamp","String","Short"],"values":["BINARY","BYTE","BOOLEAN","DATE","DECIMAL","DOUBLE","FLOAT","INTEGER","LONG","TIMESTAMP","STRING","SHORT"],"configDefinitions":null,"modelType":"VALUE_CHOOSER","valuesProviderClass":"com.streamsets.pipeline.spark.transform.common.config.TargetTypeChooserValues","filteringConfig":""},"required":true,"min":-9223372036854776000,"dependsOnMap":{"enableCastToType":[true]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":110,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"castToType","type":"MODEL","defaultValue":"STRING"},{"fieldName":"precision","label":"Precision","description":"Decimal precision","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"enableCastToType":[true],"castToType":["DECIMAL"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":120,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"precision","type":"NUMBER","defaultValue":null},{"fieldName":"scale","label":"Scale","description":"Decimal scale","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{"enableCastToType":[true],"castToType":["DECIMAL"]},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":130,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"scale","type":"NUMBER","defaultValue":null}],"modelType":"LIST_BEAN","valuesProviderClass":null,"filteringConfig":null},"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"EXPRESSIONS","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":[],"elConstantDefinitionsIdx":[],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"config.expressions","type":"MODEL","defaultValue":null},{"fieldName":"cache","label":"Cache Data","description":"If this process does light-weight processing, due to pushdown of operators from other stages, it might be beneficial to not cache. Caching will cause all data to be read once","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"cache","type":"BOOLEAN","defaultValue":false}],"configGroupDefinition":{"classNameToGroupsMap":{"com.streamsets.pipeline.spark.transform.evaluator.Groups":["EXPRESSIONS"]},"groupNameToLabelMapList":[{"name":"EXPRESSIONS","label":"Expressions"}]},"pipelineLifecycleStage":false,"statsAggregatorStage":false,"errorStage":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["BATCH","STREAMING"],"resetOffset":false,"inputStreamLabels":null,"inputStreams":1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","onlineHelpRefUrl":"index.html?contextID=task_ldk_t5p_lhb","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"beta":false,"bisectable":false,"eventDefs":[],"yamlUpgrader":"","name":"com_streamsets_pipeline_spark_transform_evaluator_SQLEvaluatorDTransform","type":"PROCESSOR","className":"com.streamsets.pipeline.spark.transform.evaluator.SQLEvaluatorDTransform","onRecordError":false,"producingEvents":false,"preconditions":false},{"icon":"scala-spiral.png","version":"3","privateClassLoader":false,"services":[],"library":"streamsets-spark-basic-lib","label":"Scala","libraryLabel":"Basic","description":"Runs Scala code to transform the input DataFrames","configDefinitions":[{"fieldName":"code","label":"Scala Code","description":"Scala code used to transform the DataFrame","max":9223372036854776000,"mode":"text/x-scala","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"SCALA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":10,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"code","type":"TEXT","defaultValue":"/* \nUse the Seq named inputs to access the input DataFrames. SparkSession is available in a variable named spark.\nAfter performing transformations, save the resulting DataFrame to the pre-defined variable named output. \nDo not declare a new variable - this will cause the script initialization to fail.\n For example, the following code increments the value of the first field by 1, and then saves the result to the output variable:\n\nimport spark.implicits._\nimport scala.collection.mutable.Buffer\noutput = inputs(0).map(r => r.getLong(0) + 1).toDF()\n*/\noutput = inputs(0)"},{"fieldName":"skipEmptyBatches","label":"Skip Empty Batches","description":"Skips running the Scala code when the input is empty","max":9223372036854776000,"mode":"text/plain","model":null,"required":true,"min":-9223372036854776000,"dependsOnMap":{},"dependsOn":"","group":"SCALA","elDefs":null,"evaluation":"IMPLICIT","triggeredByValues":null,"elFunctionDefinitionsIdx":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"],"elConstantDefinitionsIdx":["64"],"displayPosition":100,"connectionType":"","displayMode":"BASIC","upload":"NO","lines":0,"name":"skipEmptyBatches","type":"BOOLEAN","defaultValue":true}],"configGroupDefinition":{"classNameToGroupsMap":{"com.streamsets.pipeline.spark.transform.scala.Groups":["SCALA"]},"groupNameToLabelMapList":[{"name":"SCALA","label":"Scala"}]},"pipelineLifecycleStage":false,"statsAggregatorStage":false,"errorStage":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":1,"outputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultOutputStreams","outputStreamLabels":null,"executionModes":["BATCH","STREAMING"],"resetOffset":true,"inputStreamLabels":null,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","onlineHelpRefUrl":"index.html?contextID=task_jwj_lfm_y3b","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"beta":false,"bisectable":false,"eventDefs":[],"yamlUpgrader":"upgrader/ScalaDTransform.yaml","name":"com_streamsets_pipeline_spark_transform_scala_ScalaDTransform","type":"PROCESSOR","className":"com.streamsets.pipeline.spark.transform.scala.ScalaDTransform","onRecordError":false,"producingEvents":false,"preconditions":false},{"icon":"trash.png","version":"1","privateClassLoader":false,"services":[],"library":"streamsets-spark-basic-lib","label":"Trash","libraryLabel":"Basic","description":"Discards records","configDefinitions":[],"configGroupDefinition":{"classNameToGroupsMap":{},"groupNameToLabelMapList":[]},"pipelineLifecycleStage":false,"statsAggregatorStage":false,"errorStage":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["BATCH","STREAMING"],"resetOffset":false,"inputStreamLabels":null,"inputStreams":0,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","onlineHelpRefUrl":"index.html?contextID=concept_o2w_t1z_zkb","hideStage":[],"outputStreamsDrivenByConfig":"","sendsResponse":false,"beta":false,"bisectable":false,"eventDefs":[],"yamlUpgrader":"","name":"com_streamsets_pipeline_spark_destination_dev_NullDTarget","type":"TARGET","className":"com.streamsets.pipeline.spark.destination.dev.NullDTarget","onRecordError":false,"producingEvents":false,"preconditions":false},{"icon":"","version":"1","privateClassLoader":false,"services":[],"library":"streamsets-spark-basic-lib","label":"Write to Control Hub directly","libraryLabel":"Basic","description":"Writes pipeline statistics to Control Hub directly without any aggregation","configDefinitions":[],"configGroupDefinition":{"classNameToGroupsMap":{},"groupNameToLabelMapList":[]},"pipelineLifecycleStage":false,"statsAggregatorStage":true,"errorStage":false,"rawSourceDefinition":null,"connectionVerifierStage":false,"offsetCommitTrigger":false,"variableOutputStreams":false,"outputStreams":0,"outputStreamLabelProviderClass":null,"outputStreamLabels":null,"executionModes":["BATCH","STREAMING"],"resetOffset":false,"inputStreamLabels":null,"inputStreams":-1,"inputStreamLabelProviderClass":"com.streamsets.pipeline.api.StageDef$DefaultInputStreams","onlineHelpRefUrl":"","hideStage":["STATS_AGGREGATOR_STAGE"],"outputStreamsDrivenByConfig":"","sendsResponse":false,"beta":false,"bisectable":false,"eventDefs":[],"yamlUpgrader":"","name":"com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget","type":"TARGET","className":"com.streamsets.pipeline.stage.destination.devnull.StatsDpmDirectlyDTarget","onRecordError":false,"producingEvents":false,"preconditions":false}],"services":[],"rulesElMetadata":null,"elCatalog":null,"runtimeConfigs":null,"stageIcons":{"streamsets-spark-basic-lib:com_streamsets_pipeline_spark_destination_dev_NullDTarget":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABmJLR0QA/wD/AP+gvaeTAAACSUlEQVRoge3YPWsVQRQG4CcaMZpLCJKksLC2sxAstLGz9weIIthaiJUWASuxtdbGwv9grZ02NlYBCQiSaAgkBL8Si9nFzd6P7M6Me0X2heHCcM6ZOfO+Z+fcoUePHj3+FyzgOTZxMGZs4FlhmwWzuQLhKa7hET6NsTmLB5jBrYxrZ8E27jawu4etXIseyxVIkEWTjX3FYq5F20joYjEm4SrmjrC5XPzemWDzthhZsWp8ceYeq7k3X8VDfPwLcd/jSVunmBrYwEqE31FYKWK3QmwCcxhE+E7axxnhDmnt2BblKS3X5r/jZm3uZTGquI1vtblF4YPSGQMMJ7CrGSuDwraKMtZUGdjRPIGd2lwZqxMGtvDLaAbmG/jPG2ZgqfjthIF94TYdxUDTBEYx8ENoR1ohtpXYkL8Gyi62FXInkCKh1vIhbwJNi3ichFoXMNNhYJSElvwDCaQy0KmENuVnoPMaGOBUZa7pV2hUEU+lBvhzARH/FTpdjM4Z4LCMYluJ6DaCvAnsCm328Ql+szjpMAPRbQRpRXxgmIEZQQ7jMF+xLTEVBsq+pc4Ak2VUJlBlYFk4jE4ZYPhTWm5qUiGXydUltC0cSmukJFC/zEpZNGGgLqGo0ydvAikMROmfnoHWDIwq4iQGUl6n60W8h8dYq8y9qvmsFTZ7lbllvEnYRzRuCH8vTyTG+YL7sc4pEloXLq5zCTEWhDeh9dgAMwmLD/BBeCd9IbxUtF37Oi7hPD4n7CUaF/AaP7V/gd7HO1zpfNc9evTokQ2/AbbOuP9A2WXaAAAAAElFTkSuQmCC","streamsets-spark-basic-lib:com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget":null,"streamsets-spark-basic-lib:com_streamsets_pipeline_spark_transform_evaluator_SQLEvaluatorDTransform":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABmJLR0QA/wD/AP+gvaeTAAAD90lEQVRoge3ZXYhVVRQH8J+TZkyW9IE1FGpiWRj0gVIgmQkmFD0kvoRQImUfL1EPRa/VUy+BD5lhPQRFmlSWfakEQRARZIph0Qc4hWLlmFbSqDO3h3Vvc+65+5x7ZubO9HL/cF72WWvt/9p7nbXXXocuuuiiiy7GgSmTONdC3IGezNgADmaefyaRz5gwB/dgMw6jlnsOYCPuxPT/iWNlTMESbMFprc4cwwYs6PTEZ2ERHsGlHbJ5Bd7V6kRNOPcCLhrPBPPxELbhKPabgJXBapyQduQXLKtqaC7WYyt+zRn6GOd2jnMLrhdkU06cqfMqxFp8jeECAx+hd2J4N2Eh/izgMIwHihR7hIfHE4q7cPYoicwQ274eT4gFWq5ahlld4EANg7ixTHmeiPOGwo9G9xEtFeH3t8j1+8T2N+wNiG+qHXaUOPFVmeJynDISdzdXJH453sxM8pKR7+XaOvEsifvb2FsgnWIbz+0ppbk4khHaWJH8rTm9txIym3MEDlSwu7vEgU154V7szQgMilVth5VGdqyGQ9IhtylH4FgF2w+XOLAvL/xiTuCVChMs1PrRP1Yguz8nt6uC/b4SBwaygmsSAskYy6AHX+R0jkufE0tycmfqY1XwW4EDfzUEZmn9wAYLiGRxX8Lo1oRcr9juLPm1FcnDZwUOfNsQ2JB4uaeC4c8TenliM/Fh5v0PuGUU5OH1Age2EQXZYOLlzjZGZ0uf2NfU358nDrD++vhBcYJOGyV5eKPAgXunYpX0Cft7G6M3ab0QDeNBLMYNdfI7sV3UUKfGQB7OSYydxDtTxeUhhXY1z5zE2JCoYZ4SNdWJqgzbYFZibEPD/k/S2/NlG6OPJ3Te7wzfJkwXV83sPD+LENWDywoU2x1g/Ymx4bFxLMUizcVfDevETiNOw6KD4soSw32aC7Sa2M0qWIbrKsq+nJvjubzAd4odKKy769iS0FlRIt+H9/CNdFyn5LMZ8gPNXQ20eph9tleYIN9dOCyyUBbTRfk8gE8rkqe5tOkv0ru7xIEhI3m9CFfj+4TeJ6J4e7tO/CgexdSK5JcaOWf+EFfNJHrElhY58WqFyaaJWmqLCMkBcY7swWui7p9RkThcIi5RDQ67tblQrRSrlnJgWHTUJgsXaq1aayLZrFfSTXwmodR4jihOt53EBYoLt5rY2dLE8qTijsRB5Wl1vJivOWwahLeKlZ9X1dA6xc2lflEDdRorxC3utKhwn8VtxtEfnS1SaMqJ03h6PMZzWInncRfO75DN/7BYpMFUZ+CQ6PXM7PSkE4E+Ubzt0BpeJ0UrZZUJWMF2GMsPjikixK7CxepVYR1DouG018QUdl100UUXXXQW/wKLgdYjXL+F5AAAAABJRU5ErkJggg==","streamsets-spark-basic-lib:com_streamsets_pipeline_spark_transform_scala_ScalaDTransform":"iVBORw0KGgoAAAANSUhEUgAAAE8AAACACAYAAABKijTnAAAAAXNSR0IArs4c6QAAAAlwSFlzAAALEwAACxMBAJqcGAAABBFpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iCiAgICAgICAgICAgIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIgogICAgICAgICAgICB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPHhtcE1NOkRlcml2ZWRGcm9tIHJkZjpwYXJzZVR5cGU9IlJlc291cmNlIj4KICAgICAgICAgICAgPHN0UmVmOmluc3RhbmNlSUQ+eG1wLmlpZDo4RTY2QTRFNzc5MTIxMUUyQTQwNkY5NjA1MkYzNjQxQzwvc3RSZWY6aW5zdGFuY2VJRD4KICAgICAgICAgICAgPHN0UmVmOmRvY3VtZW50SUQ+eG1wLmRpZDo4RTY2QTRFODc5MTIxMUUyQTQwNkY5NjA1MkYzNjQxQzwvc3RSZWY6ZG9jdW1lbnRJRD4KICAgICAgICAgPC94bXBNTTpEZXJpdmVkRnJvbT4KICAgICAgICAgPHhtcE1NOkRvY3VtZW50SUQ+eG1wLmRpZDo4RTY2QTRFQTc5MTIxMUUyQTQwNkY5NjA1MkYzNjQxQzwveG1wTU06RG9jdW1lbnRJRD4KICAgICAgICAgPHhtcE1NOkluc3RhbmNlSUQ+eG1wLmlpZDo4RTY2QTRFOTc5MTIxMUUyQTQwNkY5NjA1MkYzNjQxQzwveG1wTU06SW5zdGFuY2VJRD4KICAgICAgICAgPHhtcDpDcmVhdG9yVG9vbD5BZG9iZSBQaG90b3Nob3AgQ1M1IE1hY2ludG9zaDwveG1wOkNyZWF0b3JUb29sPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KoZmpuQAAGMZJREFUeAHtnWuMXEV2x29Pzww2tsHjF367ZzAvY/ACSxISsZBVQEJaUADhL4kQwgICCAmifAAkbB42RCs+REGRMB+iKJFWWfMJKVIUBRArEEQBhCyW52JsjHHMw7wx2J6Zzv93+v57anput+eFzXi6pOpTderUqTr/PlW37r1V3VnWDmNCYNu2bWUq3H///Us7xlSzLZzt2bOnGxg6OzuvDxTbmBwRgRIe19PT0/nwww8f3Lx58y+q1eq/lI5YbRoLaGh2LFmypPz0008PPvnkkwNA8eCDD/51qVT653K53NUGr8A58LLnn3++87HHHjuk4ioiDz300BnytodPOOGEaw4fPpwpPdBZUHdasvCyvXv3lpcuXVpdv359v0AITxP/5/K0vxFYN8ycObP8ww8/wMfpytPS8wRESQB1XHfdddmbb75Z2rRp04AACg/Dc+65556F8rArlfwr8X954oknZgINb8MTu8VDLBCMxPH8AVgPPPAAF0e8q7p169b+FCxs37JlyxINxz8fHBy8pqOj45cCrwd+PkQPSZ76cYE1eMflsE3BAgAZe1iEoRjhiSeeyDQcZw8MDJwjxsUq/4uDBw9epGE5GwEAU57hiTd2qDyWJ5SlYaqDVxJQmbyKcYRXlb744otBGYvhdbAw+O677+4Rf62ukn+k7J8IuJ/LwyozZsxgOGaHDh3Kvv/++37xAAxchnkZMqov9lAYnhvi/5RSJXlJ6Y033iidffbZaX8HxR8s6ugdd9xxwuzZs3tl8DqBcYFkmPTPUnqxhmOAIPDCw1TGkERNp2j9psFAmSLgdJ3C/ImEAEl9wYAwgvkJT/Iaq6ifd95551wBskoGnanydYo/U/os0ZXiC6+O8CzA0nwWSwyVxxVTNOZB0bpqpxspAo28oVr16j9+Qh4TADHMtDQoAZLoQDNPokcqO+nAgQPLZECfgDhDoKwR+0zlT1UMj9KQDIAAiijP81VUImEqH6QJqI3gtClMp02LeEMaQs2EP0If8wPz0FiGGi2rXkmetEj3jcuV7lPHTxP7NAF1qugqxSXd3d1dKg/j8CQDpTLmOIaxqg33qFYASH5UQBXJjRW8YfMPnoNSFpYCqqo5qdrKe5Al3H777fMFwlIZtUoArBZrtdK95JVeojhP5RmeJF4MO4DKh91g7lFcGGgfLy7n6ToQ4hWma1Val6V1LV/EawqeQIihJVA6Fi9eXNq3bx/gMEn7Eo6+wnDXXXfN7O/vny+vWCIAVshzeiWI91RkOPPTYsV5XV1dIwBSuUGqql7aFn0tiWfACsGhQza4kRaVTYRn8EqXXnppeeHChTFRS+FAq0kaYPfv3z9fnrBIxjIPrRStyDA8h/Ry8RYJuNkAJH4YZGDsRbQjOZYGhOiL8oQYwtDI1MpHgBKVwvlGAlZUNtm8km5RykVAAdAnn3yySAtGPAdv6VPjFUXSy0Xxnh4BFHNQM4Ak47lIVQIgFp3hPZBakqIhANJ0Ubl5pqOVn2y5+LZvvPHGOZqE18gj1gqEtWqEy35FEYDmpt4jXn0OwpOIkjFAVaXRGeCIBlA11nBw0NPId76orIg3VvkiHRPhdW7YsOEJdeJqKVngBaRBySfoTPMXkzTAxBCj08QaS8xqlQl7xN0KMmmQXEvAUtmpkO6Up93EVQ2AdIsSK/bcaLwmAJIh9qS6TSkQdeY0S3TKu/YLiPk5QMO8pw1Qa2/g6lp/PmVRQGsMjUOwsXw65gFvJFLTEYlx2Ox13Tiqtqu0wZuAD7TBa4M3AQQmULXteeMET6uPahu88YEXzw3b4I0BvHyte1DPGju0Fn6rDd4owctvVXlZdAK3s6L3DLsdG6WeaSWW3G0N6sFJvL/VK8rb7rvvvqfa4DVxBYOGx+lxHU+1O/Tw5HVtu/hbbTF7muedbfAS8AwYLGEFaAzPge+++678oYL2tfzq1Vdf3c17YYkcboMnFAxa7mUBnLws++yzz7IPPviA7WZVpVdI9OVly5b9nbae/ZvSsa0AetwEgACEZsFAuRwP4wIA5TXmV199len1Q7Zjx47sxRdftFhpwYIFg998880ivTH8V8mepkd5G6e05xUBlQJnoCxHGSARLcemHgD79NNP8bJs+/bt2bfffhug8WT9pJNOYtNP9vnnn/Nw+JDqlgXcfXo1sXNKg2cA7B6AZMBSoAwWZYDFXjt7GIC9/vrrVpGx8WfevHmRN7DUwztFuwUcy5Wyyqam52EM4EAdDFa+Bgu2DA2v0TaN7Ouvv8Z7Mr1/zt57770Az3UBi4sD8xyA4WnUdTvIkSbqiwDAQS2UKz9Zz3PHoQSDBTVQppRjLIZrXophh2cBFvOXLpT1oYjsySefnGmjdugxWACGPoLbjEz+AY82TjnlFL6AjksuuaR6zMEr6qgBMjjO2zAmdoxl+GkZEV715ZdfZnoRH2AxfxGop61mGdtimbsYvgCAh1GXtNvHY9N8KMg/aI/Xr7pohOdqE2R2/vnnj3xdmFaarDQdxBCCO0vaoHhOsozlMJKYAoVn4VXaehaeBWjUAyCMYpJfuXIlKupgGDCD4z64PfLmRcX8A3lkdP4iY+gz5Pv6+rKrr746mzt37sSWKjSYdoC0O+E01DEFyR2mg+zKZPhA8Qg6SuSq52FIHhk8BIDYCIQBGIYu9OAhUAB3P9wOlH44uNz5lCJHO3wZfHGs9whXXnlltm7duvii6Muoh23aeJpGqTtlcFIesjYMowDInqR7RLayRgQ0+HTKhqHPRuBZtIMu63MaeddpbJt8UUBXWo92iPDpA97tcNlll2Vr166NYUsdyulbU/AQcgPQxmjFUGQxBGAACOUGyUBB4VGGLHUIqV6WCUQC5dZrsJx3XeSo75DyzTNFjvJUB2DRHmV8ecyZDqeffnp2zjnnZJVKJTwcsLCNQJowArzUmJDQhxu0EQwPQHAEFEca8PChnvWhy9+ujbDeVnmXuS/oI8AfbUh1sCShHwSmhUbAVq9eHXMmFwd/kbbHbbtdtltwJoE8+1E6csGSQQAge5N5yBCLDOCqRAfdYag9zTQta0y7YxOljV8a7fAFf/zxx3XVzGkXXnhhgLVo0aLwMHh4FvZhe6qnXjFPdEogltOae1g1A5TqDcjO2twiOTby8OQUhKUrHtHUv70UEKdN6bBDmkYHIeVZbjSUemldjLVO9xtbGIpcjS3LxUVLDHayZgaLuZQvnPrUxUEI5B2b9alTyjdL+bWKK9WJWfIaeXW5TIPuSO5lA+IRq+KXiFLaIcoEEOch3IgNIY+eiYZUB0AZLPj2EC48XJkBjMCVmAWtJ3ruIljr4VkGi/rEsQCW2lJzAXGuvfbaPik5U8rOEiBniHWq0iuUXqz0nLTD4tXntQRYsQdBKjxVaWh4Kx0kUs9pyQ3Lm29KuQM8Am0x9ACKxTGUKQJgHAGNO4g5c+bEkoJlDTIE9LgP/oJT7zIPWadNi3ilm2++uUvHiTheNCJo1+hMfZMLNQSWqdGVUlSRAX0SXKWOrFBcrDjPwLpjUAyFKmJ57O+TLAYEqAmNdvOyunF4A0PPV2nKAQJQAIeIJ/kOgsnd860Npo77RCPw6avTkcj54+GF592vR8p6TlXWqt2e2HJPMg2xHVcr7nka4aeoU8sEVkXsXnW2V3SVIoAvUme76bDBAVQi4DjtPIYSWAADxqxZswIcKJEhl4JkIKiD/pSSBizHKMx5Lpsoz2BZj2n9yIDuE+sy2vBd1f7l8CQLNqOXX375LIHbI+9ZJKCWyriVAroiY3rlIRUNpT7ReXgTgAAMkzdpIgDiSaozDAB/CY3tpiCRJjTSyebVgYnWmn8EmCr24biSrlxHPLWDuldeeaXr2Wefna/JfJmA7NOctUaedq4A5QDdcoFYFUAcEaiDZIDsTeghjAYgA2b5lKbpyZAzeCM8DS8bzaGUG264YYY8ZqEMXSYAYl5UJ3vVuYooxwqWKN2DJzFxKx3zEEPVIEku0pQRUup0Iz8EG2RTmTQ9Vh2jrRsXjCOd+7r11lt7ZCxzmw+k9CndK+N5fMFJngUC7kSGWToPebLOQYrhLlll4yo8Yqe8yuh3Hbxm6VZyLkvr/lg8e16mi0anHhriPRU1xpmv00VXy9Be8ZaJLpD3lDwHKR/ekgKkOsz4sWueDgMSlKjAB/lapkZcRnlhOhcfVm6eaVr3aPJKN91009Uy8lp14Fw1XBE4c/AgdwJwHCXDcaa4JObl4T3iKTscFPHqOvIiWJPK+7H0jrafnQLut7rSdbF0ACTmIlHWfRyMQw8Lo4jKc96idlctZtp5BMcTJkPHeNqdjDosvQ9opT5HlAcEAZIA5dRzIOdGnFXZpIBmvVOZxnI7B61LhgAmvAAOoNqhOQJ+WtJcol3SFIHwvKal7YKWCLTBawlP68I2eK3xaVnaBq8lPK0L2+C1xqdVafsoQSt0WpTFfWfb81ogVFSkNfGA7vV5uvF5G7wihBJew40CL8AO8+hf4Z/a4CVAFSWT21Jut8oCboa2YjylowQb2+AliDV4WZTwsESBMxglHuZqk+Q/bty48S9hjthuAbMdak+2ebCrZ5gM1bI2Su7Te+Bf3Xvvva8Kn7Kef1ZLGzZs+EpCJ0mg9uQyR87u2kgpnkyedaV603RRuXmmo5VvJgefgD4iD3wBjlef2l5Wffvtt0vPPPMMb9N/c9FFF2186aWXPlJafjiNQgp2PhwDJIAiUs7zTPYvs5FRv6OVvfbaazxhquoB8Qy9M77x5Zdfvkb5KxVfOO7AY95KQcI34JlSBlD2Lvh4GICxiXHXrl3ZCy+8UK/Di3XVKbFTQReKg9I1V/V/J/DPP+7AM0hQB3sWgBHwLu9lZv8yG77lYcGnnHfILEe8txBe/qVwbOp7ATdTF49/mNLg2aMwjoBX2bNMkeEVA/ta0uMEu3fv5kfyaxX1yQt3tnIgjyw7Q63DQxxhgNNFpCpgf3HMwcu/0boRJOh0YygCyvMUlIAMhmI8kV1THCdgTx7exbECB8CaP39+vEumHgDjjQYKL4WftosMG4qks3Teeecd+6VKClTaURsJBRyMQdbyGIkx6a4pvMVnL3bu3FkHgreBPnuBvvwlV5QzNNHltmnLACLrQB10IE/QVffogEfHbLTT7qw7R3nqSZbHEDrOvjsiHuX5CrA4UsBET5rA3hcmeTYw2nsAmciFgXbddgqa++Ey8qTpB4tj5kC+GML111+fLV++fPLPYbhxGnXj0aI+4DVGl2EIxhH5dgGp8TgBe4jhIwMwGIVXseOTPcQEg23A6IMjbTsglwb3Fx5l6GZoUxfQ+MJ6e3uzK664AuDiCx3XnIfCNLhhqL3HHTW1ATbKIKXeRAcBDB7lBPQRAcs7qOBbH4YSkYeaj4zbtjy0MSBDHbybwJ4aIl+gN3vjxRdffHHGDnmuxNhAvSOCh2I3AC2K7pA7QUcMEnMSHSEaKOdtMDoNEJS9eAT0GRzSzjudUuTR40BZUXAdl5HHy/hiSPvoFeUcWGFbLieKuBJjF32mj4Q6eFR04ylACJlPGjmUOKKMCEhphAeAyAGAA7po3F6EPqJBMjW/kVrPaKnrW94XHvKU0U/PZfA4HqXdYQEYV1aDim3GBTkCOwNifGCQQUKpjYYaCGiaxlDK3UGoA7qYj4gEypAvokX1rWestFEXXmVPoa94PTvkSTuce+65WaVSiYuMAaMOMiloxsf12KvSg1EC5bA8papYUrIsypkMdr3XjaYSnXOgAaI7jCyBvIFKy5yGTlZo1AlYNhLjucD47Bht8mUyd+m3AmK3PGs9rs7McwT6TT0iAV3WF4zko1MLyZcE1J9JIGq7Ip0SMGz24beMAYQJJTZjq4GYXOClnSdNMI1MwUeur6Bk9Cx0EPnyCLTJqPDC2Jr0A9nZBRdcEOcuAIrd8sypDEcvZQCMugTrQzfBNDINH0h0XXXVVfz1wflKrxN4a9SRXqXZ0Rk/iUnHaKAxSkYsb3SvLwkCUPEDaMkMAxgdBOuEFkXLNFLq8wXrC48hyPKFqzQep92sEVm2MPxY1LI+Ayg8zsCkbQNOGmmPYNBMi3g1eEN86IMjBFoyAN5KdfJUlbDJsU9xlSKbs1lUzbRi5QMAjCKNgUSleZCoZABMPgAVjT0y5N2q0pGEEqkPQES8whEhwAAUwPG5C/L2KIDEqxysE+o+Q9O0ZRt5zlPudJ3er2ME/HsAu97z3e64RtNJCWA1NE7RxLtMcqsUK0QBt1KUHaT85DlHsrgYiRR7GeA4ArrBASz4eAnzEAtVQAEcAwQPAClP5ziD5DbdPjRNU+58mh4rr/7NoyQJscH7ueee69DkGjLvvvtuVfnawaxEME1qbunSeoifDeZwC4dcVgqYVYorFJcKmIWK7G+epbKZihwcjFsqwAAUR/JEbrcYcnhSM2+iDzYcmqbTsjRtmYnwmoGHzmEBDxUDL4Xy/19RVwZW33///X79TFBtxh1Wayij+t0fffTRafLY88T9U3nX5QLuVHlPVTFeruBFgElMg0COoZzyDJJBMEXG6UZaVDYR3gjwAInTQLolKen2pCRjB4/0VzJ0QPU69XxsvrxjsYznXwlWifaKVlTMEQP4PUrPEmUvs5K1ADjNguWgaRp559P00eTFBH7LLbd0cnRKK+t+gVC7HBZYc9ttt82W3Qy9pSpeoY6uUqwoDV0uyh+B8EchsWtesuExgOP5rREoG3skKr11sBpli8qOBm/o66c1BV0QurUeAhwuApy3cAQgwFkoGv9zkc5BKTgCiC8g/usiNzSGvHjK1lwuJ2KNBGW0vMnQMdq2iuQ69WxqvuaaS+Ulf6zO8O9N/EER/6Ayi8ka7yHgMY4ApduWOFagOt41r2RtQ7jESdfuy6L2yA90SWZkwRTidOpK9r+a9PvcZwEXIOUAAUwMY1EmJpEw2J4UiykDYYoup02t3zTXM6UB5KnKUm6WFfgTSP8RkUECqfqTF4RSMAwA/OkY8KAfchC4FcOT4AGao5LtUISAhx9lzdcLRTXbvPCyNgzjRADPa4dxItAGb5zAUa0NXhu8CSAwzqpambSPEowTOy13q0P/JDxOJdOuGh6H0aKH23PeGL9+edwBPdnmBuI/ht16jVHPtBD37WjucT9ou8UsvXDar2cAd7Y9L3EBgEoDgCnycISn5CW9Q5mp98BvKX2RnnvuboOXoCWQnONJ0mGByVGpDj114vHagF5z/lpPm36mAyx/4Mn5tB22yXA0YGJV+4UVwOlRZu2f6PWu5ht527+r7NcC7D2EAU6x/7gDz6BgZKuAl4GWZHjiDWDdAiz+6JhHdIqvif9bPfT9zZYtWz5EF6Bt2rQJ+XiLeNwdYsFIgEmp0sxdAOVlBnlGZAfvfnmdwPY3DcnXxf9PxacEVP3/HLZu3dqlDZSD69evH9odJGVT2vOKvEyGh0eJsk2ByT6AwlZebQIUkSfmetH+hTzsFZX9t2SfFn+7QIsn5+JlSnfrTWJVL8gKX6tOafAAKg8A5eEndil++FRgCY/atgt2JOit36cC7U3V+R853e9U9rIAqv38dq5o27ZtZf1dTUnvp6sq4+l603DMwSvynma9layHH95BliFaZvjpXUwHQAEoXqW56nvR95VnKL4qWeLvH3nkkdqv7ueNCCD/auWgflhxxNBs1hf4xxw8jM2DwfDcFPNTXoYQGPE+mBhAUSZvIn4nsHZKF161XXPXdoH5ts5d7C74PVS2kvjNnt9T14cqOkcbjip4OVABDkgpHx6Ud1bZ+KEvQIp5CZrXCW+SJx0SUP+HR6nOWyr7vegbGp47Nm/evFf5FPBQa89iB4QYg/nVsuVwjIqj+JjQ1Rb9Nq6R5mV+dRkgSSY8SGVMRvXJGzaR153MTcL1S+X3Se4DxfcE4jui7wi497WtbI8Aqf0YsphpACjlY+G/Zs2aquauqsDyUE9FJyU9avBorWZ7tEuHIiEanpRQ+BggfIYAwosIAiaGmoD6Ttl9qrdHPIbcDsnskFft0vLhQ+0b/rhgyIUOexMZeRTtD4o3rqEXCsf50XTYyqAYVhgLMIpuQsnIyNbY0RQ/lpqnQ4Y6eJAA+lLxY3nMbvF2SYbhtlO8D5Teq5X7ZwLogBU3UrrAPhpW+9oCEiBJxkAddbAa+wd4RqVfnQUkOskupvihQUDBg6A1zGreIw/Bi7ii8YRhn8oBaKfiLsnvVP5DraH2aafVfl3Fhi0uGzvh4aY1lfvCsSjAGWzmfY06jkWe3ZscwJ0jg+PWBJAI8o4YXgImvEdy3KLsFgUghtlu1dmjm+ZPHn30UYZg0wA47D7VLqwAhzUUwvwaLlTl0BEbJ9UGxT/ZwG+GcgScdRH/pLFDHX6HKID+IHB2acjt1VmrT2RgyyuUyjvzjY+cBaviOVpwxgr/J2v9BDvGBePvBdR/aZJ+4/HHHx86kFqg2ABpe36JOYhbl3zjI0MsvKig2nHL+n/QmwFoEbRBhgAAAABJRU5ErkJggg==","streamsets-spark-file-lib:com_streamsets_pipeline_spark_origin_fs_FileDOrigin":"iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAD9CzEMAAAAe0lEQVR4Ae3VMQ7CMBBE0X+a4GNyIKSQHCxExCUw1BZGFN5t0Pzp/SQ3i3Ou6cSVO/qyM4NN3FC7WOKCUCaxoVzigUJWWSl0eqKw7ZRcQCzZwJEN6B+BlwEDw4AMGPg9AwZq9tFfQ4GZjwo7CtrGBD1i4Qj4nLl93jn3BjR5layvzA8cAAAAAElFTkSuQmCC"},"legacyStageLibs":null,"eventDefinitions":null,"version":null,"executorVersion":null,"category":null,"categoryLabel":null,"stageDefinitionMap":null,"stageDefinitionMinimalList":null}
}