From 14d8604eacb922cff78018dcb99175573cc75b02 Mon Sep 17 00:00:00 2001 From: Rui Wang Date: Wed, 19 Oct 2022 10:47:53 +0800 Subject: [PATCH] [SPARK-40823][CONNECT] Connect Proto should carry unparsed identifiers ### What changes were proposed in this pull request? Before this PR, connect proto defines the UnresolvedRelation as ``` message NamedTable { repeated string parts = 1; } ``` which asked clients to provide multiple name parts of the relation. For example, user could offer `a.b.c.d` as the table name. However, this actually asks clients to implement `CatalystSqlParser.parseMultipartIdentifier`. The problem to clients is they cannot access the catalyst parser thus needs to re-invent the wheel. Another problem is clients might not be able to implement the parsing correctly. This PR proposes to change the proto to ``` message NamedTable { string unparsed_identifier = 1; } ``` which only needs clients to provide the user's table name. Server side as it can access catalyst, can parse the identifier. This PR also changes on the Column identifier accordingly. ### Why are the changes needed? This proposal reduced the required work on the client side. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing UT Closes #38264 from amaliujia/unparsed_identifier. Authored-by: Rui Wang Signed-off-by: Wenchen Fan --- .../protobuf/spark/connect/expressions.proto | 2 +- .../protobuf/spark/connect/relations.proto | 2 +- .../spark/sql/connect/dsl/package.scala | 6 +- .../connect/planner/SparkConnectPlanner.scala | 7 +- .../planner/SparkConnectPlannerSuite.scala | 19 ++--- python/pyspark/sql/connect/column.py | 14 ++-- python/pyspark/sql/connect/plan.py | 10 ++- .../sql/connect/proto/expressions_pb2.py | 26 +++---- .../sql/connect/proto/expressions_pb2.pyi | 14 ++-- .../sql/connect/proto/relations_pb2.py | 70 +++++++++---------- .../sql/connect/proto/relations_pb2.pyi | 14 ++-- 11 files changed, 91 insertions(+), 93 deletions(-) diff --git a/connector/connect/src/main/protobuf/spark/connect/expressions.proto b/connector/connect/src/main/protobuf/spark/connect/expressions.proto index 4b5a81d2a568c..426e04341d9f4 100644 --- a/connector/connect/src/main/protobuf/spark/connect/expressions.proto +++ b/connector/connect/src/main/protobuf/spark/connect/expressions.proto @@ -142,7 +142,7 @@ message Expression { // An unresolved attribute that is not explicitly bound to a specific column, but the column // is resolved during analysis by name. message UnresolvedAttribute { - repeated string parts = 1; + string unparsed_identifier = 1; } // An unresolved function is not explicitly bound to one explicit function, but the function diff --git a/connector/connect/src/main/protobuf/spark/connect/relations.proto b/connector/connect/src/main/protobuf/spark/connect/relations.proto index e93bb02894a09..618720ef4931a 100644 --- a/connector/connect/src/main/protobuf/spark/connect/relations.proto +++ b/connector/connect/src/main/protobuf/spark/connect/relations.proto @@ -69,7 +69,7 @@ message Read { } message NamedTable { - repeated string parts = 1; + string unparsed_identifier = 1; } } diff --git a/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala b/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala index 0c392130562d8..7b8b58e1abac5 100644 --- a/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala +++ b/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala @@ -22,7 +22,6 @@ import scala.language.implicitConversions import org.apache.spark.connect.proto import org.apache.spark.connect.proto.Join.JoinType import org.apache.spark.sql.SaveMode -import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.connect.planner.DataTypeProtoConverter /** @@ -33,16 +32,13 @@ package object dsl { object expressions { // scalastyle:ignore implicit class DslString(val s: String) { - val identifier = CatalystSqlParser.parseMultipartIdentifier(s) - def protoAttr: proto.Expression = proto.Expression .newBuilder() .setUnresolvedAttribute( proto.Expression.UnresolvedAttribute .newBuilder() - .addAllParts(identifier.asJava) - .build()) + .setUnparsedIdentifier(s)) .build() def struct( diff --git a/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 7ffce9082218c..a5606f278f909 100644 --- a/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Expression} +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.plans.{logical, FullOuter, Inner, JoinType, LeftAnti, LeftOuter, LeftSemi, RightOuter} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Sample, SubqueryAlias} import org.apache.spark.sql.types._ @@ -102,7 +103,9 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { common: Option[proto.RelationCommon]): LogicalPlan = { val baseRelation = rel.getReadTypeCase match { case proto.Read.ReadTypeCase.NAMED_TABLE => - val child = UnresolvedRelation(rel.getNamedTable.getPartsList.asScala.toSeq) + val multipartIdentifier = + CatalystSqlParser.parseMultipartIdentifier(rel.getNamedTable.getUnparsedIdentifier) + val child = UnresolvedRelation(multipartIdentifier) if (common.nonEmpty && common.get.getAlias.nonEmpty) { SubqueryAlias(identifier = common.get.getAlias, child = child) } else { @@ -139,7 +142,7 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { } private def transformUnresolvedExpression(exp: proto.Expression): UnresolvedAttribute = { - UnresolvedAttribute(exp.getUnresolvedAttribute.getPartsList.asScala.toSeq) + UnresolvedAttribute(exp.getUnresolvedAttribute.getUnparsedIdentifier) } private def transformExpression(exp: proto.Expression): Expression = { diff --git a/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala index 74788ce5593a8..ef73eb8d21e2a 100644 --- a/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala +++ b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala @@ -41,7 +41,7 @@ trait SparkConnectPlanTest { .setRead( proto.Read .newBuilder() - .setNamedTable(proto.Read.NamedTable.newBuilder().addParts("table")) + .setNamedTable(proto.Read.NamedTable.newBuilder().setUnparsedIdentifier("table")) .build()) .build() @@ -100,7 +100,7 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { // Invalid read without Table name. intercept[InvalidPlanInput](transform(proto.Relation.newBuilder.setRead(read).build())) val readWithTable = read.toBuilder - .setNamedTable(proto.Read.NamedTable.newBuilder.addParts("name").build()) + .setNamedTable(proto.Read.NamedTable.newBuilder.setUnparsedIdentifier("name").build()) .build() val res = transform(proto.Relation.newBuilder.setRead(readWithTable).build()) assert(res !== null) @@ -110,7 +110,7 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { test("Simple Project") { val readWithTable = proto.Read .newBuilder() - .setNamedTable(proto.Read.NamedTable.newBuilder.addParts("name").build()) + .setNamedTable(proto.Read.NamedTable.newBuilder.setUnparsedIdentifier("name").build()) .build() val project = proto.Project @@ -139,10 +139,11 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { .newBuilder() .setNulls(proto.Sort.SortNulls.SORT_NULLS_LAST) .setDirection(proto.Sort.SortDirection.SORT_DIRECTION_DESCENDING) - .setExpression(proto.Expression.newBuilder - .setUnresolvedAttribute( - proto.Expression.UnresolvedAttribute.newBuilder.addAllParts(Seq("col").asJava).build()) - .build()) + .setExpression( + proto.Expression.newBuilder + .setUnresolvedAttribute( + proto.Expression.UnresolvedAttribute.newBuilder.setUnparsedIdentifier("col").build()) + .build()) .build() val res = transform( @@ -192,7 +193,7 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { val unresolvedAttribute = proto.Expression .newBuilder() .setUnresolvedAttribute( - proto.Expression.UnresolvedAttribute.newBuilder().addAllParts(Seq("left").asJava).build()) + proto.Expression.UnresolvedAttribute.newBuilder().setUnparsedIdentifier("left").build()) .build() val joinCondition = proto.Expression.newBuilder.setUnresolvedFunction( @@ -236,7 +237,7 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { val unresolvedAttribute = proto.Expression .newBuilder() .setUnresolvedAttribute( - proto.Expression.UnresolvedAttribute.newBuilder().addAllParts(Seq("left").asJava).build()) + proto.Expression.UnresolvedAttribute.newBuilder().setUnparsedIdentifier("left").build()) .build() val agg = proto.Aggregate.newBuilder diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py index b291c5fb21163..d0890396fd50c 100644 --- a/python/pyspark/sql/connect/column.py +++ b/python/pyspark/sql/connect/column.py @@ -15,7 +15,7 @@ # limitations under the License. # -from typing import List, cast, get_args, TYPE_CHECKING, Optional, Callable, Any +from typing import cast, get_args, TYPE_CHECKING, Optional, Callable, Any import pyspark.sql.connect.proto as proto @@ -121,20 +121,20 @@ class ColumnRef(Expression): @classmethod def from_qualified_name(cls, name: str) -> "ColumnRef": - return ColumnRef(*name.split(".")) + return ColumnRef(name) - def __init__(self, *parts: str) -> None: + def __init__(self, name: str) -> None: super().__init__() - self._parts: List[str] = list(parts) + self._unparsed_identifier: str = name def name(self) -> str: """Returns the qualified name of the column reference.""" - return ".".join(self._parts) + return self._unparsed_identifier def to_plan(self, session: Optional["RemoteSparkSession"]) -> proto.Expression: """Returns the Proto representation of the expression.""" expr = proto.Expression() - expr.unresolved_attribute.parts.extend(self._parts) + expr.unresolved_attribute.unparsed_identifier = self._unparsed_identifier return expr def desc(self) -> "SortOrder": @@ -144,7 +144,7 @@ def asc(self) -> "SortOrder": return SortOrder(self, ascending=True) def __str__(self) -> str: - return f"Column({'.'.join(self._parts)})" + return f"Column({self._unparsed_identifier})" class SortOrder(Expression): diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index 9351998c19561..7bb2a3356c8f6 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -49,10 +49,10 @@ class LogicalPlan(object): def __init__(self, child: Optional["LogicalPlan"]) -> None: self._child = child - def unresolved_attr(self, *colNames: str) -> proto.Expression: + def unresolved_attr(self, colName: str) -> proto.Expression: """Creates an unresolved attribute from a column name.""" exp = proto.Expression() - exp.unresolved_attribute.parts.extend(list(colNames)) + exp.unresolved_attribute.unparsed_identifier = colName return exp def to_attr_or_expression( @@ -118,7 +118,7 @@ def __init__(self, table_name: str) -> None: def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: plan = proto.Relation() - plan.read.named_table.parts.extend(self.table_name.split(".")) + plan.read.named_table.unparsed_identifier = self.table_name return plan def print(self, indent: int = 0) -> str: @@ -165,9 +165,7 @@ def withAlias(self, alias: str) -> LogicalPlan: def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: assert self._child is not None proj_exprs = [ - c.to_plan(session) - if isinstance(c, Expression) - else self.unresolved_attr(*(c.split("."))) + c.to_plan(session) if isinstance(c, Expression) else self.unresolved_attr(c) for c in self._raw_columns ] common = proto.RelationCommon() diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py index 84aa22d8bde39..89718650571f4 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.py +++ b/python/pyspark/sql/connect/proto/expressions_pb2.py @@ -33,7 +33,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto\x1a\x19google/protobuf/any.proto"\xa7\x17\n\nExpression\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x1a\xa3\x10\n\x07Literal\x12\x1a\n\x07\x62oolean\x18\x01 \x01(\x08H\x00R\x07\x62oolean\x12\x10\n\x02i8\x18\x02 \x01(\x05H\x00R\x02i8\x12\x12\n\x03i16\x18\x03 \x01(\x05H\x00R\x03i16\x12\x12\n\x03i32\x18\x05 \x01(\x05H\x00R\x03i32\x12\x12\n\x03i64\x18\x07 \x01(\x03H\x00R\x03i64\x12\x14\n\x04\x66p32\x18\n \x01(\x02H\x00R\x04\x66p32\x12\x14\n\x04\x66p64\x18\x0b \x01(\x01H\x00R\x04\x66p64\x12\x18\n\x06string\x18\x0c \x01(\tH\x00R\x06string\x12\x18\n\x06\x62inary\x18\r \x01(\x0cH\x00R\x06\x62inary\x12\x1e\n\ttimestamp\x18\x0e \x01(\x03H\x00R\ttimestamp\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x14\n\x04time\x18\x11 \x01(\x03H\x00R\x04time\x12l\n\x16interval_year_to_month\x18\x13 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalYearToMonthH\x00R\x13intervalYearToMonth\x12l\n\x16interval_day_to_second\x18\x14 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalDayToSecondH\x00R\x13intervalDayToSecond\x12\x1f\n\nfixed_char\x18\x15 \x01(\tH\x00R\tfixedChar\x12\x46\n\x08var_char\x18\x16 \x01(\x0b\x32).spark.connect.Expression.Literal.VarCharH\x00R\x07varChar\x12#\n\x0c\x66ixed_binary\x18\x17 \x01(\x0cH\x00R\x0b\x66ixedBinary\x12\x45\n\x07\x64\x65\x63imal\x18\x18 \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x42\n\x06struct\x18\x19 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x39\n\x03map\x18\x1a \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12#\n\x0ctimestamp_tz\x18\x1b \x01(\x03H\x00R\x0btimestampTz\x12\x14\n\x04uuid\x18\x1c \x01(\x0cH\x00R\x04uuid\x12-\n\x04null\x18\x1d \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12<\n\x04list\x18\x1e \x01(\x0b\x32&.spark.connect.Expression.Literal.ListH\x00R\x04list\x12=\n\nempty_list\x18\x1f \x01(\x0b\x32\x1c.spark.connect.DataType.ListH\x00R\temptyList\x12:\n\tempty_map\x18 \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x08\x65mptyMap\x12R\n\x0cuser_defined\x18! \x01(\x0b\x32-.spark.connect.Expression.Literal.UserDefinedH\x00R\x0buserDefined\x12\x1a\n\x08nullable\x18\x32 \x01(\x08R\x08nullable\x12\x38\n\x18type_variation_reference\x18\x33 \x01(\rR\x16typeVariationReference\x1a\x37\n\x07VarChar\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12\x16\n\x06length\x18\x02 \x01(\rR\x06length\x1aS\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value\x12\x1c\n\tprecision\x18\x02 \x01(\x05R\tprecision\x12\x14\n\x05scale\x18\x03 \x01(\x05R\x05scale\x1a\xce\x01\n\x03Map\x12M\n\nkey_values\x18\x01 \x03(\x0b\x32..spark.connect.Expression.Literal.Map.KeyValueR\tkeyValues\x1ax\n\x08KeyValue\x12\x33\n\x03key\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value\x1a\x43\n\x13IntervalYearToMonth\x12\x14\n\x05years\x18\x01 \x01(\x05R\x05years\x12\x16\n\x06months\x18\x02 \x01(\x05R\x06months\x1ag\n\x13IntervalDayToSecond\x12\x12\n\x04\x64\x61ys\x18\x01 \x01(\x05R\x04\x64\x61ys\x12\x18\n\x07seconds\x18\x02 \x01(\x05R\x07seconds\x12"\n\x0cmicroseconds\x18\x03 \x01(\x05R\x0cmicroseconds\x1a\x43\n\x06Struct\x12\x39\n\x06\x66ields\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06\x66ields\x1a\x41\n\x04List\x12\x39\n\x06values\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a`\n\x0bUserDefined\x12%\n\x0etype_reference\x18\x01 \x01(\rR\rtypeReference\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05valueB\x0e\n\x0cliteral_type\x1a+\n\x13UnresolvedAttribute\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x1a\x63\n\x12UnresolvedFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a\x10\n\x0eUnresolvedStar\x1aU\n\x12QualifiedAttribute\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12+\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x04type\x1aJ\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x01(\tR\x04nameB\x0b\n\texpr_typeB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto\x1a\x19google/protobuf/any.proto"\xc2\x17\n\nExpression\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x1a\xa3\x10\n\x07Literal\x12\x1a\n\x07\x62oolean\x18\x01 \x01(\x08H\x00R\x07\x62oolean\x12\x10\n\x02i8\x18\x02 \x01(\x05H\x00R\x02i8\x12\x12\n\x03i16\x18\x03 \x01(\x05H\x00R\x03i16\x12\x12\n\x03i32\x18\x05 \x01(\x05H\x00R\x03i32\x12\x12\n\x03i64\x18\x07 \x01(\x03H\x00R\x03i64\x12\x14\n\x04\x66p32\x18\n \x01(\x02H\x00R\x04\x66p32\x12\x14\n\x04\x66p64\x18\x0b \x01(\x01H\x00R\x04\x66p64\x12\x18\n\x06string\x18\x0c \x01(\tH\x00R\x06string\x12\x18\n\x06\x62inary\x18\r \x01(\x0cH\x00R\x06\x62inary\x12\x1e\n\ttimestamp\x18\x0e \x01(\x03H\x00R\ttimestamp\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x14\n\x04time\x18\x11 \x01(\x03H\x00R\x04time\x12l\n\x16interval_year_to_month\x18\x13 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalYearToMonthH\x00R\x13intervalYearToMonth\x12l\n\x16interval_day_to_second\x18\x14 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalDayToSecondH\x00R\x13intervalDayToSecond\x12\x1f\n\nfixed_char\x18\x15 \x01(\tH\x00R\tfixedChar\x12\x46\n\x08var_char\x18\x16 \x01(\x0b\x32).spark.connect.Expression.Literal.VarCharH\x00R\x07varChar\x12#\n\x0c\x66ixed_binary\x18\x17 \x01(\x0cH\x00R\x0b\x66ixedBinary\x12\x45\n\x07\x64\x65\x63imal\x18\x18 \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x42\n\x06struct\x18\x19 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x39\n\x03map\x18\x1a \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12#\n\x0ctimestamp_tz\x18\x1b \x01(\x03H\x00R\x0btimestampTz\x12\x14\n\x04uuid\x18\x1c \x01(\x0cH\x00R\x04uuid\x12-\n\x04null\x18\x1d \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12<\n\x04list\x18\x1e \x01(\x0b\x32&.spark.connect.Expression.Literal.ListH\x00R\x04list\x12=\n\nempty_list\x18\x1f \x01(\x0b\x32\x1c.spark.connect.DataType.ListH\x00R\temptyList\x12:\n\tempty_map\x18 \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x08\x65mptyMap\x12R\n\x0cuser_defined\x18! \x01(\x0b\x32-.spark.connect.Expression.Literal.UserDefinedH\x00R\x0buserDefined\x12\x1a\n\x08nullable\x18\x32 \x01(\x08R\x08nullable\x12\x38\n\x18type_variation_reference\x18\x33 \x01(\rR\x16typeVariationReference\x1a\x37\n\x07VarChar\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12\x16\n\x06length\x18\x02 \x01(\rR\x06length\x1aS\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value\x12\x1c\n\tprecision\x18\x02 \x01(\x05R\tprecision\x12\x14\n\x05scale\x18\x03 \x01(\x05R\x05scale\x1a\xce\x01\n\x03Map\x12M\n\nkey_values\x18\x01 \x03(\x0b\x32..spark.connect.Expression.Literal.Map.KeyValueR\tkeyValues\x1ax\n\x08KeyValue\x12\x33\n\x03key\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value\x1a\x43\n\x13IntervalYearToMonth\x12\x14\n\x05years\x18\x01 \x01(\x05R\x05years\x12\x16\n\x06months\x18\x02 \x01(\x05R\x06months\x1ag\n\x13IntervalDayToSecond\x12\x12\n\x04\x64\x61ys\x18\x01 \x01(\x05R\x04\x64\x61ys\x12\x18\n\x07seconds\x18\x02 \x01(\x05R\x07seconds\x12"\n\x0cmicroseconds\x18\x03 \x01(\x05R\x0cmicroseconds\x1a\x43\n\x06Struct\x12\x39\n\x06\x66ields\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06\x66ields\x1a\x41\n\x04List\x12\x39\n\x06values\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a`\n\x0bUserDefined\x12%\n\x0etype_reference\x18\x01 \x01(\rR\rtypeReference\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05valueB\x0e\n\x0cliteral_type\x1a\x46\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x1a\x63\n\x12UnresolvedFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a\x10\n\x0eUnresolvedStar\x1aU\n\x12QualifiedAttribute\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12+\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x04type\x1aJ\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x01(\tR\x04nameB\x0b\n\texpr_typeB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -43,7 +43,7 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001" _EXPRESSION._serialized_start = 105 - _EXPRESSION._serialized_end = 3088 + _EXPRESSION._serialized_end = 3115 _EXPRESSION_LITERAL._serialized_start = 613 _EXPRESSION_LITERAL._serialized_end = 2696 _EXPRESSION_LITERAL_VARCHAR._serialized_start = 1923 @@ -65,15 +65,15 @@ _EXPRESSION_LITERAL_USERDEFINED._serialized_start = 2584 _EXPRESSION_LITERAL_USERDEFINED._serialized_end = 2680 _EXPRESSION_UNRESOLVEDATTRIBUTE._serialized_start = 2698 - _EXPRESSION_UNRESOLVEDATTRIBUTE._serialized_end = 2741 - _EXPRESSION_UNRESOLVEDFUNCTION._serialized_start = 2743 - _EXPRESSION_UNRESOLVEDFUNCTION._serialized_end = 2842 - _EXPRESSION_EXPRESSIONSTRING._serialized_start = 2844 - _EXPRESSION_EXPRESSIONSTRING._serialized_end = 2894 - _EXPRESSION_UNRESOLVEDSTAR._serialized_start = 2896 - _EXPRESSION_UNRESOLVEDSTAR._serialized_end = 2912 - _EXPRESSION_QUALIFIEDATTRIBUTE._serialized_start = 2914 - _EXPRESSION_QUALIFIEDATTRIBUTE._serialized_end = 2999 - _EXPRESSION_ALIAS._serialized_start = 3001 - _EXPRESSION_ALIAS._serialized_end = 3075 + _EXPRESSION_UNRESOLVEDATTRIBUTE._serialized_end = 2768 + _EXPRESSION_UNRESOLVEDFUNCTION._serialized_start = 2770 + _EXPRESSION_UNRESOLVEDFUNCTION._serialized_end = 2869 + _EXPRESSION_EXPRESSIONSTRING._serialized_start = 2871 + _EXPRESSION_EXPRESSIONSTRING._serialized_end = 2921 + _EXPRESSION_UNRESOLVEDSTAR._serialized_start = 2923 + _EXPRESSION_UNRESOLVEDSTAR._serialized_end = 2939 + _EXPRESSION_QUALIFIEDATTRIBUTE._serialized_start = 2941 + _EXPRESSION_QUALIFIEDATTRIBUTE._serialized_end = 3026 + _EXPRESSION_ALIAS._serialized_start = 3028 + _EXPRESSION_ALIAS._serialized_end = 3102 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi index 534c427e36baa..ffaf03eade88e 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi +++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi @@ -536,17 +536,17 @@ class Expression(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - PARTS_FIELD_NUMBER: builtins.int - @property - def parts( - self, - ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + UNPARSED_IDENTIFIER_FIELD_NUMBER: builtins.int + unparsed_identifier: builtins.str def __init__( self, *, - parts: collections.abc.Iterable[builtins.str] | None = ..., + unparsed_identifier: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal["unparsed_identifier", b"unparsed_identifier"], ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["parts", b"parts"]) -> None: ... class UnresolvedFunction(google.protobuf.message.Message): """An unresolved function is not explicitly bound to one explicit function, but the function diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py index 99116f1d59af3..4b5c4e60d57db 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.py +++ b/python/pyspark/sql/connect/proto/relations_pb2.py @@ -32,7 +32,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\x9e\x05\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12,\n\x05union\x18\x06 \x01(\x0b\x32\x14.spark.connect.UnionH\x00R\x05union\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05\x66\x65tch\x18\x08 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"G\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias"\x1b\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query"z\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x1a"\n\nNamedTable\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05partsB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x9d\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType"\xbb\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06"\xcd\x01\n\x05Union\x12/\n\x06inputs\x18\x01 \x03(\x0b\x32\x17.spark.connect.RelationR\x06inputs\x12=\n\nunion_type\x18\x02 \x01(\x0e\x32\x1e.spark.connect.Union.UnionTypeR\tunionType"T\n\tUnionType\x12\x1a\n\x16UNION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13UNION_TYPE_DISTINCT\x10\x01\x12\x12\n\x0eUNION_TYPE_ALL\x10\x02"d\n\x05\x46\x65tch\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit\x12\x16\n\x06offset\x18\x03 \x01(\x05R\x06offset"\xc5\x02\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12Y\n\x12result_expressions\x18\x03 \x03(\x0b\x32*.spark.connect.Aggregate.AggregateFunctionR\x11resultExpressions\x1a`\n\x11\x41ggregateFunction\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\xf6\x03\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\x0bsort_fields\x18\x02 \x03(\x0b\x32\x1d.spark.connect.Sort.SortFieldR\nsortFields\x1a\xbc\x01\n\tSortField\x12\x39\n\nexpression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nexpression\x12?\n\tdirection\x18\x02 \x01(\x0e\x32!.spark.connect.Sort.SortDirectionR\tdirection\x12\x33\n\x05nulls\x18\x03 \x01(\x0e\x32\x1d.spark.connect.Sort.SortNullsR\x05nulls"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"R\n\tSortNulls\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02"]\n\rLocalRelation\x12L\n\nattributes\x18\x01 \x03(\x0b\x32,.spark.connect.Expression.QualifiedAttributeR\nattributes"\xb8\x01\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12)\n\x10with_replacement\x18\x04 \x01(\x08R\x0fwithReplacement\x12\x12\n\x04seed\x18\x05 \x01(\x03R\x04seedB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\x9e\x05\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12,\n\x05union\x18\x06 \x01(\x0b\x32\x14.spark.connect.UnionH\x00R\x05union\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05\x66\x65tch\x18\x08 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"G\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias"\x1b\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query"\x95\x01\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x1a=\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifierB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x9d\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType"\xbb\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06"\xcd\x01\n\x05Union\x12/\n\x06inputs\x18\x01 \x03(\x0b\x32\x17.spark.connect.RelationR\x06inputs\x12=\n\nunion_type\x18\x02 \x01(\x0e\x32\x1e.spark.connect.Union.UnionTypeR\tunionType"T\n\tUnionType\x12\x1a\n\x16UNION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13UNION_TYPE_DISTINCT\x10\x01\x12\x12\n\x0eUNION_TYPE_ALL\x10\x02"d\n\x05\x46\x65tch\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit\x12\x16\n\x06offset\x18\x03 \x01(\x05R\x06offset"\xc5\x02\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12Y\n\x12result_expressions\x18\x03 \x03(\x0b\x32*.spark.connect.Aggregate.AggregateFunctionR\x11resultExpressions\x1a`\n\x11\x41ggregateFunction\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\xf6\x03\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\x0bsort_fields\x18\x02 \x03(\x0b\x32\x1d.spark.connect.Sort.SortFieldR\nsortFields\x1a\xbc\x01\n\tSortField\x12\x39\n\nexpression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nexpression\x12?\n\tdirection\x18\x02 \x01(\x0e\x32!.spark.connect.Sort.SortDirectionR\tdirection\x12\x33\n\x05nulls\x18\x03 \x01(\x0e\x32\x1d.spark.connect.Sort.SortNullsR\x05nulls"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"R\n\tSortNulls\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02"]\n\rLocalRelation\x12L\n\nattributes\x18\x01 \x03(\x0b\x32,.spark.connect.Expression.QualifiedAttributeR\nattributes"\xb8\x01\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12)\n\x10with_replacement\x18\x04 \x01(\x08R\x0fwithReplacement\x12\x12\n\x04seed\x18\x05 \x01(\x03R\x04seedB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -49,38 +49,38 @@ _RELATIONCOMMON._serialized_end = 836 _SQL._serialized_start = 838 _SQL._serialized_end = 865 - _READ._serialized_start = 867 - _READ._serialized_end = 989 - _READ_NAMEDTABLE._serialized_start = 942 - _READ_NAMEDTABLE._serialized_end = 976 - _PROJECT._serialized_start = 991 - _PROJECT._serialized_end = 1108 - _FILTER._serialized_start = 1110 - _FILTER._serialized_end = 1222 - _JOIN._serialized_start = 1225 - _JOIN._serialized_end = 1638 - _JOIN_JOINTYPE._serialized_start = 1451 - _JOIN_JOINTYPE._serialized_end = 1638 - _UNION._serialized_start = 1641 - _UNION._serialized_end = 1846 - _UNION_UNIONTYPE._serialized_start = 1762 - _UNION_UNIONTYPE._serialized_end = 1846 - _FETCH._serialized_start = 1848 - _FETCH._serialized_end = 1948 - _AGGREGATE._serialized_start = 1951 - _AGGREGATE._serialized_end = 2276 - _AGGREGATE_AGGREGATEFUNCTION._serialized_start = 2180 - _AGGREGATE_AGGREGATEFUNCTION._serialized_end = 2276 - _SORT._serialized_start = 2279 - _SORT._serialized_end = 2781 - _SORT_SORTFIELD._serialized_start = 2399 - _SORT_SORTFIELD._serialized_end = 2587 - _SORT_SORTDIRECTION._serialized_start = 2589 - _SORT_SORTDIRECTION._serialized_end = 2697 - _SORT_SORTNULLS._serialized_start = 2699 - _SORT_SORTNULLS._serialized_end = 2781 - _LOCALRELATION._serialized_start = 2783 - _LOCALRELATION._serialized_end = 2876 - _SAMPLE._serialized_start = 2879 - _SAMPLE._serialized_end = 3063 + _READ._serialized_start = 868 + _READ._serialized_end = 1017 + _READ_NAMEDTABLE._serialized_start = 943 + _READ_NAMEDTABLE._serialized_end = 1004 + _PROJECT._serialized_start = 1019 + _PROJECT._serialized_end = 1136 + _FILTER._serialized_start = 1138 + _FILTER._serialized_end = 1250 + _JOIN._serialized_start = 1253 + _JOIN._serialized_end = 1666 + _JOIN_JOINTYPE._serialized_start = 1479 + _JOIN_JOINTYPE._serialized_end = 1666 + _UNION._serialized_start = 1669 + _UNION._serialized_end = 1874 + _UNION_UNIONTYPE._serialized_start = 1790 + _UNION_UNIONTYPE._serialized_end = 1874 + _FETCH._serialized_start = 1876 + _FETCH._serialized_end = 1976 + _AGGREGATE._serialized_start = 1979 + _AGGREGATE._serialized_end = 2304 + _AGGREGATE_AGGREGATEFUNCTION._serialized_start = 2208 + _AGGREGATE_AGGREGATEFUNCTION._serialized_end = 2304 + _SORT._serialized_start = 2307 + _SORT._serialized_end = 2809 + _SORT_SORTFIELD._serialized_start = 2427 + _SORT_SORTFIELD._serialized_end = 2615 + _SORT_SORTDIRECTION._serialized_start = 2617 + _SORT_SORTDIRECTION._serialized_end = 2725 + _SORT_SORTNULLS._serialized_start = 2727 + _SORT_SORTNULLS._serialized_end = 2809 + _LOCALRELATION._serialized_start = 2811 + _LOCALRELATION._serialized_end = 2904 + _SAMPLE._serialized_start = 2907 + _SAMPLE._serialized_end = 3091 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi index 402c08083c7ae..1036414143a01 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.pyi +++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi @@ -259,17 +259,17 @@ class Read(google.protobuf.message.Message): class NamedTable(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor - PARTS_FIELD_NUMBER: builtins.int - @property - def parts( - self, - ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + UNPARSED_IDENTIFIER_FIELD_NUMBER: builtins.int + unparsed_identifier: builtins.str def __init__( self, *, - parts: collections.abc.Iterable[builtins.str] | None = ..., + unparsed_identifier: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal["unparsed_identifier", b"unparsed_identifier"], ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["parts", b"parts"]) -> None: ... NAMED_TABLE_FIELD_NUMBER: builtins.int @property