Skip to content

Commit

Permalink
Revert "[SPARK-48567][SS] StreamingQuery.lastProgress should return t…
Browse files Browse the repository at this point in the history
…he actual StreamingQueryProgress"

This reverts commit 042804a.
  • Loading branch information
HyukjinKwon committed Jun 19, 2024
1 parent 1e868b2 commit d067fc6
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 227 deletions.
9 changes: 4 additions & 5 deletions python/pyspark/sql/connect/streaming/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
QueryProgressEvent,
QueryIdleEvent,
QueryTerminatedEvent,
StreamingQueryProgress,
)
from pyspark.sql.streaming.query import (
StreamingQuery as PySparkStreamingQuery,
Expand Down Expand Up @@ -111,21 +110,21 @@ def status(self) -> Dict[str, Any]:
status.__doc__ = PySparkStreamingQuery.status.__doc__

@property
def recentProgress(self) -> List[StreamingQueryProgress]:
def recentProgress(self) -> List[Dict[str, Any]]:
cmd = pb2.StreamingQueryCommand()
cmd.recent_progress = True
progress = self._execute_streaming_query_cmd(cmd).recent_progress.recent_progress_json
return [StreamingQueryProgress.fromJson(json.loads(p)) for p in progress]
return [json.loads(p) for p in progress]

recentProgress.__doc__ = PySparkStreamingQuery.recentProgress.__doc__

@property
def lastProgress(self) -> Optional[StreamingQueryProgress]:
def lastProgress(self) -> Optional[Dict[str, Any]]:
cmd = pb2.StreamingQueryCommand()
cmd.last_progress = True
progress = self._execute_streaming_query_cmd(cmd).recent_progress.recent_progress_json
if len(progress) > 0:
return StreamingQueryProgress.fromJson(json.loads(progress[-1]))
return json.loads(progress[-1])
else:
return None

Expand Down
Loading

0 comments on commit d067fc6

Please sign in to comment.