Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-51334][CONNECT] Add java/scala version in analyze spark_version response #50102

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+162 −126
Open

[SPARK-51334][CONNECT] Add java/scala version in analyze spark_version response #50102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1853892
Add java/scala version in response
garlandz-db Feb 27, 2025
9a965e8
add test
garlandz-db Mar 3, 2025
d81e912
Add comments
garlandz-db Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 125 additions & 125 deletions python/pyspark/sql/connect/proto/base_pb2.py
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 15 additions & 1 deletion python/pyspark/sql/connect/proto/base_pb2.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,14 +813,28 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor

VERSION_FIELD_NUMBER: builtins.int
JAVA_VERSION_FIELD_NUMBER: builtins.int
SCALA_VERSION_FIELD_NUMBER: builtins.int
version: builtins.str
java_version: builtins.str
scala_version: builtins.str
def __init__(
self,
*,
version: builtins.str = ...,
java_version: builtins.str = ...,
scala_version: builtins.str = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["version", b"version"]
self,
field_name: typing_extensions.Literal[
"java_version",
b"java_version",
"scala_version",
b"scala_version",
"version",
b"version",
],
) -> None: ...

class DDLParse(google.protobuf.message.Message):
Expand Down
4 changes: 4 additions & 0 deletions sql/connect/common/src/main/protobuf/spark/connect/base.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,10 @@ message AnalyzePlanResponse {

message SparkVersion {
string version = 1;
// The java version the Spark Connect Service is built with.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this correct, @garlandz-db ?

string java_version = 2;
// The scala version the Spark Connect Service is built with.
string scala_version = 3;
}

message DDLParse {
Expand Down
2 changes: 2 additions & 0 deletions ...rver/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ private[connect] class SparkConnectAnalyzeHandler(
proto.AnalyzePlanResponse.SparkVersion
.newBuilder()
.setVersion(session.version)
.setJavaVersion(System.getProperty("java.version"))
.setScalaVersion(scala.util.Properties.versionNumberString)
.build())

case proto.AnalyzePlanRequest.AnalyzeCase.DDL_PARSE =>
Expand Down
16 changes: 16 additions & 0 deletions ...ver/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectServiceE2ESuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import org.scalatest.concurrent.Eventually
import org.scalatest.time.SpanSugar._

import org.apache.spark.SparkException
import org.apache.spark.connect.proto
import org.apache.spark.sql.connect.SparkConnectServerTest
import org.apache.spark.sql.connect.config.Connect

Expand All @@ -46,6 +47,21 @@ class SparkConnectServiceE2ESuite extends SparkConnectServerTest {
}
}

test("AnalyzePlanRequest for spark version returns java and scala versions") {
withClient { client =>
val request = proto.AnalyzePlanRequest
.newBuilder()
.setSparkVersion(
proto.AnalyzePlanRequest.SparkVersion.newBuilder().build()
)

val response = client.analyze(request)
val sparkVersionResponse = response.getSparkVersion
assert(sparkVersionResponse.getJavaVersion == System.getProperty("java.version"))
assert(sparkVersionResponse.getScalaVersion == scala.util.Properties.versionNumberString)
}
}

test("ReleaseSession releases all queries and does not allow more requests in the session") {
withClient { client =>
val query1 = client.execute(buildPlan(BIG_ENOUGH_QUERY))
Expand Down