Summary/Reported Issue
When using the ArrowFlightJdbcDriver and trying to connect to Dremio the following error occurs
py4j.protocol.Py4JJavaError: An error occurred while calling o40.load.
: java.sql.SQLException: org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightRuntimeException: UNAVAILABLE: Network closed for unknown reason
at org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler$Builder.build(ArrowFlightSqlClientHandler.java:914)
at org.apache.arrow.driver.jdbc.ArrowFlightConnection.createNewClientHandler(ArrowFlightConnection.java:116)
at org.apache.arrow.driver.jdbc.ArrowFlightConnection.createNewConnection(ArrowFlightConnection.java:88)
at org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver.connect(ArrowFlightJdbcDriver.java:90)
at org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver.connect(ArrowFlightJdbcDriver.java:46)
at org.apache.spark.sql.execution.datasources.jdbc.connection.BasicConnectionProvider.getConnection(BasicConnectionProvider.scala:49)
at org.apache.spark.sql.execution.datasources.jdbc.connection.ConnectionProviderBase.create(ConnectionProvider.scala:102)
at org.apache.spark.sql.jdbc.JdbcDialect.$anonfun$createConnectionFactory$1(JdbcDialects.scala:160)
at org.apache.spark.sql.jdbc.JdbcDialect.$anonfun$createConnectionFactory$1$adapted(JdbcDialects.scala:156)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.getQueryOutputSchema(JDBCRDD.scala:63)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:241)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:37)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:346)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:229)
at org.apache.spark.sql.DataFrameReader.$anonfun$load$2(DataFrameReader.scala:211)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:172)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:750)
Caused by: org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightRuntimeException: UNAVAILABLE: Network closed for unknown reason
at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallStatus.toRuntimeException(CallStatus.java:121)
at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.grpc.StatusUtils.fromGrpcRuntimeException(StatusUtils.java:161)
at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.grpc.StatusUtils.fromThrowable(StatusUtils.java:182)
at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.auth2.ClientHandshakeWrapper.doClientHandshake(ClientHandshakeWrapper.java:55)
at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient.handshake(FlightClient.java:212)
at org.apache.arrow.driver.jdbc.client.utils.ClientAuthenticationUtils.getAuthenticate(ClientAuthenticationUtils.java:107)
at org.apache.arrow.driver.jdbc.client.utils.ClientAuthenticationUtils.getAuthenticate(ClientAuthenticationUtils.java:92)
at org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler$Builder.build(ArrowFlightSqlClientHandler.java:897)
... 30 more
Steps to Resolve
Check your connection string for a rogue ; at the end of the connection like the example below
arrow_jdbc_url = "jdbc:arrow-flight-sql://myDremio:32010/?disableCertificateVerification=true&useEncryption=true;"
Removing the ; addresses the issue.
arrow_jdbc_url = "jdbc:arrow-flight-sql://myDremio:32010/?disableCertificateVerification=true&useEncryption=true"