databricks - identifying whether native query was set to TRUE or FALSE

[edward-burn]

I think at the moment there is no way to easily tell whether a databricks connection has native query set to TRUE or FALSE (unless I'm missing it). I was wondering if this could be included as an attribute of the connection object? This would be very useful for packages taking the connection object as an input to a function (and would allow for informative errors if, for example, the user tries to use DBI::dbWriteTable() when native query was set to TRUE)

con_non_native <- DBI::dbConnect(
  odbc::databricks(),
  httpPath = Sys.getenv("DATABRICKS_HTTPPATH"),
  useNativeQuery = FALSE
)
con_native <- DBI::dbConnect(
  odbc::databricks(),
  httpPath = Sys.getenv("DATABRICKS_HTTPPATH"),
  useNativeQuery = TRUE
)

waldo::compare(
  DBI::dbGetInfo(con_non_native),
  DBI::dbGetInfo(con_non_native))
#> ✔ No differences

DBI::dbWriteTable(con_native, "cars", cars, overwrite = TRUE)
#> Error: nanodbc/nanodbc.cpp:1772: 42601
#> [Simba][Hardy] (80) Syntax or semantic analysis error thrown in server while executing query. Error message from server: org.apache.hive.service.cli.HiveSQLException: Error running query: [PARSE_SYNTAX_ERROR] org.apache.spark.sql.catalyst.parser.ParseException: 
#> [PARSE_SYNTAX_ERROR] Syntax error at or near '?'.(line 2, pos 8)
#> 
#> == SQL ==
#> INSERT INTO `cars` (`speed`, `dist`)
#> VALUES (?, ?)
#> --------^^^
#> 
#>  at org.apache.spark.sql.hive.thriftserver.HiveThriftServerErrors$.runningQueryError(HiveThriftServerErrors.scala:48)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$execute$1(SparkExecuteStatementOperation.scala:619)
#>  at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
#>  at com.databricks.unity.EmptyHandle$.runWith(UCSHandle.scala:125)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:509)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:369)
#>  at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
#>  at com.databricks.spark.util.IdentityClaim$.withClaim(IdentityClaim.scala:48)
#>  at org.apache.spark.sql.hive.thriftserver.ThriftLocalProperties.withLocalProperties(ThriftLocalProperties.scala:156)
#>  at org.apache.spark.sql.hive.thriftserver.ThriftLocalProperties.withLocalProperties$(ThriftLocalProperties.scala:51)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:63)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:347)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:332)
#>  at java.security.AccessController.doPrivileged(Native Method)
#>  at javax.security.auth.Subject.doAs(Subject.java:422)
#>  at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:381)
#>  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
#>  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
#>  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
#>  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
#>  at java.lang.Thread.run(Thread.java:750)
#> Caused by: org.apache.spark.sql.catalyst.parser.ParseException: 
#> [PARSE_SYNTAX_ERROR] Syntax error at or near '?'.(line 2, pos 8)
#> 
#> == SQL ==
#> INSERT INTO `cars` (`speed`, `dist`)
#> VALUES (?, ?)
#> --------^^^
#> 
#>  at org.apache.spark.sql.catalyst.parser.ParseException.withCommand(ParseDriver.scala:334)
#>  at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:165)
#>  at org.apache.spark.sql.execution.SparkSqlParser.parse(SparkSqlParser.scala:94)
#>  at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:110)
#>  at com.databricks.sql.parser.DatabricksSqlParser.$anonfun$parsePlan$1(DatabricksSqlParser.scala:77)
#>  at com.databricks.sql.parser.DatabricksSqlParser.parse(DatabricksSqlParser.scala:98)
#>  at com.databricks.sql.parser.DatabricksSqlParser.parsePlan(DatabricksSqlParser.scala:74)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$analyzeQuery$3(SparkExecuteStatementOperation.scala:487)
#>  at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:80)
#>  at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:352)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$analyzeQuery$2(SparkExecuteStatementOperation.scala:486)
#>  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1073)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$analyzeQuery$1(SparkExecuteStatementOperation.scala:484)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.getOrCreateDF(SparkExecuteStatementOperation.scala:474)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.analyzeQuery(SparkExecuteStatementOperation.scala:484)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$execute$5(SparkExecuteStatementOperation.scala:544)
#>  at org.apache.spark.util.Utils$.timeTakenMs(Utils.scala:708)
#>  at org.apache.spark.sql.execution.qrc.CacheEventLogger.recordLatency(QueryResultCache.scala:149)
#>  at org.apache.spark.sql.execution.qrc.CacheEventLogger.recordLatency$(QueryResultCache.scala:145)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.recordLatency(SparkExecuteStatementOperation.scala:63)
#>  at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$execute$1(SparkExecuteStatementOperation.scala:544)
#>  ... 20 more
#>  
#> <SQL> 'INSERT INTO `cars` (`speed`, `dist`)
#> VALUES (?, ?)'

^{Created on 2025-06-28 with reprex v2.0.2}