"Cannot evaluate expression: NamePlaceholder" when extracting Avro schema from multiple columns

import org.apache.spark.sql.DataFrame
import org.scalatest.FunSuite
import za.co.absa.utils.spark.test.SparkSessionFixture
import org.apache.spark.sql.avro.SchemaConverters.toAvroType
import org.apache.spark.sql.functions.struct

case class TestData(name: String, id: Int)

class TestConversion extends FunSuite with SparkSessionFixture {

  test(testName = "whatever") {
    val data = getTestDataFrame()
    data.show()
    print(getSchema(data))
  }

  private def getTestDataFrame(): DataFrame = {
    val testData = Seq(
      TestData("a", 1),
      TestData("b", 2),
      TestData("c", 3)
    )

    spark.createDataFrame(testData)
  }

  private def getSchema(data: DataFrame) = {
    val columns = struct(data.columns.head, data.columns.tail: _*)
    val expression = columns.expr
    toAvroType(expression.dataType, expression.nullable)
  }
}

is throwing

Cannot evaluate expression: NamePlaceholder
java.lang.UnsupportedOperationException: Cannot evaluate expression: NamePlaceholder
	at org.apache.spark.sql.catalyst.expressions.Unevaluable$class.eval(Expression.scala:258)
	at org.apache.spark.sql.catalyst.expressions.NamePlaceholder$.eval(complexTypeCreator.scala:317)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStructLike$$anonfun$names$1.apply(complexTypeCreator.scala:363)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStructLike$$anonfun$names$1.apply(complexTypeCreator.scala:363)
	at scala.collection.immutable.List.map(List.scala:273)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStructLike$class.names(complexTypeCreator.scala:363)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStruct.names$lzycompute(complexTypeCreator.scala:425)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStruct.names(complexTypeCreator.scala:425)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStructLike$class.dataType(complexTypeCreator.scala:370)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStruct.dataType$lzycompute(complexTypeCreator.scala:425)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStruct.dataType(complexTypeCreator.scala:425)
	at org.apache.spark.sql.catalyst.expressions.CreateNamedStruct.dataType(complexTypeCreator.scala:425)

Probably because running

val expression = columns.expr

is resulting in

named_struct(NamePlaceHolder(), name, NamePlaceHolder(), id)

[cerveada]

dataFrame.columns.map(col) This creates a new generic column instead of getting one from the dataframe. The column is not associated with dataframe so it makes sense it doesn’t work.

col Is supposed to be used inside select.

[kevinwallimann]

That makes sense, however I also tried using

import org.apache.spark.sql.functions.col
val mapColumns: Array[Column] = dataFrame.columns.map(col)

which fails with the NamePlaceholder exception.

So it can’t be only because of the Column type, there must be something implementation-specific, but I couldn’t figure out yet, what it is.