Using Temporary Columns in Spark

Things on this page are fragmentary and immature notes/thoughts of the author. Please read with your own judgement!

import org.apache.spark.sql.SparkSession

val spark = SparkSession.builder().master("local[2]")
    .appName("Nested")
    .getOrCreate()
spark

org.apache.spark.sql.SparkSession@74a7255c

import spark.implicits._
import org.apache.spark.sql.functions._

org.apache.spark.sql.SparkSession$implicits$@1cdc4527

val df = Seq(
  (8, "bat"),
  (64, "mouse"),
  (-27, "horse")
).toDF("number", "words")
df.show

+------+-----+
|number|words|
+------+-----+
|     8|  bat|
|    64|mouse|
|   -27|horse|
+------+-----+

null

df.withColumn("n2", {
    val tempCol = df("number") / 4
    when(tempCol > 0, tempCol).otherwise(0)
}).show

+------+-----+----+
|number|words|  n2|
+------+-----+----+
|     8|  bat| 2.0|
|    64|mouse|16.0|
|   -27|horse| 0.0|
+------+-----+----+

df.withColumn("n2", {
    val tempCol = $"number" / 4
    when(tempCol > 0, tempCol).otherwise(0)
}).show

+------+-----+----+
|number|words|  n2|
+------+-----+----+
|     8|  bat| 2.0|
|    64|mouse|16.0|
|   -27|horse| 0.0|
+------+-----+----+

df.withColumn("n2", {
    val tempCol = $"number" / 4
    when(tempCol > 0, tempCol).otherwise(0)
}).withColumn("n3", {
    val tempCol = $"n2" / 2
    when(tempCol < 2, tempCol + 1).otherwise(tempCol)
}).show

+------+-----+----+---+
|number|words|  n2| n3|
+------+-----+----+---+
|     8|  bat| 2.0|2.0|
|    64|mouse|16.0|8.0|
|   -27|horse| 0.0|1.0|
+------+-----+----+---+