The filter Function in Spark DataFrame

Things on this page are fragmentary and immature notes/thoughts of the author. Please read with your own judgement!

https://spark.apache.org/docs/latest/sql-programming-guide.html

https://spark.apache.org/docs/latest/api/scala/index.html#org.apache.spark.sql.functions$

%%classpath add mvn
org.apache.spark spark-core_2.11 2.1.1
org.apache.spark spark-sql_2.11 2.1.1

import org.apache.spark.sql.SparkSession

val spark = SparkSession.
    builder().
    master("local").
    appName("Spark SQL basic example").
    config("spark.some.config.option", "some-value").
    getOrCreate()
spark

import spark.implicits._

org.apache.spark.sql.SparkSession$implicits$@7e12bd6b

val df = spark.read.json("../../data/people.json")
df.show

+----+-------+
| age|   name|
+----+-------+
|null|Michael|
|  30|   Andy|
|  19| Justin|
+----+-------+

null

Filtering¶

df.filter($"age" > 21).show()

+---+----+
|age|name|
+---+----+
| 30|Andy|
+---+----+

null

import org.apache.spark.sql.Column


def greaterThan(column: Column, v: Int):
    Column = {column > v}

import org.apache.spark.sql.Column
greaterThan: (column: org.apache.spark.sql.Column, v: Int)org.apache.spark.sql.Column

df.filter(greaterThan(df("age"), 21)).show

+---+----+
|age|name|
+---+----+
| 30|Andy|
+---+----+

null

df.filter(greaterThan($"age", 21)).show

+---+----+
|age|name|
+---+----+
| 30|Andy|
+---+----+

null

df.filter($"age" === 30).show

+---+----+
|age|name|
+---+----+
| 30|Andy|
+---+----+

null

df.filter($"age" === null).show

+---+----+
|age|name|
+---+----+
+---+----+

null

df.filter($"age" <=> 30).show

+---+----+
|age|name|
+---+----+
| 30|Andy|
+---+----+

null

df.filter($"age" <=> null).show

+----+-------+
| age|   name|
+----+-------+
|null|Michael|
+----+-------+

null