over

Apply a window specification to the column.

Syntax

Python

over(window)

Parameters

Parameter	Type	Description
window	WindowSpec	Window specification

Returns

Column

Examples

Python

from pyspark.sql import Window
from pyspark.sql.functions import rank, min, desc
window = (
    Window.partitionBy("name")
    .orderBy("age")
    .rowsBetween(Window.unboundedPreceding, Window.currentRow)
)
df = spark.createDataFrame(
     [(2, "Alice"), (5, "Bob")], ["age", "name"])
df.withColumn(
     "rank", rank().over(window)
).withColumn(
     "min", min('age').over(window)
).sort(desc("age")).show()

Output

# +---+-----+----+---+
# |age| name|rank|min|
# +---+-----+----+---+
# |  5|  Bob|   1|  5|
# |  2|Alice|   1|  2|
# +---+-----+----+---+