(An Untitled Masterwork)
ActivePublic
Actions

Authored by Addshore on Jul 22 2025, 8:24 PM.

Project Tags

None

Referenced Files

	F65589909: raw-paste-data.txt
	Jul 22 2025, 8:24 PM

Subscribers

None

	from wmfdata import spark

	spark = spark.create_custom_session(
	master='yarn',
	spark_config={
	"spark.driver.memory": "16g",
	"spark.driver.maxResultSize": "16g",
	"spark.dynamicAllocation.maxExecutors": "128",
	"spark.executor.memory": "16g",
	"spark.executor.cores": "4",

	# KEEP higher partitions for large data
	"spark.sql.shuffle.partitions": "2048", # Up from 1024 for 250M rows

	# Enable adaptive query execution (crucial for large data)
	"spark.sql.adaptive.enabled": "true",
	"spark.sql.adaptive.coalescePartitions.enabled": "true",
	"spark.sql.adaptive.skewJoin.enabled": "true",

	# Optimize for large datasets
	"spark.sql.adaptive.advisoryPartitionSizeInBytes": "128MB",
	"spark.sql.adaptive.coalescePartitions.minPartitionNum": "32",

	# Larger broadcast threshold for dimension tables
	"spark.sql.autoBroadcastJoinThreshold": "200MB",

	# Better memory management for large data
	"spark.sql.execution.arrow.pyspark.enabled": "true",
	"spark.serializer": "org.apache.spark.serializer.KryoSerializer",

	# FIX: Increase Kryo buffer sizes for large serialization
	"spark.kryoserializer.buffer.max": "512m", # Up from default 64m
	"spark.kryoserializer.buffer": "64k", # Up from default 64k

	# Optimize JSON processing for large volume
	"spark.sql.json.filterPushdown.enabled": "true",
	"spark.sql.parquet.filterPushdown": "true",

	# Better caching for repeated access
	"spark.sql.inMemoryColumnarStorage.compressed": "true",
	"spark.sql.inMemoryColumnarStorage.batchSize": "20000"
	}
	)

Event Timeline