change spark config

bpblanken · bpblanken · commit 95acd6d24734 · 2024-10-10T15:42:21.000-04:00
diff --git a/v03_pipeline/var/spark_config/spark-defaults.conf b/v03_pipeline/var/spark_config/spark-defaults.conf
@@ -1,8 +1,8 @@
-spark.driver.memory 5G
-spark.master local[2]
+spark.driver.memory 10G
+spark.master local[*]
 spark.driver.extraJavaOptions -Xss4M
 spark.executor.extraJavaOptions -Xss4M
-spark.executor.memoryOverhead 5g
+spark.executor.memoryOverhead 4g
 spark.driver.maxResultSize 30g
 spark.kryoserializer.buffer.max 1g
 spark.memory.fraction 0.1
