cc_staff
123
edits
Apache Spark: Difference between revisions
Changing "examples_2.11-2.2.0.jar" into "examples_2.11-2.3.0.jar" as module spark/2.3 is loaded, otherwise getting error: file not found
(*0.95->*95/100 because bash doesn't support decimal points) |
(Changing "examples_2.11-2.2.0.jar" into "examples_2.11-2.3.0.jar" as module spark/2.3 is loaded, otherwise getting error: file not found) |
||
| Line 84: | Line 84: | ||
SLURM_SPARK_SUBMIT="srun -n 1 -N 1 spark-submit --master ${MASTER_URL} --executor-memory ${SLURM_SPARK_MEM}M" | SLURM_SPARK_SUBMIT="srun -n 1 -N 1 spark-submit --master ${MASTER_URL} --executor-memory ${SLURM_SPARK_MEM}M" | ||
$SLURM_SPARK_SUBMIT --class org.apache.spark.examples.SparkPi $SPARK_HOME/examples/jars/spark-examples_2.11-2. | $SLURM_SPARK_SUBMIT --class org.apache.spark.examples.SparkPi $SPARK_HOME/examples/jars/spark-examples_2.11-2.3.0.jar 1000 | ||
$SLURM_SPARK_SUBMIT --class org.apache.spark.examples.SparkLR $SPARK_HOME/examples/jars/spark-examples_2.11-2. | $SLURM_SPARK_SUBMIT --class org.apache.spark.examples.SparkLR $SPARK_HOME/examples/jars/spark-examples_2.11-2.3.0.jar 1000 | ||
kill $slaves_pid | kill $slaves_pid | ||