Apache Spark: Difference between revisions

Apache Spark (view source)

904 bytes added , 7 years ago

cc_staff

505

edits

@@ Line 15: / Line 15: @@
 = Utilisation =
+== PySpark ==
 {{File
    |name=pyspark_submit.sh
@@ Line 29: / Line 30: @@
 module load spark/2.2.0
 module load python/2.7.13
+export SPARK_IDENT_STRING=$SLURM_JOBID
+export SPARK_WORKER_DIR=$SLURM_TMPDIR
+start-master.sh
+sleep 1
+MASTER_URL=$(grep -Po '(?=spark://).*' $SPARK_LOG_DIR/spark-${SPARK_IDENT_STRING}-org.apache.spark.deploy.master*.out)
+NWORKERS=$((SLURM_NTASKS - 1))
+SPARK_NO_DAEMONIZE=1 srun -n ${NWORKERS} -N ${NWORKERS} --label --output=$SPARK_LOG_DIR/spark-%j-workers.out start-slave.sh -m ${SLURM_MEM_PER_NODE}M -c ${SLURM_CPUS_PER_TASK} ${MASTER_URL} &
+slaves_pid=$!
+srun -n 1 -N 1 spark-submit --master ${MASTER_URL} --executor-memory ${SLURM_MEM_PER_NODE}M $SPARK_HOME/examples/src/main/python/pi.py
+kill $slaves_pid
+stop-master.sh
+}}
+== Java Jars  ==
+{{File
+  |name=pyspark_submit.sh
+  |lang="sh"
+  |contents=
+#!/bin/bash
+#SBATCH --account=def-someuser
+#SBATCH --time=00:01:00
+#SBATCH --nodes=4
+#SBATCH --mem=4G
+#SBATCH --cpus-per-task=8
+#SBATCH --ntasks-per-node=1
+module load spark/2.2.0
 export SPARK_IDENT_STRING=$SLURM_JOBID