cc_staff
505
edits
Apache Spark: Difference between revisions
Jump to navigation
Jump to search
→Utilisation
No edit summary |
|||
| Line 12: | Line 12: | ||
= Utilisation = | = Utilisation = | ||
{{File | |||
|name=pyspark_submit.sh | |||
|lang="sh" | |||
|contents= | |||
#!/bin/bash | |||
#SBATCH --account=def-someuser | |||
#SBATCH --time=00:01:00 | |||
#SBATCH --job-name=test | |||
#SBATCH --output=%x-%j.out | |||
module load spark/2.2.0 | |||
module load python/2.7.13 | |||
export SPARK_IDENT_STRING=$SLURM_JOBID | |||
export SPARK_WORKER_DIR=$SLURM_TMPDIR | |||
start-master.sh | |||
( | |||
export SPARK_NO_DAEMONIZE=1; | |||
export -n HOSTNAME; | |||
srun -x $(hostname) -n $((SLURM_NTASKS -1)) --label --output=$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-workers.out | |||
start-slave.sh -m ${SLURM_MEM_PER_NODE}M -c ${SLURM_CPUS_PER_TASK} spark://$(hostname -f):7077 | |||
) & | |||
spark-submit --executor-memory ${SPARK_MEM_PER_NODE}M $SPARK_HOME/examples/src/main/python/pi.py 100000 | |||
srun stop-slave.sh | |||
stop-master.sh | |||
}} | |||