38,907
edits
Apache Spark/en: Difference between revisions
Updating to match new version of source page
No edit summary |
(Updating to match new version of source page) |
||
| Line 22: | Line 22: | ||
module load spark/2.3.0 | module load spark/2.3.0 | ||
module load python/ | module load python/3.7 | ||
# Recommended settings for calling Intel MKL routines from multi-threaded applications | # Recommended settings for calling Intel MKL routines from multi-threaded applications | ||
| Line 29: | Line 29: | ||
export SPARK_IDENT_STRING=$SLURM_JOBID | export SPARK_IDENT_STRING=$SLURM_JOBID | ||
export SPARK_WORKER_DIR=$SLURM_TMPDIR | export SPARK_WORKER_DIR=$SLURM_TMPDIR | ||
export SLURM_SPARK_MEM=$(printf "%.0f" $((${SLURM_MEM_PER_NODE} * | export SLURM_SPARK_MEM=$(printf "%.0f" $((${SLURM_MEM_PER_NODE} *95/100))) | ||
| Line 46: | Line 46: | ||
stop-master.sh | stop-master.sh | ||
}} | }} | ||
== Java Jars == | == Java Jars == | ||
{{File | {{File | ||
| Line 67: | Line 68: | ||
export SPARK_IDENT_STRING=$SLURM_JOBID | export SPARK_IDENT_STRING=$SLURM_JOBID | ||
export SPARK_WORKER_DIR=$SLURM_TMPDIR | export SPARK_WORKER_DIR=$SLURM_TMPDIR | ||
export SLURM_SPARK_MEM=$(printf "%.0f" $((${SLURM_MEM_PER_NODE} * | export SLURM_SPARK_MEM=$(printf "%.0f" $((${SLURM_MEM_PER_NODE} *95/100))) | ||
start-master.sh | start-master.sh | ||
| Line 84: | Line 85: | ||
stop-master.sh | stop-master.sh | ||
}} | }} | ||
= Monitoring = | = Monitoring = | ||