#!/bin/bash
# parameters for slurm
#SBATCH -J gpu-burn                   # job name, don't use spaces, keep it short
#SBATCH -c 2                          # number of cores, 1
#SBATCH --gres=gpu:1                  # number of gpus 1, some clusters don't have GPUs
#SBATCH --mem=1gb                     # Job memory request
#SBATCH --mail-type=END,FAIL          # email status changes (NONE, BEGIN, END, FAIL, ALL)
#SBATCH --mail-user=myname@utwente.nl   # Where to send mail to
#SBATCH --time=1:00:00                # time limit 1h
#SBATCH --output=job_test_%j.log      # Standard output and error log
#SBATCH --error=%j.err                # if yoou want the errors logged seperately
#SBATCH --partition=50_procent_max_7_days # Here 50..is the partition name..can be checked via sinfo

# Create a directory for this job on the node
cd /local
mkdir ${SLURM_JOBID}
cd ${SLURM_JOBID}
# Copy input and executable to the node
cp -r ${SLURM_SUBMIT_DIR}/input/* .

# load all modules needed 
module load nvidia/cuda-10.1 
module load mpi/openmpi-x86_64

# It's nice to have some information logged for debugging
echo "Date              = $(date)"
echo "Hostname          = $(hostname -s)" # log hostname
echo "Working Directory = $(pwd)"
echo "Number of nodes used        : "$SLURM_NNODES
echo "Number of MPI ranks         : "$SLURM_NTASKS
echo "Number of threads           : "$SLURM_CPUS_PER_TASK
echo "Number of MPI ranks per node: "$SLURM_TASKS_PER_NODE
echo "Number of threads per core  : "$SLURM_THREADS_PER_CORE
echo "Name of nodes used          : "$SLURM_JOB_NODELIST
echo "Gpu devices                 : "$CUDA_VISIBLE_DEVICES
echo "Starting worker: "

caseName=${PWD##*/} # to distinguish several log files
# Run the job -- make sure that it terminates itself before time is up
# Do not submit into the background (i.e. no & at the end of the line).
mpirun comsol batch -in inputFile > loga_$caseName.out

# Copy output back to the master, comment with # if not used
cp log_file.txt ${SLURM_SUBMIT_DIR}
cp simulation_data.csv ${SLURM_SUBMIT_DIR}
cp warnings_data.txt ${SLURM_SUBMIT_DIR}
mv output ${SLURM_SUBMIT_DIR}
# Clean up on the node ! make sure you are still on the node...
#rm *
#cd ..
#rmdir ${SLURM_JOBID}

# Done.