forked from SiVeGCS/dask_template
32 lines
1.2 KiB
Bash
32 lines
1.2 KiB
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
#Get the current workspace directory and the master node
|
||
|
|
export CURRENT_WORKSPACE=$1
|
||
|
|
export DASK_SCHEDULER_HOST=$2
|
||
|
|
|
||
|
|
# Path to localscratch
|
||
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Setting up Dask environment"
|
||
|
|
export DASK_ENV="/localscratch/${PBS_JOBID}/dask"
|
||
|
|
mkdir -p $DASK_ENV
|
||
|
|
|
||
|
|
# Extract Dask environment in localscratch
|
||
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Extracting Dask environment to $DASK_ENV"
|
||
|
|
tar -xzf $CURRENT_WORKSPACE/dask-env.tar.gz -C $DASK_ENV
|
||
|
|
chmod -R 700 $DASK_ENV
|
||
|
|
|
||
|
|
# Start the dask environment
|
||
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Setting up Dask environment"
|
||
|
|
source $DASK_ENV/bin/activate
|
||
|
|
conda-unpack
|
||
|
|
|
||
|
|
# Start Dask worker
|
||
|
|
export DASK_SCHEDULER_PORT="8786" # Replace with the port on which the Dask scheduler is running
|
||
|
|
|
||
|
|
# Additional Dask worker options can be added here if needed
|
||
|
|
# Change local directory if memory is an issue
|
||
|
|
|
||
|
|
# Change directory to localscratch and start Dask worker
|
||
|
|
cd $DASK_ENV
|
||
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Starting Dask worker at $DASK_SCHEDULER_HOST on port $DASK_SCHEDULER_PORT"
|
||
|
|
dask worker $DASK_SCHEDULER_HOST:$DASK_SCHEDULER_PORT
|