2023-12-07 09:26:25 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
|
|
|
|
#Get the current workspace directory and the master node
|
|
|
|
|
export CURRENT_WORKSPACE=$1
|
|
|
|
|
export DASK_SCHEDULER_HOST=$2
|
|
|
|
|
|
|
|
|
|
# Path to localscratch
|
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Setting up Dask environment"
|
2024-04-25 12:18:00 +00:00
|
|
|
export DASK_ENV="$HOME/dask"
|
2023-12-07 09:26:25 +00:00
|
|
|
mkdir -p $DASK_ENV
|
|
|
|
|
|
|
|
|
|
# Extract Dask environment in localscratch
|
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Extracting Dask environment to $DASK_ENV"
|
2024-04-25 12:18:00 +00:00
|
|
|
#tar -xzf $CURRENT_WORKSPACE/dask-env.tar.gz -C $DASK_ENV
|
|
|
|
|
#chmod -R 700 $DASK_ENV
|
2023-12-07 09:26:25 +00:00
|
|
|
|
|
|
|
|
# Start the dask environment
|
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Setting up Dask environment"
|
|
|
|
|
source $DASK_ENV/bin/activate
|
|
|
|
|
conda-unpack
|
|
|
|
|
|
|
|
|
|
# Start Dask worker
|
|
|
|
|
export DASK_SCHEDULER_PORT="8786" # Replace with the port on which the Dask scheduler is running
|
|
|
|
|
|
|
|
|
|
# Additional Dask worker options can be added here if needed
|
|
|
|
|
# Change local directory if memory is an issue
|
|
|
|
|
|
|
|
|
|
# Change directory to localscratch and start Dask worker
|
|
|
|
|
cd $DASK_ENV
|
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S') - Worker $HOSTNAME] INFO: Starting Dask worker at $DASK_SCHEDULER_HOST on port $DASK_SCHEDULER_PORT"
|
|
|
|
|
dask worker $DASK_SCHEDULER_HOST:$DASK_SCHEDULER_PORT
|
