From 1eb6024fd090fec8e48544d4241b1026e223a07e Mon Sep 17 00:00:00 2001
From: Rishabh Saxena <rishabh.saxena@hlrs.de>
Date: Wed, 22 May 2024 11:43:24 +0200
Subject: [PATCH] changes made accroding to issue #1

---
 .gitignore                             |  3 ++-
 README.md                              | 26 +++++++++++++-------
 deployment_scripts/environment.yaml    |  9 -------
 deployment_scripts/submit-dask-job.pbs | 33 ++++++++++++++++++++++++++
 4 files changed, 53 insertions(+), 18 deletions(-)
 delete mode 100644 deployment_scripts/environment.yaml
 create mode 100644 deployment_scripts/submit-dask-job.pbs

diff --git a/.gitignore b/.gitignore
index e0cce53..83fbde5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 __pycache__
 notebooks/
 /deployment_scripts/create-env.sh
-/deployment_scripts/deploy-env.sh
\ No newline at end of file
+/deployment_scripts/deploy-env.sh
+/deployment_scripts/environment.yaml
\ No newline at end of file
diff --git a/README.md b/README.md
index ef21140..8781778 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,21 @@ source deploy-dask.sh "$(pwd)"
 ## Usage
 
 ### Single Node
-To run the application interactively on a single node, execute the following command after all the cluster's nodes are up and running:
+To run the application interactively on a single node, follow points 4, 5, 6 and, 7 from [Getting Started](#getting-started), and execute the following command after all the job has started:
+
+```bash
+# Load the Conda module
+module load bigdata/conda
+source activate # activates the base environment
+
+# List available Conda environments for verification purposes
+conda env list
+
+# Activate a specific Conda environment.
+conda activate dask_environment # you need to execute `source activate` first, or use `source [ENV_PATH]/bin/activate`
+```
+
+After the environment is activated, you can run the python interpretor:
 
 ```bash
 python
@@ -72,19 +86,13 @@ Or to run a full script:
 python <your-script>.py
 ```
 
-Note: If you don't see your environment in the python interpretor, then manually activate it using:
-```bash
-conda activate <your-env>
-```
-Do this before using the python interpretor.
-
 ### Multiple Nodes
 To run the application on multiple nodes, you need to write a `.pbs` script and submit it using `qsub`. Follow lines 1-4 from the [Getting Started](#getting-started) section. Write a `submit-dask-job.pbs` script:
 
 ```bash
 #!/bin/bash
 #PBS -N dask-job
-#PBS -l select=3:node_type=rome-ai
+#PBS -l select=3:node_type=rome
 #PBS -l walltime=1:00:00
 
 #Go to the directory where the code is
@@ -97,6 +105,8 @@ source deploy-dask.sh "$(pwd)"
 python <your-script>.py
 ```
 
+A more thorough example is available in the `deployment_scripts` directory under `submit-dask-job.pbs`.
+
 And then execute the following commands to submit the job:
 
 ```bash
diff --git a/deployment_scripts/environment.yaml b/deployment_scripts/environment.yaml
deleted file mode 100644
index 862e77c..0000000
--- a/deployment_scripts/environment.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.8.18
-  - dask
-  - numpy
-  - scikit-learn
-  - conda-pack
diff --git a/deployment_scripts/submit-dask-job.pbs b/deployment_scripts/submit-dask-job.pbs
new file mode 100644
index 0000000..61c6c5f
--- /dev/null
+++ b/deployment_scripts/submit-dask-job.pbs
@@ -0,0 +1,33 @@
+#!/bin/bash
+#PBS -N dask-job
+#PBS -l select=2:node_type=rome
+#PBS -l walltime=1:00:00
+
+export PYTHON_FILE= # Path to the Python file you want to run
+export CURRENT_WORKSPACE= # Path to the workspace where you have pulled this repo and the dask-env.tar.gz file
+
+export ALL_NODES=$(cat $PBS_NODEFILE)
+export SCHEDULER_NODE="$(head -n1 $PBS_NODEFILE)-ib"
+export WORKER_NODES=$(tail -n+2 $PBS_NODEFILE)
+
+export DASK_SCHEDULER_PORT=8786
+export DASK_UI_PORT=8787
+
+export DASK_ENV="$HOME/dask"
+mkdir -p $DASK_ENV
+tar -xzf $CURRENT_WORKSPACE/dask-env.tar.gz -C $DASK_ENV
+chmod -R 700 $DASK_ENV
+
+source $DASK_ENV/bin/activate
+conda-unpack
+
+dask scheduler --host $SCHEDULER_NODE --port $DASK_SCHEDULER_PORT &
+export NUM_NODES=$(sort $PBS_NODEFILE |uniq | wc -l)
+
+# Assuming you have a Dask worker script named 'dask-worker-script.py', modify this accordingly
+for ((i=1;i<$NUM_NODES;i++)); do
+    echo "[$(date '+%Y-%m-%d %H:%M:%S') - Master] INFO: Starting Dask Worker at $i"
+	pbsdsh -n $i -o -- bash -l -c "source /deplyment_scripts/dask-worker.sh $CURRENT_WORKSPACE $SCHEDULER_NODE"
+done
+
+python3 $PYTHON_FILE
\ No newline at end of file